# HG changeset patch
# Parent 8464bb6dfd0210f5e1a72a938f3dd16b1aad78ca
[Clean-up] Remove most of the rust meat
diff --git a/Kbuild b/Kbuild
--- a/Kbuild
+++ b/Kbuild
@@ -89,7 +89,6 @@ obj-y += security/
obj-y += crypto/
obj-$(CONFIG_BLOCK) += block/
obj-$(CONFIG_IO_URING) += io_uring/
-obj-$(CONFIG_RUST) += rust/
obj-y += $(ARCH_LIB)
obj-y += drivers/
obj-y += sound/
diff --git a/rust/.gitignore b/rust/.gitignore
deleted file mode 100644
--- a/rust/.gitignore
+++ /dev/null
@@ -1,10 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-bindings_generated.rs
-bindings_helpers_generated.rs
-doctests_kernel_generated.rs
-doctests_kernel_generated_kunit.c
-uapi_generated.rs
-exports_*_generated.h
-doc/
-test/
diff --git a/rust/Makefile b/rust/Makefile
deleted file mode 100644
--- a/rust/Makefile
+++ /dev/null
@@ -1,463 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-# Where to place rustdoc generated documentation
-rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc
-
-obj-$(CONFIG_RUST) += core.o compiler_builtins.o
-always-$(CONFIG_RUST) += exports_core_generated.h
-
-# Missing prototypes are expected in the helpers since these are exported
-# for Rust only, thus there is no header nor prototypes.
-obj-$(CONFIG_RUST) += helpers.o
-CFLAGS_REMOVE_helpers.o = -Wmissing-prototypes -Wmissing-declarations
-
-always-$(CONFIG_RUST) += libmacros.so
-no-clean-files += libmacros.so
-
-always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs
-obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o
-always-$(CONFIG_RUST) += exports_alloc_generated.h exports_bindings_generated.h \
- exports_kernel_generated.h
-
-always-$(CONFIG_RUST) += uapi/uapi_generated.rs
-obj-$(CONFIG_RUST) += uapi.o
-
-ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW
-obj-$(CONFIG_RUST) += build_error.o
-else
-always-$(CONFIG_RUST) += build_error.o
-endif
-
-obj-$(CONFIG_RUST) += exports.o
-
-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs
-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c
-
-obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o
-obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o
-
-# Avoids running `$(RUSTC)` for the sysroot when it may not be available.
-ifdef CONFIG_RUST
-
-# `$(rust_flags)` is passed in case the user added `--sysroot`.
-rustc_sysroot := $(shell MAKEFLAGS= $(RUSTC) $(rust_flags) --print sysroot)
-rustc_host_target := $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2)
-RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library
-
-ifeq ($(quiet),silent_)
-cargo_quiet=-q
-rust_test_quiet=-q
-rustdoc_test_quiet=--test-args -q
-rustdoc_test_kernel_quiet=>/dev/null
-else ifeq ($(quiet),quiet_)
-rust_test_quiet=-q
-rustdoc_test_quiet=--test-args -q
-rustdoc_test_kernel_quiet=>/dev/null
-else
-cargo_quiet=--verbose
-endif
-
-core-cfgs = \
- --cfg no_fp_fmt_parse
-
-alloc-cfgs = \
- --cfg no_global_oom_handling \
- --cfg no_rc \
- --cfg no_sync
-
-quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $<
- cmd_rustdoc = \
- OBJTREE=$(abspath $(objtree)) \
- $(RUSTDOC) $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags)) \
- $(rustc_target_flags) -L$(objtree)/$(obj) \
- --output $(rustdoc_output) \
- --crate-name $(subst rustdoc-,,$@) \
- $(if $(rustdoc_host),,--sysroot=/dev/null) \
- @$(objtree)/include/generated/rustc_cfg $<
-
-# The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute
-# can be used to specify a custom logo. However:
-# - The given value is used as-is, thus it cannot be relative or a local file
-# (unlike the non-custom case) since the generated docs have subfolders.
-# - It requires adding it to every crate.
-# - It requires changing `core` which comes from the sysroot.
-#
-# Using `-Zcrate-attr` would solve the last two points, but not the first.
-# The https://github.com/rust-lang/rfcs/pull/3226 RFC suggests two new
-# command-like flags to solve the issue. Meanwhile, we use the non-custom case
-# and then retouch the generated files.
-rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \
- rustdoc-alloc rustdoc-kernel
- $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/
- $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/
- $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \
- -e 's:rust-logo-[0-9a-f]+\.svg:logo.svg:g' \
- -e 's:favicon-[0-9a-f]+\.svg:logo.svg:g' \
- -e 's:::g' \
- -e 's:::g'
- $(Q)for f in $(rustdoc_output)/static.files/rustdoc-*.css; do \
- echo ".logo-container > img { object-fit: contain; }" >> $$f; done
-
-rustdoc-macros: private rustdoc_host = yes
-rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \
- --extern proc_macro
-rustdoc-macros: $(src)/macros/lib.rs FORCE
- +$(call if_changed,rustdoc)
-
-rustdoc-core: private rustc_target_flags = $(core-cfgs)
-rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE
- +$(call if_changed,rustdoc)
-
-rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE
- +$(call if_changed,rustdoc)
-
-# We need to allow `rustdoc::broken_intra_doc_links` because some
-# `no_global_oom_handling` functions refer to non-`no_global_oom_handling`
-# functions. Ideally `rustdoc` would have a way to distinguish broken links
-# due to things that are "configured out" vs. entirely non-existing ones.
-rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \
- -Arustdoc::broken_intra_doc_links
-rustdoc-alloc: $(RUST_LIB_SRC)/alloc/src/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE
- +$(call if_changed,rustdoc)
-
-rustdoc-kernel: private rustc_target_flags = --extern alloc \
- --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \
- --extern bindings --extern uapi
-rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \
- rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \
- $(obj)/bindings.o FORCE
- +$(call if_changed,rustdoc)
-
-quiet_cmd_rustc_test_library = RUSTC TL $<
- cmd_rustc_test_library = \
- OBJTREE=$(abspath $(objtree)) \
- $(RUSTC) $(rust_common_flags) \
- @$(objtree)/include/generated/rustc_cfg $(rustc_target_flags) \
- --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \
- --out-dir $(objtree)/$(obj)/test --cfg testlib \
- --sysroot $(objtree)/$(obj)/test/sysroot \
- -L$(objtree)/$(obj)/test \
- --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $<
-
-rusttestlib-build_error: $(src)/build_error.rs rusttest-prepare FORCE
- +$(call if_changed,rustc_test_library)
-
-rusttestlib-macros: private rustc_target_flags = --extern proc_macro
-rusttestlib-macros: private rustc_test_library_proc = yes
-rusttestlib-macros: $(src)/macros/lib.rs rusttest-prepare FORCE
- +$(call if_changed,rustc_test_library)
-
-rusttestlib-bindings: $(src)/bindings/lib.rs rusttest-prepare FORCE
- +$(call if_changed,rustc_test_library)
-
-rusttestlib-uapi: $(src)/uapi/lib.rs rusttest-prepare FORCE
- +$(call if_changed,rustc_test_library)
-
-quiet_cmd_rustdoc_test = RUSTDOC T $<
- cmd_rustdoc_test = \
- OBJTREE=$(abspath $(objtree)) \
- $(RUSTDOC) --test $(rust_common_flags) \
- @$(objtree)/include/generated/rustc_cfg \
- $(rustc_target_flags) $(rustdoc_test_target_flags) \
- --sysroot $(objtree)/$(obj)/test/sysroot $(rustdoc_test_quiet) \
- -L$(objtree)/$(obj)/test --output $(rustdoc_output) \
- --crate-name $(subst rusttest-,,$@) $<
-
-quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $<
- cmd_rustdoc_test_kernel = \
- rm -rf $(objtree)/$(obj)/test/doctests/kernel; \
- mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \
- OBJTREE=$(abspath $(objtree)) \
- $(RUSTDOC) --test $(rust_flags) \
- -L$(objtree)/$(obj) --extern alloc --extern kernel \
- --extern build_error --extern macros \
- --extern bindings --extern uapi \
- --no-run --crate-name kernel -Zunstable-options \
- --sysroot=/dev/null \
- --test-builder $(objtree)/scripts/rustdoc_test_builder \
- $< $(rustdoc_test_kernel_quiet); \
- $(objtree)/scripts/rustdoc_test_gen
-
-%/doctests_kernel_generated.rs %/doctests_kernel_generated_kunit.c: \
- $(src)/kernel/lib.rs $(obj)/kernel.o \
- $(objtree)/scripts/rustdoc_test_builder \
- $(objtree)/scripts/rustdoc_test_gen FORCE
- +$(call if_changed,rustdoc_test_kernel)
-
-# We cannot use `-Zpanic-abort-tests` because some tests are dynamic,
-# so for the moment we skip `-Cpanic=abort`.
-quiet_cmd_rustc_test = RUSTC T $<
- cmd_rustc_test = \
- OBJTREE=$(abspath $(objtree)) \
- $(RUSTC) --test $(rust_common_flags) \
- @$(objtree)/include/generated/rustc_cfg \
- $(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \
- --sysroot $(objtree)/$(obj)/test/sysroot \
- -L$(objtree)/$(obj)/test \
- --crate-name $(subst rusttest-,,$@) $<; \
- $(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \
- $(rustc_test_run_flags)
-
-rusttest: rusttest-macros rusttest-kernel
-
-# This prepares a custom sysroot with our custom `alloc` instead of
-# the standard one.
-#
-# This requires several hacks:
-# - Unlike `core` and `alloc`, `std` depends on more than a dozen crates,
-# including third-party crates that need to be downloaded, plus custom
-# `build.rs` steps. Thus hardcoding things here is not maintainable.
-# - `cargo` knows how to build the standard library, but it is an unstable
-# feature so far (`-Zbuild-std`).
-# - `cargo` only considers the use case of building the standard library
-# to use it in a given package. Thus we need to create a dummy package
-# and pick the generated libraries from there.
-# - The usual ways of modifying the dependency graph in `cargo` do not seem
-# to apply for the `-Zbuild-std` steps, thus we have to mislead it
-# by modifying the sources in the sysroot.
-# - To avoid messing with the user's Rust installation, we create a clone
-# of the sysroot. However, `cargo` ignores `RUSTFLAGS` in the `-Zbuild-std`
-# steps, thus we use a wrapper binary passed via `RUSTC` to pass the flag.
-#
-# In the future, we hope to avoid the whole ordeal by either:
-# - Making the `test` crate not depend on `std` (either improving upstream
-# or having our own custom crate).
-# - Making the tests run in kernel space (requires the previous point).
-# - Making `std` and friends be more like a "normal" crate, so that
-# `-Zbuild-std` and related hacks are not needed.
-quiet_cmd_rustsysroot = RUSTSYSROOT
- cmd_rustsysroot = \
- rm -rf $(objtree)/$(obj)/test; \
- mkdir -p $(objtree)/$(obj)/test; \
- cp -a $(rustc_sysroot) $(objtree)/$(obj)/test/sysroot; \
- echo '\#!/bin/sh' > $(objtree)/$(obj)/test/rustc_sysroot; \
- echo "$(RUSTC) --sysroot=$(abspath $(objtree)/$(obj)/test/sysroot) \"\$$@\"" \
- >> $(objtree)/$(obj)/test/rustc_sysroot; \
- chmod u+x $(objtree)/$(obj)/test/rustc_sysroot; \
- $(CARGO) -q new $(objtree)/$(obj)/test/dummy; \
- RUSTC=$(objtree)/$(obj)/test/rustc_sysroot $(CARGO) $(cargo_quiet) \
- test -Zbuild-std --target $(rustc_host_target) \
- --manifest-path $(objtree)/$(obj)/test/dummy/Cargo.toml; \
- rm $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib/*; \
- cp $(objtree)/$(obj)/test/dummy/target/$(rustc_host_target)/debug/deps/* \
- $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib
-
-rusttest-prepare: FORCE
- +$(call if_changed,rustsysroot)
-
-rusttest-macros: private rustc_target_flags = --extern proc_macro
-rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro
-rusttest-macros: $(src)/macros/lib.rs rusttest-prepare FORCE
- +$(call if_changed,rustc_test)
- +$(call if_changed,rustdoc_test)
-
-rusttest-kernel: private rustc_target_flags = --extern alloc \
- --extern build_error --extern macros --extern bindings --extern uapi
-rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \
- rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \
- rusttestlib-uapi FORCE
- +$(call if_changed,rustc_test)
- +$(call if_changed,rustc_test_library)
-
-ifdef CONFIG_CC_IS_CLANG
-bindgen_c_flags = $(c_flags)
-else
-# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC
-# plugin backend and/or the Clang driver would be perfectly compatible with GCC.
-#
-# For the moment, here we are tweaking the flags on the fly. This is a hack,
-# and some kernel configurations may not work (e.g. `GCC_PLUGIN_RANDSTRUCT`
-# if we end up using one of those structs).
-bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \
- -mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \
- -mindirect-branch=thunk-extern -mindirect-branch-register \
- -mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \
- -mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \
- -mno-pointers-to-nested-functions -mno-string \
- -mno-strict-align -mstrict-align \
- -fconserve-stack -falign-jumps=% -falign-loops=% \
- -femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \
- -fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \
- -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \
- -fzero-call-used-regs=% -fno-stack-clash-protection \
- -fno-inline-functions-called-once -fsanitize=bounds-strict \
- -fstrict-flex-arrays=% \
- --param=% --param asan-%
-
-# Derived from `scripts/Makefile.clang`.
-BINDGEN_TARGET_x86 := x86_64-linux-gnu
-BINDGEN_TARGET_arm64 := aarch64-linux-gnu
-BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH))
-
-# All warnings are inhibited since GCC builds are very experimental,
-# many GCC warnings are not supported by Clang, they may only appear in
-# some configurations, with new GCC versions, etc.
-bindgen_extra_c_flags = -w --target=$(BINDGEN_TARGET)
-
-# Auto variable zero-initialization requires an additional special option with
-# clang that is going to be removed sometime in the future (likely in
-# clang-18), so make sure to pass this option only if clang supports it
-# (libclang major version < 16).
-#
-# https://github.com/llvm/llvm-project/issues/44842
-# https://github.com/llvm/llvm-project/blob/llvmorg-16.0.0-rc2/clang/docs/ReleaseNotes.rst#deprecated-compiler-flags
-ifdef CONFIG_INIT_STACK_ALL_ZERO
-libclang_maj_ver=$(shell $(BINDGEN) $(srctree)/scripts/rust_is_available_bindgen_libclang.h 2>&1 | sed -ne 's/.*clang version \([0-9]*\).*/\1/p')
-ifeq ($(shell expr $(libclang_maj_ver) \< 16), 1)
-bindgen_extra_c_flags += -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang
-endif
-endif
-
-bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \
- $(bindgen_extra_c_flags)
-endif
-
-ifdef CONFIG_LTO
-bindgen_c_flags_lto = $(filter-out $(CC_FLAGS_LTO), $(bindgen_c_flags))
-else
-bindgen_c_flags_lto = $(bindgen_c_flags)
-endif
-
-bindgen_c_flags_final = $(bindgen_c_flags_lto) -D__BINDGEN__
-
-quiet_cmd_bindgen = BINDGEN $@
- cmd_bindgen = \
- $(BINDGEN) $< $(bindgen_target_flags) \
- --use-core --with-derive-default --ctypes-prefix core::ffi --no-layout-tests \
- --no-debug '.*' \
- -o $@ -- $(bindgen_c_flags_final) -DMODULE \
- $(bindgen_target_cflags) $(bindgen_target_extra)
-
-$(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \
- $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters)
-$(obj)/bindings/bindings_generated.rs: private bindgen_target_extra = ; \
- sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \1/g' $@
-$(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \
- $(src)/bindgen_parameters FORCE
- $(call if_changed_dep,bindgen)
-
-$(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \
- $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters)
-$(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \
- $(src)/bindgen_parameters FORCE
- $(call if_changed_dep,bindgen)
-
-# See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn
-# with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here
-# given it is `libclang`; but for consistency, future Clang changes and/or
-# a potential future GCC backend for `bindgen`, we disable it too.
-$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \
- --blocklist-type '.*' --allowlist-var '' \
- --allowlist-function 'rust_helper_.*'
-$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \
- -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations
-$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \
- sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n pub fn \1/g' $@
-$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers.c FORCE
- $(call if_changed_dep,bindgen)
-
-quiet_cmd_exports = EXPORTS $@
- cmd_exports = \
- $(NM) -p --defined-only $< \
- | awk '/ (T|R|D|B) / {printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3}' > $@
-
-$(obj)/exports_core_generated.h: $(obj)/core.o FORCE
- $(call if_changed,exports)
-
-$(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE
- $(call if_changed,exports)
-
-$(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE
- $(call if_changed,exports)
-
-$(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE
- $(call if_changed,exports)
-
-quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@
- cmd_rustc_procmacro = \
- $(RUSTC_OR_CLIPPY) $(rust_common_flags) \
- -Clinker-flavor=gcc -Clinker=$(HOSTCC) \
- -Clink-args='$(call escsq,$(KBUILD_HOSTLDFLAGS))' \
- --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \
- --crate-type proc-macro \
- --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<
-
-# Procedural macros can only be used with the `rustc` that compiled it.
-# Therefore, to get `libmacros.so` automatically recompiled when the compiler
-# version changes, we add `core.o` as a dependency (even if it is not needed).
-$(obj)/libmacros.so: $(src)/macros/lib.rs $(obj)/core.o FORCE
- +$(call if_changed_dep,rustc_procmacro)
-
-quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@
- cmd_rustc_library = \
- OBJTREE=$(abspath $(objtree)) \
- $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \
- $(filter-out $(skip_flags),$(rust_flags) $(rustc_target_flags)) \
- --emit=dep-info=$(depfile) --emit=obj=$@ \
- --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \
- --crate-type rlib -L$(objtree)/$(obj) \
- --crate-name $(patsubst %.o,%,$(notdir $@)) $< \
- --sysroot=/dev/null \
- $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
-
-rust-analyzer:
- $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
- --cfgs='core=$(core-cfgs)' --cfgs='alloc=$(alloc-cfgs)' \
- $(realpath $(srctree)) $(realpath $(objtree)) \
- $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
- $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
-
-redirect-intrinsics = \
- __addsf3 __eqsf2 __gesf2 __lesf2 __ltsf2 __mulsf3 __nesf2 __unordsf2 \
- __adddf3 __ledf2 __ltdf2 __muldf3 __unorddf2 \
- __muloti4 __multi3 \
- __udivmodti4 __udivti3 __umodti3
-
-ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),)
- # These intrinsics are defined for ARM64 and RISCV64
- redirect-intrinsics += \
- __ashrti3 \
- __ashlti3 __lshrti3
-endif
-
-$(obj)/core.o: private skip_clippy = 1
-$(obj)/core.o: private skip_flags = -Dunreachable_pub
-$(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym))
-$(obj)/core.o: private rustc_target_flags = $(core-cfgs)
-$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs FORCE
- +$(call if_changed_dep,rustc_library)
-ifdef CONFIG_X86_64
-$(obj)/core.o: scripts/target.json
-endif
-
-$(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*'
-$(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE
- +$(call if_changed_dep,rustc_library)
-
-$(obj)/alloc.o: private skip_clippy = 1
-$(obj)/alloc.o: private skip_flags = -Dunreachable_pub
-$(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs)
-$(obj)/alloc.o: $(RUST_LIB_SRC)/alloc/src/lib.rs $(obj)/compiler_builtins.o FORCE
- +$(call if_changed_dep,rustc_library)
-
-$(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE
- +$(call if_changed_dep,rustc_library)
-
-$(obj)/bindings.o: $(src)/bindings/lib.rs \
- $(obj)/compiler_builtins.o \
- $(obj)/bindings/bindings_generated.rs \
- $(obj)/bindings/bindings_helpers_generated.rs FORCE
- +$(call if_changed_dep,rustc_library)
-
-$(obj)/uapi.o: $(src)/uapi/lib.rs \
- $(obj)/compiler_builtins.o \
- $(obj)/uapi/uapi_generated.rs FORCE
- +$(call if_changed_dep,rustc_library)
-
-$(obj)/kernel.o: private rustc_target_flags = --extern alloc \
- --extern build_error --extern macros --extern bindings --extern uapi
-$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \
- $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE
- +$(call if_changed_dep,rustc_library)
-
-endif # CONFIG_RUST
diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters
deleted file mode 100644
--- a/rust/bindgen_parameters
+++ /dev/null
@@ -1,26 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
---opaque-type xregs_state
---opaque-type desc_struct
---opaque-type arch_lbr_state
---opaque-type local_apic
-
-# Packed type cannot transitively contain a `#[repr(align)]` type.
---opaque-type alt_instr
---opaque-type x86_msi_data
---opaque-type x86_msi_addr_lo
-
-# `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2,
-# commit 2aed6b021680 ("context: Escape the try keyword properly").
---opaque-type kunit_try_catch
-
-# If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust
-# warning. We don't need to peek into it anyway.
---opaque-type spinlock
-
-# `seccomp`'s comment gets understood as a doctest
---no-doc-comments
-
-# These functions use the `__preserve_most` calling convention, which neither bindgen
-# nor Rust currently understand, and which Clang currently declares to be unstable.
---blocklist-function __list_.*_report
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
deleted file mode 100644
--- a/rust/bindings/bindings_helper.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Header that contains the code (mostly headers) for which Rust bindings
- * will be automatically generated by `bindgen`.
- *
- * Sorted alphabetically.
- */
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-/* `bindgen` gets confused at certain things. */
-const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN;
-const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC;
-const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL;
-const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT;
-const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT;
-const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO;
diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs
deleted file mode 100644
--- a/rust/bindings/lib.rs
+++ /dev/null
@@ -1,50 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Bindings.
-//!
-//! Imports the generated bindings by `bindgen`.
-//!
-//! This crate may not be directly used. If you need a kernel C API that is
-//! not ported or wrapped in the `kernel` crate, then do so first instead of
-//! using this crate.
-
-#![no_std]
-// See .
-#![cfg_attr(test, allow(deref_nullptr))]
-#![cfg_attr(test, allow(unaligned_references))]
-#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))]
-#![allow(
- clippy::all,
- missing_docs,
- non_camel_case_types,
- non_upper_case_globals,
- non_snake_case,
- improper_ctypes,
- unreachable_pub,
- unsafe_op_in_unsafe_fn
-)]
-
-mod bindings_raw {
- // Use glob import here to expose all helpers.
- // Symbols defined within the module will take precedence to the glob import.
- pub use super::bindings_helper::*;
- include!(concat!(
- env!("OBJTREE"),
- "/rust/bindings/bindings_generated.rs"
- ));
-}
-
-// When both a directly exposed symbol and a helper exists for the same function,
-// the directly exposed symbol is preferred and the helper becomes dead code, so
-// ignore the warning here.
-#[allow(dead_code)]
-mod bindings_helper {
- // Import the generated bindings for types.
- use super::bindings_raw::*;
- include!(concat!(
- env!("OBJTREE"),
- "/rust/bindings/bindings_helpers_generated.rs"
- ));
-}
-
-pub use bindings_raw::*;
diff --git a/rust/build_error.rs b/rust/build_error.rs
deleted file mode 100644
--- a/rust/build_error.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Build-time error.
-//!
-//! This crate provides a [const function][const-functions] `build_error`, which will panic in
-//! compile-time if executed in [const context][const-context], and will cause a build error
-//! if not executed at compile time and the optimizer does not optimise away the call.
-//!
-//! It is used by `build_assert!` in the kernel crate, allowing checking of
-//! conditions that could be checked statically, but could not be enforced in
-//! Rust yet (e.g. perform some checks in [const functions][const-functions], but those
-//! functions could still be called in the runtime).
-//!
-//! For details on constant evaluation in Rust, please see the [Reference][const-eval].
-//!
-//! [const-eval]: https://doc.rust-lang.org/reference/const_eval.html
-//! [const-functions]: https://doc.rust-lang.org/reference/const_eval.html#const-functions
-//! [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context
-
-#![no_std]
-
-/// Panics if executed in [const context][const-context], or triggers a build error if not.
-///
-/// [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context
-#[inline(never)]
-#[cold]
-#[export_name = "rust_build_error"]
-#[track_caller]
-pub const fn build_error(msg: &'static str) -> ! {
- panic!("{}", msg);
-}
diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs
deleted file mode 100644
--- a/rust/compiler_builtins.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Our own `compiler_builtins`.
-//!
-//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`].
-//! Since we do not need the vast majority of them, we avoid the dependency
-//! by providing this file.
-//!
-//! At the moment, some builtins are required that should not be. For instance,
-//! [`core`] has 128-bit integers functionality which we should not be compiling
-//! in. We will work with upstream [`core`] to provide feature flags to disable
-//! the parts we do not need. For the moment, we define them to [`panic!`] at
-//! runtime for simplicity to catch mistakes, instead of performing surgery
-//! on `core.o`.
-//!
-//! In any case, all these symbols are weakened to ensure we do not override
-//! those that may be provided by the rest of the kernel.
-//!
-//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins
-//! [`compiler-rt`]: https://compiler-rt.llvm.org/
-
-#![allow(internal_features)]
-#![feature(compiler_builtins)]
-#![compiler_builtins]
-#![no_builtins]
-#![no_std]
-
-macro_rules! define_panicking_intrinsics(
- ($reason: tt, { $($ident: ident, )* }) => {
- $(
- #[doc(hidden)]
- #[export_name = concat!("__rust", stringify!($ident))]
- pub extern "C" fn $ident() {
- panic!($reason);
- }
- )*
- }
-);
-
-define_panicking_intrinsics!("`f32` should not be used", {
- __addsf3,
- __eqsf2,
- __gesf2,
- __lesf2,
- __ltsf2,
- __mulsf3,
- __nesf2,
- __unordsf2,
-});
-
-define_panicking_intrinsics!("`f64` should not be used", {
- __adddf3,
- __ledf2,
- __ltdf2,
- __muldf3,
- __unorddf2,
-});
-
-define_panicking_intrinsics!("`i128` should not be used", {
- __ashrti3,
- __muloti4,
- __multi3,
-});
-
-define_panicking_intrinsics!("`u128` should not be used", {
- __ashlti3,
- __lshrti3,
- __udivmodti4,
- __udivti3,
- __umodti3,
-});
-
-// NOTE: if you are adding a new intrinsic here, you should also add it to
-// `redirect-intrinsics` in `rust/Makefile`.
diff --git a/rust/exports.c b/rust/exports.c
deleted file mode 100644
--- a/rust/exports.c
+++ /dev/null
@@ -1,26 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * A hack to export Rust symbols for loadable modules without having to redo
- * the entire `include/linux/export.h` logic in Rust.
- *
- * This requires the Rust's new/future `v0` mangling scheme because the default
- * one ("legacy") uses invalid characters for C identifiers (thus we cannot use
- * the `EXPORT_SYMBOL_*` macros).
- *
- * All symbols are exported as GPL-only to guarantee no GPL-only feature is
- * accidentally exposed.
- */
-
-#include
-
-#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym)
-
-#include "exports_core_generated.h"
-#include "exports_alloc_generated.h"
-#include "exports_bindings_generated.h"
-#include "exports_kernel_generated.h"
-
-// For modules using `rust/build_error.rs`.
-#ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW
-EXPORT_SYMBOL_RUST_GPL(rust_build_error);
-#endif
diff --git a/rust/helpers.c b/rust/helpers.c
deleted file mode 100644
--- a/rust/helpers.c
+++ /dev/null
@@ -1,188 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions
- * cannot be called either. This file explicitly creates functions ("helpers")
- * that wrap those so that they can be called from Rust.
- *
- * Even though Rust kernel modules should never use the bindings directly, some
- * of these helpers need to be exported because Rust generics and inlined
- * functions may not get their code generated in the crate where they are
- * defined. Other helpers, called from non-inline functions, may not be
- * exported, in principle. However, in general, the Rust compiler does not
- * guarantee codegen will be performed for a non-inline function either.
- * Therefore, this file exports all the helpers. In the future, this may be
- * revisited to reduce the number of exports after the compiler is informed
- * about the places codegen is required.
- *
- * All symbols are exported as GPL-only to guarantee no GPL-only feature is
- * accidentally exposed.
- *
- * Sorted alphabetically.
- */
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-__noreturn void rust_helper_BUG(void)
-{
- BUG();
-}
-EXPORT_SYMBOL_GPL(rust_helper_BUG);
-
-void rust_helper_mutex_lock(struct mutex *lock)
-{
- mutex_lock(lock);
-}
-EXPORT_SYMBOL_GPL(rust_helper_mutex_lock);
-
-void rust_helper___spin_lock_init(spinlock_t *lock, const char *name,
- struct lock_class_key *key)
-{
-#ifdef CONFIG_DEBUG_SPINLOCK
- __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG);
-#else
- spin_lock_init(lock);
-#endif
-}
-EXPORT_SYMBOL_GPL(rust_helper___spin_lock_init);
-
-void rust_helper_spin_lock(spinlock_t *lock)
-{
- spin_lock(lock);
-}
-EXPORT_SYMBOL_GPL(rust_helper_spin_lock);
-
-void rust_helper_spin_unlock(spinlock_t *lock)
-{
- spin_unlock(lock);
-}
-EXPORT_SYMBOL_GPL(rust_helper_spin_unlock);
-
-void rust_helper_init_wait(struct wait_queue_entry *wq_entry)
-{
- init_wait(wq_entry);
-}
-EXPORT_SYMBOL_GPL(rust_helper_init_wait);
-
-int rust_helper_signal_pending(struct task_struct *t)
-{
- return signal_pending(t);
-}
-EXPORT_SYMBOL_GPL(rust_helper_signal_pending);
-
-refcount_t rust_helper_REFCOUNT_INIT(int n)
-{
- return (refcount_t)REFCOUNT_INIT(n);
-}
-EXPORT_SYMBOL_GPL(rust_helper_REFCOUNT_INIT);
-
-void rust_helper_refcount_inc(refcount_t *r)
-{
- refcount_inc(r);
-}
-EXPORT_SYMBOL_GPL(rust_helper_refcount_inc);
-
-bool rust_helper_refcount_dec_and_test(refcount_t *r)
-{
- return refcount_dec_and_test(r);
-}
-EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test);
-
-__force void *rust_helper_ERR_PTR(long err)
-{
- return ERR_PTR(err);
-}
-EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR);
-
-bool rust_helper_IS_ERR(__force const void *ptr)
-{
- return IS_ERR(ptr);
-}
-EXPORT_SYMBOL_GPL(rust_helper_IS_ERR);
-
-long rust_helper_PTR_ERR(__force const void *ptr)
-{
- return PTR_ERR(ptr);
-}
-EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR);
-
-const char *rust_helper_errname(int err)
-{
- return errname(err);
-}
-EXPORT_SYMBOL_GPL(rust_helper_errname);
-
-struct task_struct *rust_helper_get_current(void)
-{
- return current;
-}
-EXPORT_SYMBOL_GPL(rust_helper_get_current);
-
-void rust_helper_get_task_struct(struct task_struct *t)
-{
- get_task_struct(t);
-}
-EXPORT_SYMBOL_GPL(rust_helper_get_task_struct);
-
-void rust_helper_put_task_struct(struct task_struct *t)
-{
- put_task_struct(t);
-}
-EXPORT_SYMBOL_GPL(rust_helper_put_task_struct);
-
-struct kunit *rust_helper_kunit_get_current_test(void)
-{
- return kunit_get_current_test();
-}
-EXPORT_SYMBOL_GPL(rust_helper_kunit_get_current_test);
-
-void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func,
- bool onstack, const char *name,
- struct lock_class_key *key)
-{
- __init_work(work, onstack);
- work->data = (atomic_long_t)WORK_DATA_INIT();
- lockdep_init_map(&work->lockdep_map, name, key, 0);
- INIT_LIST_HEAD(&work->entry);
- work->func = func;
-}
-EXPORT_SYMBOL_GPL(rust_helper_init_work_with_key);
-
-void * __must_check __realloc_size(2)
-rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags)
-{
- return krealloc(objp, new_size, flags);
-}
-EXPORT_SYMBOL_GPL(rust_helper_krealloc);
-
-/*
- * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can
- * use it in contexts where Rust expects a `usize` like slice (array) indices.
- * `usize` is defined to be the same as C's `uintptr_t` type (can hold any
- * pointer) but not necessarily the same as `size_t` (can hold the size of any
- * single object). Most modern platforms use the same concrete integer type for
- * both of them, but in case we find ourselves on a platform where
- * that's not true, fail early instead of risking ABI or
- * integer-overflow issues.
- *
- * If your platform fails this assertion, it means that you are in
- * danger of integer-overflow bugs (even if you attempt to add
- * `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on
- * your platform such that `size_t` matches `uintptr_t` (i.e., to increase
- * `size_t`, because `uintptr_t` has to be at least as big as `size_t`).
- */
-static_assert(
- sizeof(size_t) == sizeof(uintptr_t) &&
- __alignof__(size_t) == __alignof__(uintptr_t),
- "Rust code expects C `size_t` to match Rust `usize`"
-);
diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs
deleted file mode 100644
--- a/rust/kernel/alloc.rs
+++ /dev/null
@@ -1,73 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Extensions to the [`alloc`] crate.
-
-#[cfg(not(test))]
-#[cfg(not(testlib))]
-mod allocator;
-pub mod box_ext;
-pub mod vec_ext;
-
-/// Indicates an allocation error.
-#[derive(Copy, Clone, PartialEq, Eq, Debug)]
-pub struct AllocError;
-
-/// Flags to be used when allocating memory.
-///
-/// They can be combined with the operators `|`, `&`, and `!`.
-///
-/// Values can be used from the [`flags`] module.
-#[derive(Clone, Copy)]
-pub struct Flags(u32);
-
-impl core::ops::BitOr for Flags {
- type Output = Self;
- fn bitor(self, rhs: Self) -> Self::Output {
- Self(self.0 | rhs.0)
- }
-}
-
-impl core::ops::BitAnd for Flags {
- type Output = Self;
- fn bitand(self, rhs: Self) -> Self::Output {
- Self(self.0 & rhs.0)
- }
-}
-
-impl core::ops::Not for Flags {
- type Output = Self;
- fn not(self) -> Self::Output {
- Self(!self.0)
- }
-}
-
-/// Allocation flags.
-///
-/// These are meant to be used in functions that can allocate memory.
-pub mod flags {
- use super::Flags;
-
- /// Zeroes out the allocated memory.
- ///
- /// This is normally or'd with other flags.
- pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO);
-
- /// Users can not sleep and need the allocation to succeed.
- ///
- /// A lower watermark is applied to allow access to "atomic reserves". The current
- /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g.
- /// raw_spin_lock). The same applies to [`GFP_NOWAIT`].
- pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC);
-
- /// Typical for kernel-internal allocations. The caller requires ZONE_NORMAL or a lower zone
- /// for direct access but can direct reclaim.
- pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL);
-
- /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg.
- pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT);
-
- /// Ror kernel allocations that should not stall for direct reclaim, start physical IO or
- /// use any filesystem callback. It is very likely to fail to allocate memory, even for very
- /// small allocations.
- pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT);
-}
diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs
deleted file mode 100644
--- a/rust/kernel/alloc/allocator.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Allocator support.
-
-use super::{flags::*, Flags};
-use core::alloc::{GlobalAlloc, Layout};
-use core::ptr;
-
-struct KernelAllocator;
-
-/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment.
-///
-/// # Safety
-///
-/// - `ptr` can be either null or a pointer which has been allocated by this allocator.
-/// - `new_layout` must have a non-zero size.
-pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 {
- // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first.
- let layout = new_layout.pad_to_align();
-
- let mut size = layout.size();
-
- if layout.align() > bindings::ARCH_SLAB_MINALIGN {
- // The alignment requirement exceeds the slab guarantee, thus try to enlarge the size
- // to use the "power-of-two" size/alignment guarantee (see comments in `kmalloc()` for
- // more information).
- //
- // Note that `layout.size()` (after padding) is guaranteed to be a multiple of
- // `layout.align()`, so `next_power_of_two` gives enough alignment guarantee.
- size = size.next_power_of_two();
- }
-
- // SAFETY:
- // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the
- // function safety requirement.
- // - `size` is greater than 0 since it's either a `layout.size()` (which cannot be zero
- // according to the function safety requirement) or a result from `next_power_of_two()`.
- unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 }
-}
-
-unsafe impl GlobalAlloc for KernelAllocator {
- unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
- // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety
- // requirement.
- unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) }
- }
-
- unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
- unsafe {
- bindings::kfree(ptr as *const core::ffi::c_void);
- }
- }
-
- unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
- // SAFETY:
- // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not
- // overflow `isize` by the function safety requirement.
- // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two).
- let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
-
- // SAFETY:
- // - `ptr` is either null or a pointer allocated by this allocator by the function safety
- // requirement.
- // - the size of `layout` is not zero because `new_size` is not zero by the function safety
- // requirement.
- unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) }
- }
-
- unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
- // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety
- // requirement.
- unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) }
- }
-}
-
-#[global_allocator]
-static ALLOCATOR: KernelAllocator = KernelAllocator;
-
-// See .
-#[no_mangle]
-static __rust_no_alloc_shim_is_unstable: u8 = 0;
diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs
deleted file mode 100644
--- a/rust/kernel/alloc/box_ext.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Extensions to [`Box`] for fallible allocations.
-
-use super::{AllocError, Flags};
-use alloc::boxed::Box;
-use core::mem::MaybeUninit;
-
-/// Extensions to [`Box`].
-pub trait BoxExt: Sized {
- /// Allocates a new box.
- ///
- /// The allocation may fail, in which case an error is returned.
- fn new(x: T, flags: Flags) -> Result;
-
- /// Allocates a new uninitialised box.
- ///
- /// The allocation may fail, in which case an error is returned.
- fn new_uninit(flags: Flags) -> Result>, AllocError>;
-}
-
-impl BoxExt for Box {
- fn new(x: T, flags: Flags) -> Result {
- let b = >::new_uninit(flags)?;
- Ok(Box::write(b, x))
- }
-
- #[cfg(any(test, testlib))]
- fn new_uninit(_flags: Flags) -> Result>, AllocError> {
- Ok(Box::new_uninit())
- }
-
- #[cfg(not(any(test, testlib)))]
- fn new_uninit(flags: Flags) -> Result>, AllocError> {
- let ptr = if core::mem::size_of::>() == 0 {
- core::ptr::NonNull::<_>::dangling().as_ptr()
- } else {
- let layout = core::alloc::Layout::new::>();
-
- // SAFETY: Memory is being allocated (first arg is null). The only other source of
- // safety issues is sleeping on atomic context, which is addressed by klint. Lastly,
- // the type is not a SZT (checked above).
- let ptr =
- unsafe { super::allocator::krealloc_aligned(core::ptr::null_mut(), layout, flags) };
- if ptr.is_null() {
- return Err(AllocError);
- }
-
- ptr.cast::>()
- };
-
- // SAFETY: For non-zero-sized types, we allocate above using the global allocator. For
- // zero-sized types, we use `NonNull::dangling`.
- Ok(unsafe { Box::from_raw(ptr) })
- }
-}
diff --git a/rust/kernel/alloc/vec_ext.rs b/rust/kernel/alloc/vec_ext.rs
deleted file mode 100644
--- a/rust/kernel/alloc/vec_ext.rs
+++ /dev/null
@@ -1,185 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Extensions to [`Vec`] for fallible allocations.
-
-use super::{AllocError, Flags};
-use alloc::vec::Vec;
-
-/// Extensions to [`Vec`].
-pub trait VecExt: Sized {
- /// Creates a new [`Vec`] instance with at least the given capacity.
- ///
- /// # Examples
- ///
- /// ```
- /// let v = Vec::::with_capacity(20, GFP_KERNEL)?;
- ///
- /// assert!(v.capacity() >= 20);
- /// # Ok::<(), Error>(())
- /// ```
- fn with_capacity(capacity: usize, flags: Flags) -> Result;
-
- /// Appends an element to the back of the [`Vec`] instance.
- ///
- /// # Examples
- ///
- /// ```
- /// let mut v = Vec::new();
- /// v.push(1, GFP_KERNEL)?;
- /// assert_eq!(&v, &[1]);
- ///
- /// v.push(2, GFP_KERNEL)?;
- /// assert_eq!(&v, &[1, 2]);
- /// # Ok::<(), Error>(())
- /// ```
- fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError>;
-
- /// Pushes clones of the elements of slice into the [`Vec`] instance.
- ///
- /// # Examples
- ///
- /// ```
- /// let mut v = Vec::new();
- /// v.push(1, GFP_KERNEL)?;
- ///
- /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?;
- /// assert_eq!(&v, &[1, 20, 30, 40]);
- ///
- /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?;
- /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]);
- /// # Ok::<(), Error>(())
- /// ```
- fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError>
- where
- T: Clone;
-
- /// Ensures that the capacity exceeds the length by at least `additional` elements.
- ///
- /// # Examples
- ///
- /// ```
- /// let mut v = Vec::new();
- /// v.push(1, GFP_KERNEL)?;
- ///
- /// v.reserve(10, GFP_KERNEL)?;
- /// let cap = v.capacity();
- /// assert!(cap >= 10);
- ///
- /// v.reserve(10, GFP_KERNEL)?;
- /// let new_cap = v.capacity();
- /// assert_eq!(new_cap, cap);
- ///
- /// # Ok::<(), Error>(())
- /// ```
- fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError>;
-}
-
-impl VecExt for Vec {
- fn with_capacity(capacity: usize, flags: Flags) -> Result {
- let mut v = Vec::new();
- >::reserve(&mut v, capacity, flags)?;
- Ok(v)
- }
-
- fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> {
- >::reserve(self, 1, flags)?;
- let s = self.spare_capacity_mut();
- s[0].write(v);
-
- // SAFETY: We just initialised the first spare entry, so it is safe to increase the length
- // by 1. We also know that the new length is <= capacity because of the previous call to
- // `reserve` above.
- unsafe { self.set_len(self.len() + 1) };
- Ok(())
- }
-
- fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError>
- where
- T: Clone,
- {
- >::reserve(self, other.len(), flags)?;
- for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) {
- slot.write(item.clone());
- }
-
- // SAFETY: We just initialised the `other.len()` spare entries, so it is safe to increase
- // the length by the same amount. We also know that the new length is <= capacity because
- // of the previous call to `reserve` above.
- unsafe { self.set_len(self.len() + other.len()) };
- Ok(())
- }
-
- #[cfg(any(test, testlib))]
- fn reserve(&mut self, additional: usize, _flags: Flags) -> Result<(), AllocError> {
- Vec::reserve(self, additional);
- Ok(())
- }
-
- #[cfg(not(any(test, testlib)))]
- fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> {
- let len = self.len();
- let cap = self.capacity();
-
- if cap - len >= additional {
- return Ok(());
- }
-
- if core::mem::size_of::() == 0 {
- // The capacity is already `usize::MAX` for SZTs, we can't go higher.
- return Err(AllocError);
- }
-
- // We know cap is <= `isize::MAX` because `Layout::array` fails if the resulting byte size
- // is greater than `isize::MAX`. So the multiplication by two won't overflow.
- let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?);
- let layout = core::alloc::Layout::array::(new_cap).map_err(|_| AllocError)?;
-
- let (old_ptr, len, cap) = destructure(self);
-
- // We need to make sure that `ptr` is either NULL or comes from a previous call to
- // `krealloc_aligned`. A `Vec`'s `ptr` value is not guaranteed to be NULL and might be
- // dangling after being created with `Vec::new`. Instead, we can rely on `Vec`'s capacity
- // to be zero if no memory has been allocated yet.
- let ptr = if cap == 0 {
- core::ptr::null_mut()
- } else {
- old_ptr
- };
-
- // SAFETY: `ptr` is valid because it's either NULL or comes from a previous call to
- // `krealloc_aligned`. We also verified that the type is not a ZST.
- let new_ptr = unsafe { super::allocator::krealloc_aligned(ptr.cast(), layout, flags) };
- if new_ptr.is_null() {
- // SAFETY: We are just rebuilding the existing `Vec` with no changes.
- unsafe { rebuild(self, old_ptr, len, cap) };
- Err(AllocError)
- } else {
- // SAFETY: `ptr` has been reallocated with the layout for `new_cap` elements. New cap
- // is greater than `cap`, so it continues to be >= `len`.
- unsafe { rebuild(self, new_ptr.cast::(), len, new_cap) };
- Ok(())
- }
- }
-}
-
-#[cfg(not(any(test, testlib)))]
-fn destructure(v: &mut Vec) -> (*mut T, usize, usize) {
- let mut tmp = Vec::new();
- core::mem::swap(&mut tmp, v);
- let mut tmp = core::mem::ManuallyDrop::new(tmp);
- let len = tmp.len();
- let cap = tmp.capacity();
- (tmp.as_mut_ptr(), len, cap)
-}
-
-/// Rebuilds a `Vec` from a pointer, length, and capacity.
-///
-/// # Safety
-///
-/// The same as [`Vec::from_raw_parts`].
-#[cfg(not(any(test, testlib)))]
-unsafe fn rebuild(v: &mut Vec, ptr: *mut T, len: usize, cap: usize) {
- // SAFETY: The safety requirements from this function satisfy those of `from_raw_parts`.
- let mut tmp = unsafe { Vec::from_raw_parts(ptr, len, cap) };
- core::mem::swap(&mut tmp, v);
-}
diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs
deleted file mode 100644
--- a/rust/kernel/build_assert.rs
+++ /dev/null
@@ -1,84 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Build-time assert.
-
-/// Fails the build if the code path calling `build_error!` can possibly be executed.
-///
-/// If the macro is executed in const context, `build_error!` will panic.
-/// If the compiler or optimizer cannot guarantee that `build_error!` can never
-/// be called, a build error will be triggered.
-///
-/// # Examples
-///
-/// ```
-/// # use kernel::build_error;
-/// #[inline]
-/// fn foo(a: usize) -> usize {
-/// a.checked_add(1).unwrap_or_else(|| build_error!("overflow"))
-/// }
-///
-/// assert_eq!(foo(usize::MAX - 1), usize::MAX); // OK.
-/// // foo(usize::MAX); // Fails to compile.
-/// ```
-#[macro_export]
-macro_rules! build_error {
- () => {{
- $crate::build_error("")
- }};
- ($msg:expr) => {{
- $crate::build_error($msg)
- }};
-}
-
-/// Asserts that a boolean expression is `true` at compile time.
-///
-/// If the condition is evaluated to `false` in const context, `build_assert!`
-/// will panic. If the compiler or optimizer cannot guarantee the condition will
-/// be evaluated to `true`, a build error will be triggered.
-///
-/// [`static_assert!`] should be preferred to `build_assert!` whenever possible.
-///
-/// # Examples
-///
-/// These examples show that different types of [`assert!`] will trigger errors
-/// at different stage of compilation. It is preferred to err as early as
-/// possible, so [`static_assert!`] should be used whenever possible.
-/// ```ignore
-/// fn foo() {
-/// static_assert!(1 > 1); // Compile-time error
-/// build_assert!(1 > 1); // Build-time error
-/// assert!(1 > 1); // Run-time error
-/// }
-/// ```
-///
-/// When the condition refers to generic parameters or parameters of an inline function,
-/// [`static_assert!`] cannot be used. Use `build_assert!` in this scenario.
-/// ```
-/// fn foo() {
-/// // `static_assert!(N > 1);` is not allowed
-/// build_assert!(N > 1); // Build-time check
-/// assert!(N > 1); // Run-time check
-/// }
-///
-/// #[inline]
-/// fn bar(n: usize) {
-/// // `static_assert!(n > 1);` is not allowed
-/// build_assert!(n > 1); // Build-time check
-/// assert!(n > 1); // Run-time check
-/// }
-/// ```
-///
-/// [`static_assert!`]: crate::static_assert!
-#[macro_export]
-macro_rules! build_assert {
- ($cond:expr $(,)?) => {{
- if !$cond {
- $crate::build_error(concat!("assertion failed: ", stringify!($cond)));
- }
- }};
- ($cond:expr, $msg:expr) => {{
- if !$cond {
- $crate::build_error($msg);
- }
- }};
-}
diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs
deleted file mode 100644
--- a/rust/kernel/error.rs
+++ /dev/null
@@ -1,327 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Kernel errors.
-//!
-//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h)
-
-use crate::{alloc::AllocError, str::CStr};
-
-use alloc::alloc::LayoutError;
-
-use core::fmt;
-use core::num::TryFromIntError;
-use core::str::Utf8Error;
-
-/// Contains the C-compatible error codes.
-#[rustfmt::skip]
-pub mod code {
- macro_rules! declare_err {
- ($err:tt $(,)? $($doc:expr),+) => {
- $(
- #[doc = $doc]
- )*
- pub const $err: super::Error = super::Error(-(crate::bindings::$err as i32));
- };
- }
-
- declare_err!(EPERM, "Operation not permitted.");
- declare_err!(ENOENT, "No such file or directory.");
- declare_err!(ESRCH, "No such process.");
- declare_err!(EINTR, "Interrupted system call.");
- declare_err!(EIO, "I/O error.");
- declare_err!(ENXIO, "No such device or address.");
- declare_err!(E2BIG, "Argument list too long.");
- declare_err!(ENOEXEC, "Exec format error.");
- declare_err!(EBADF, "Bad file number.");
- declare_err!(ECHILD, "No child processes.");
- declare_err!(EAGAIN, "Try again.");
- declare_err!(ENOMEM, "Out of memory.");
- declare_err!(EACCES, "Permission denied.");
- declare_err!(EFAULT, "Bad address.");
- declare_err!(ENOTBLK, "Block device required.");
- declare_err!(EBUSY, "Device or resource busy.");
- declare_err!(EEXIST, "File exists.");
- declare_err!(EXDEV, "Cross-device link.");
- declare_err!(ENODEV, "No such device.");
- declare_err!(ENOTDIR, "Not a directory.");
- declare_err!(EISDIR, "Is a directory.");
- declare_err!(EINVAL, "Invalid argument.");
- declare_err!(ENFILE, "File table overflow.");
- declare_err!(EMFILE, "Too many open files.");
- declare_err!(ENOTTY, "Not a typewriter.");
- declare_err!(ETXTBSY, "Text file busy.");
- declare_err!(EFBIG, "File too large.");
- declare_err!(ENOSPC, "No space left on device.");
- declare_err!(ESPIPE, "Illegal seek.");
- declare_err!(EROFS, "Read-only file system.");
- declare_err!(EMLINK, "Too many links.");
- declare_err!(EPIPE, "Broken pipe.");
- declare_err!(EDOM, "Math argument out of domain of func.");
- declare_err!(ERANGE, "Math result not representable.");
- declare_err!(ERESTARTSYS, "Restart the system call.");
- declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted.");
- declare_err!(ERESTARTNOHAND, "Restart if no handler.");
- declare_err!(ENOIOCTLCMD, "No ioctl command.");
- declare_err!(ERESTART_RESTARTBLOCK, "Restart by calling sys_restart_syscall.");
- declare_err!(EPROBE_DEFER, "Driver requests probe retry.");
- declare_err!(EOPENSTALE, "Open found a stale dentry.");
- declare_err!(ENOPARAM, "Parameter not supported.");
- declare_err!(EBADHANDLE, "Illegal NFS file handle.");
- declare_err!(ENOTSYNC, "Update synchronization mismatch.");
- declare_err!(EBADCOOKIE, "Cookie is stale.");
- declare_err!(ENOTSUPP, "Operation is not supported.");
- declare_err!(ETOOSMALL, "Buffer or request is too small.");
- declare_err!(ESERVERFAULT, "An untranslatable error occurred.");
- declare_err!(EBADTYPE, "Type not supported by server.");
- declare_err!(EJUKEBOX, "Request initiated, but will not complete before timeout.");
- declare_err!(EIOCBQUEUED, "iocb queued, will get completion event.");
- declare_err!(ERECALLCONFLICT, "Conflict with recalled state.");
- declare_err!(ENOGRACE, "NFS file lock reclaim refused.");
-}
-
-/// Generic integer kernel error.
-///
-/// The kernel defines a set of integer generic error codes based on C and
-/// POSIX ones. These codes may have a more specific meaning in some contexts.
-///
-/// # Invariants
-///
-/// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`).
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub struct Error(core::ffi::c_int);
-
-impl Error {
- /// Creates an [`Error`] from a kernel error code.
- ///
- /// It is a bug to pass an out-of-range `errno`. `EINVAL` would
- /// be returned in such a case.
- pub(crate) fn from_errno(errno: core::ffi::c_int) -> Error {
- if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 {
- // TODO: Make it a `WARN_ONCE` once available.
- crate::pr_warn!(
- "attempted to create `Error` with out of range `errno`: {}",
- errno
- );
- return code::EINVAL;
- }
-
- // INVARIANT: The check above ensures the type invariant
- // will hold.
- Error(errno)
- }
-
- /// Creates an [`Error`] from a kernel error code.
- ///
- /// # Safety
- ///
- /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`).
- unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error {
- // INVARIANT: The contract ensures the type invariant
- // will hold.
- Error(errno)
- }
-
- /// Returns the kernel error code.
- pub fn to_errno(self) -> core::ffi::c_int {
- self.0
- }
-
- /// Returns the error encoded as a pointer.
- #[allow(dead_code)]
- pub(crate) fn to_ptr(self) -> *mut T {
- // SAFETY: `self.0` is a valid error due to its invariant.
- unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ }
- }
-
- /// Returns a string representing the error, if one exists.
- #[cfg(not(testlib))]
- pub fn name(&self) -> Option<&'static CStr> {
- // SAFETY: Just an FFI call, there are no extra safety requirements.
- let ptr = unsafe { bindings::errname(-self.0) };
- if ptr.is_null() {
- None
- } else {
- // SAFETY: The string returned by `errname` is static and `NUL`-terminated.
- Some(unsafe { CStr::from_char_ptr(ptr) })
- }
- }
-
- /// Returns a string representing the error, if one exists.
- ///
- /// When `testlib` is configured, this always returns `None` to avoid the dependency on a
- /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still
- /// run in userspace.
- #[cfg(testlib)]
- pub fn name(&self) -> Option<&'static CStr> {
- None
- }
-}
-
-impl fmt::Debug for Error {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- match self.name() {
- // Print out number if no name can be found.
- None => f.debug_tuple("Error").field(&-self.0).finish(),
- // SAFETY: These strings are ASCII-only.
- Some(name) => f
- .debug_tuple(unsafe { core::str::from_utf8_unchecked(name) })
- .finish(),
- }
- }
-}
-
-impl From for Error {
- fn from(_: AllocError) -> Error {
- code::ENOMEM
- }
-}
-
-impl From for Error {
- fn from(_: TryFromIntError) -> Error {
- code::EINVAL
- }
-}
-
-impl From for Error {
- fn from(_: Utf8Error) -> Error {
- code::EINVAL
- }
-}
-
-impl From for Error {
- fn from(_: LayoutError) -> Error {
- code::ENOMEM
- }
-}
-
-impl From for Error {
- fn from(_: core::fmt::Error) -> Error {
- code::EINVAL
- }
-}
-
-impl From for Error {
- fn from(e: core::convert::Infallible) -> Error {
- match e {}
- }
-}
-
-/// A [`Result`] with an [`Error`] error type.
-///
-/// To be used as the return type for functions that may fail.
-///
-/// # Error codes in C and Rust
-///
-/// In C, it is common that functions indicate success or failure through
-/// their return value; modifying or returning extra data through non-`const`
-/// pointer parameters. In particular, in the kernel, functions that may fail
-/// typically return an `int` that represents a generic error code. We model
-/// those as [`Error`].
-///
-/// In Rust, it is idiomatic to model functions that may fail as returning
-/// a [`Result`]. Since in the kernel many functions return an error code,
-/// [`Result`] is a type alias for a [`core::result::Result`] that uses
-/// [`Error`] as its error type.
-///
-/// Note that even if a function does not return anything when it succeeds,
-/// it should still be modeled as returning a `Result` rather than
-/// just an [`Error`].
-pub type Result = core::result::Result;
-
-/// Converts an integer as returned by a C kernel function to an error if it's negative, and
-/// `Ok(())` otherwise.
-pub fn to_result(err: core::ffi::c_int) -> Result {
- if err < 0 {
- Err(Error::from_errno(err))
- } else {
- Ok(())
- }
-}
-
-/// Transform a kernel "error pointer" to a normal pointer.
-///
-/// Some kernel C API functions return an "error pointer" which optionally
-/// embeds an `errno`. Callers are supposed to check the returned pointer
-/// for errors. This function performs the check and converts the "error pointer"
-/// to a normal pointer in an idiomatic fashion.
-///
-/// # Examples
-///
-/// ```ignore
-/// # use kernel::from_err_ptr;
-/// # use kernel::bindings;
-/// fn devm_platform_ioremap_resource(
-/// pdev: &mut PlatformDevice,
-/// index: u32,
-/// ) -> Result<*mut core::ffi::c_void> {
-/// // SAFETY: `pdev` points to a valid platform device. There are no safety requirements
-/// // on `index`.
-/// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) })
-/// }
-/// ```
-// TODO: Remove `dead_code` marker once an in-kernel client is available.
-#[allow(dead_code)]
-pub(crate) fn from_err_ptr(ptr: *mut T) -> Result<*mut T> {
- // CAST: Casting a pointer to `*const core::ffi::c_void` is always valid.
- let const_ptr: *const core::ffi::c_void = ptr.cast();
- // SAFETY: The FFI function does not deref the pointer.
- if unsafe { bindings::IS_ERR(const_ptr) } {
- // SAFETY: The FFI function does not deref the pointer.
- let err = unsafe { bindings::PTR_ERR(const_ptr) };
- // CAST: If `IS_ERR()` returns `true`,
- // then `PTR_ERR()` is guaranteed to return a
- // negative value greater-or-equal to `-bindings::MAX_ERRNO`,
- // which always fits in an `i16`, as per the invariant above.
- // And an `i16` always fits in an `i32`. So casting `err` to
- // an `i32` can never overflow, and is always valid.
- //
- // SAFETY: `IS_ERR()` ensures `err` is a
- // negative value greater-or-equal to `-bindings::MAX_ERRNO`.
- #[allow(clippy::unnecessary_cast)]
- return Err(unsafe { Error::from_errno_unchecked(err as core::ffi::c_int) });
- }
- Ok(ptr)
-}
-
-/// Calls a closure returning a [`crate::error::Result`] and converts the result to
-/// a C integer result.
-///
-/// This is useful when calling Rust functions that return [`crate::error::Result`]
-/// from inside `extern "C"` functions that need to return an integer error result.
-///
-/// `T` should be convertible from an `i16` via `From`.
-///
-/// # Examples
-///
-/// ```ignore
-/// # use kernel::from_result;
-/// # use kernel::bindings;
-/// unsafe extern "C" fn probe_callback(
-/// pdev: *mut bindings::platform_device,
-/// ) -> core::ffi::c_int {
-/// from_result(|| {
-/// let ptr = devm_alloc(pdev)?;
-/// bindings::platform_set_drvdata(pdev, ptr);
-/// Ok(0)
-/// })
-/// }
-/// ```
-// TODO: Remove `dead_code` marker once an in-kernel client is available.
-#[allow(dead_code)]
-pub(crate) fn from_result(f: F) -> T
-where
- T: From,
- F: FnOnce() -> Result,
-{
- match f() {
- Ok(v) => v,
- // NO-OVERFLOW: negative `errno`s are no smaller than `-bindings::MAX_ERRNO`,
- // `-bindings::MAX_ERRNO` fits in an `i16` as per invariant above,
- // therefore a negative `errno` always fits in an `i16` and will not overflow.
- Err(e) => T::from(e.to_errno() as i16),
- }
-}
-
-/// Error message for calling a default function of a [`#[vtable]`](macros::vtable) trait.
-pub const VTABLE_DEFAULT_ERROR: &str =
- "This function must not be called, see the #[vtable] documentation.";
diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs
deleted file mode 100644
--- a/rust/kernel/init.rs
+++ /dev/null
@@ -1,1352 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
-//!
-//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
-//! overflow.
-//!
-//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
-//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
-//!
-//! # Overview
-//!
-//! To initialize a `struct` with an in-place constructor you will need two things:
-//! - an in-place constructor,
-//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc`],
-//! [`UniqueArc`], [`Box`] or any other smart pointer that implements [`InPlaceInit`]).
-//!
-//! To get an in-place constructor there are generally three options:
-//! - directly creating an in-place constructor using the [`pin_init!`] macro,
-//! - a custom function/macro returning an in-place constructor provided by someone else,
-//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
-//!
-//! Aside from pinned initialization, this API also supports in-place construction without pinning,
-//! the macros/types/functions are generally named like the pinned variants without the `pin`
-//! prefix.
-//!
-//! # Examples
-//!
-//! ## Using the [`pin_init!`] macro
-//!
-//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
-//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
-//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
-//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
-//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
-//!
-//! ```rust
-//! # #![allow(clippy::disallowed_names)]
-//! use kernel::sync::{new_mutex, Mutex};
-//! # use core::pin::Pin;
-//! #[pin_data]
-//! struct Foo {
-//! #[pin]
-//! a: Mutex,
-//! b: u32,
-//! }
-//!
-//! let foo = pin_init!(Foo {
-//! a <- new_mutex!(42, "Foo::a"),
-//! b: 24,
-//! });
-//! ```
-//!
-//! `foo` now is of the type [`impl PinInit`]. We can now use any smart pointer that we like
-//! (or just the stack) to actually initialize a `Foo`:
-//!
-//! ```rust
-//! # #![allow(clippy::disallowed_names)]
-//! # use kernel::sync::{new_mutex, Mutex};
-//! # use core::pin::Pin;
-//! # #[pin_data]
-//! # struct Foo {
-//! # #[pin]
-//! # a: Mutex,
-//! # b: u32,
-//! # }
-//! # let foo = pin_init!(Foo {
-//! # a <- new_mutex!(42, "Foo::a"),
-//! # b: 24,
-//! # });
-//! let foo: Result>> = Box::pin_init(foo, GFP_KERNEL);
-//! ```
-//!
-//! For more information see the [`pin_init!`] macro.
-//!
-//! ## Using a custom function/macro that returns an initializer
-//!
-//! Many types from the kernel supply a function/macro that returns an initializer, because the
-//! above method only works for types where you can access the fields.
-//!
-//! ```rust
-//! # use kernel::sync::{new_mutex, Arc, Mutex};
-//! let mtx: Result>> =
-//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
-//! ```
-//!
-//! To declare an init macro/function you just return an [`impl PinInit`]:
-//!
-//! ```rust
-//! # #![allow(clippy::disallowed_names)]
-//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
-//! #[pin_data]
-//! struct DriverData {
-//! #[pin]
-//! status: Mutex,
-//! buffer: Box<[u8; 1_000_000]>,
-//! }
-//!
-//! impl DriverData {
-//! fn new() -> impl PinInit {
-//! try_pin_init!(Self {
-//! status <- new_mutex!(0, "DriverData::status"),
-//! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?,
-//! })
-//! }
-//! }
-//! ```
-//!
-//! ## Manual creation of an initializer
-//!
-//! Often when working with primitives the previous approaches are not sufficient. That is where
-//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
-//! [`impl PinInit`] directly from a closure. Of course you have to ensure that the closure
-//! actually does the initialization in the correct way. Here are the things to look out for
-//! (we are calling the parameter to the closure `slot`):
-//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
-//! `slot` now contains a valid bit pattern for the type `T`,
-//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
-//! you need to take care to clean up anything if your initialization fails mid-way,
-//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
-//! `slot` gets called.
-//!
-//! ```rust
-//! # #![allow(unreachable_pub, clippy::disallowed_names)]
-//! use kernel::{init, types::Opaque};
-//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
-//! # mod bindings {
-//! # #![allow(non_camel_case_types)]
-//! # pub struct foo;
-//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
-//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
-//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
-//! # }
-//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
-//! # trait FromErrno {
-//! # fn from_errno(errno: core::ffi::c_int) -> Error {
-//! # // Dummy error that can be constructed outside the `kernel` crate.
-//! # Error::from(core::fmt::Error)
-//! # }
-//! # }
-//! # impl FromErrno for Error {}
-//! /// # Invariants
-//! ///
-//! /// `foo` is always initialized
-//! #[pin_data(PinnedDrop)]
-//! pub struct RawFoo {
-//! #[pin]
-//! foo: Opaque,
-//! #[pin]
-//! _p: PhantomPinned,
-//! }
-//!
-//! impl RawFoo {
-//! pub fn new(flags: u32) -> impl PinInit {
-//! // SAFETY:
-//! // - when the closure returns `Ok(())`, then it has successfully initialized and
-//! // enabled `foo`,
-//! // - when it returns `Err(e)`, then it has cleaned up before
-//! unsafe {
-//! init::pin_init_from_closure(move |slot: *mut Self| {
-//! // `slot` contains uninit memory, avoid creating a reference.
-//! let foo = addr_of_mut!((*slot).foo);
-//!
-//! // Initialize the `foo`
-//! bindings::init_foo(Opaque::raw_get(foo));
-//!
-//! // Try to enable it.
-//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
-//! if err != 0 {
-//! // Enabling has failed, first clean up the foo and then return the error.
-//! bindings::destroy_foo(Opaque::raw_get(foo));
-//! return Err(Error::from_errno(err));
-//! }
-//!
-//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
-//! Ok(())
-//! })
-//! }
-//! }
-//! }
-//!
-//! #[pinned_drop]
-//! impl PinnedDrop for RawFoo {
-//! fn drop(self: Pin<&mut Self>) {
-//! // SAFETY: Since `foo` is initialized, destroying is safe.
-//! unsafe { bindings::destroy_foo(self.foo.get()) };
-//! }
-//! }
-//! ```
-//!
-//! For the special case where initializing a field is a single FFI-function call that cannot fail,
-//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
-//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
-//! with [`pin_init!`].
-//!
-//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
-//! the `kernel` crate. The [`sync`] module is a good starting point.
-//!
-//! [`sync`]: kernel::sync
-//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
-//! [structurally pinned fields]:
-//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
-//! [stack]: crate::stack_pin_init
-//! [`Arc`]: crate::sync::Arc
-//! [`impl PinInit`]: PinInit
-//! [`impl PinInit`]: PinInit
-//! [`impl Init`]: Init
-//! [`Opaque`]: kernel::types::Opaque
-//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
-//! [`pin_data`]: ::macros::pin_data
-//! [`pin_init!`]: crate::pin_init!
-
-use crate::{
- alloc::{box_ext::BoxExt, AllocError, Flags},
- error::{self, Error},
- sync::UniqueArc,
- types::{Opaque, ScopeGuard},
-};
-use alloc::boxed::Box;
-use core::{
- cell::UnsafeCell,
- convert::Infallible,
- marker::PhantomData,
- mem::MaybeUninit,
- num::*,
- pin::Pin,
- ptr::{self, NonNull},
-};
-
-#[doc(hidden)]
-pub mod __internal;
-#[doc(hidden)]
-pub mod macros;
-
-/// Initialize and pin a type directly on the stack.
-///
-/// # Examples
-///
-/// ```rust
-/// # #![allow(clippy::disallowed_names)]
-/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex,
-/// b: Bar,
-/// }
-///
-/// #[pin_data]
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_pin_init!(let foo = pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: Bar {
-/// x: 64,
-/// },
-/// }));
-/// let foo: Pin<&mut Foo> = foo;
-/// pr_info!("a: {}", &*foo.a.lock());
-/// ```
-///
-/// # Syntax
-///
-/// A normal `let` binding with optional type annotation. The expression is expected to implement
-/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
-/// type, then use [`stack_try_pin_init!`].
-///
-/// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
-#[macro_export]
-macro_rules! stack_pin_init {
- (let $var:ident $(: $t:ty)? = $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
- Ok(res) => res,
- Err(x) => {
- let x: ::core::convert::Infallible = x;
- match x {}
- }
- };
- };
-}
-
-/// Initialize and pin a type directly on the stack.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// # #![allow(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
-/// # use macros::pin_data;
-/// # use core::{alloc::AllocError, pin::Pin};
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex,
-/// b: Box,
-/// }
-///
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_try_pin_init!(let foo: Result, AllocError> = pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: Box::new(Bar {
-/// x: 64,
-/// }, GFP_KERNEL)?,
-/// }));
-/// let foo = foo.unwrap();
-/// pr_info!("a: {}", &*foo.a.lock());
-/// ```
-///
-/// ```rust,ignore
-/// # #![allow(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
-/// # use macros::pin_data;
-/// # use core::{alloc::AllocError, pin::Pin};
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex,
-/// b: Box,
-/// }
-///
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: Box::new(Bar {
-/// x: 64,
-/// }, GFP_KERNEL)?,
-/// }));
-/// pr_info!("a: {}", &*foo.a.lock());
-/// # Ok::<_, AllocError>(())
-/// ```
-///
-/// # Syntax
-///
-/// A normal `let` binding with optional type annotation. The expression is expected to implement
-/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
-/// `=` will propagate this error.
-#[macro_export]
-macro_rules! stack_try_pin_init {
- (let $var:ident $(: $t:ty)? = $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = $crate::init::__internal::StackInit::init($var, val);
- };
- (let $var:ident $(: $t:ty)? =? $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
- };
-}
-
-/// Construct an in-place, pinned initializer for `struct`s.
-///
-/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
-/// [`try_pin_init!`].
-///
-/// The syntax is almost identical to that of a normal `struct` initializer:
-///
-/// ```rust
-/// # #![allow(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// a: usize,
-/// b: Bar,
-/// }
-///
-/// #[pin_data]
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// # fn demo() -> impl PinInit {
-/// let a = 42;
-///
-/// let initializer = pin_init!(Foo {
-/// a,
-/// b: Bar {
-/// x: 64,
-/// },
-/// });
-/// # initializer }
-/// # Box::pin_init(demo(), GFP_KERNEL).unwrap();
-/// ```
-///
-/// Arbitrary Rust expressions can be used to set the value of a variable.
-///
-/// The fields are initialized in the order that they appear in the initializer. So it is possible
-/// to read already initialized fields using raw pointers.
-///
-/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
-/// initializer.
-///
-/// # Init-functions
-///
-/// When working with this API it is often desired to let others construct your types without
-/// giving access to all fields. This is where you would normally write a plain function `new`
-/// that would return a new instance of your type. With this API that is also possible.
-/// However, there are a few extra things to keep in mind.
-///
-/// To create an initializer function, simply declare it like this:
-///
-/// ```rust
-/// # #![allow(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// impl Foo {
-/// fn new() -> impl PinInit {
-/// pin_init!(Self {
-/// a: 42,
-/// b: Bar {
-/// x: 64,
-/// },
-/// })
-/// }
-/// }
-/// ```
-///
-/// Users of `Foo` can now create it like this:
-///
-/// ```rust
-/// # #![allow(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// # impl Foo {
-/// # fn new() -> impl PinInit {
-/// # pin_init!(Self {
-/// # a: 42,
-/// # b: Bar {
-/// # x: 64,
-/// # },
-/// # })
-/// # }
-/// # }
-/// let foo = Box::pin_init(Foo::new(), GFP_KERNEL);
-/// ```
-///
-/// They can also easily embed it into their own `struct`s:
-///
-/// ```rust
-/// # #![allow(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// # impl Foo {
-/// # fn new() -> impl PinInit {
-/// # pin_init!(Self {
-/// # a: 42,
-/// # b: Bar {
-/// # x: 64,
-/// # },
-/// # })
-/// # }
-/// # }
-/// #[pin_data]
-/// struct FooContainer {
-/// #[pin]
-/// foo1: Foo,
-/// #[pin]
-/// foo2: Foo,
-/// other: u32,
-/// }
-///
-/// impl FooContainer {
-/// fn new(other: u32) -> impl PinInit {
-/// pin_init!(Self {
-/// foo1 <- Foo::new(),
-/// foo2 <- Foo::new(),
-/// other,
-/// })
-/// }
-/// }
-/// ```
-///
-/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
-/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
-/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
-///
-/// # Syntax
-///
-/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
-/// the following modifications is expected:
-/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
-/// - In front of the initializer you can write `&this in` to have access to a [`NonNull`]
-/// pointer named `this` inside of the initializer.
-/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
-/// struct, this initializes every field with 0 and then runs all initializers specified in the
-/// body. This can only be done if [`Zeroable`] is implemented for the struct.
-///
-/// For instance:
-///
-/// ```rust
-/// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
-/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
-/// #[pin_data]
-/// #[derive(Zeroable)]
-/// struct Buf {
-/// // `ptr` points into `buf`.
-/// ptr: *mut u8,
-/// buf: [u8; 64],
-/// #[pin]
-/// pin: PhantomPinned,
-/// }
-/// pin_init!(&this in Buf {
-/// buf: [0; 64],
-/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
-/// pin: PhantomPinned,
-/// });
-/// pin_init!(Buf {
-/// buf: [1; 64],
-/// ..Zeroable::zeroed()
-/// });
-/// ```
-///
-/// [`try_pin_init!`]: kernel::try_pin_init
-/// [`NonNull`]: core::ptr::NonNull
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! pin_init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error(::core::convert::Infallible),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
-}
-
-/// Construct an in-place, fallible pinned initializer for `struct`s.
-///
-/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
-///
-/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
-/// initialization and return the error.
-///
-/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
-/// initialization fails, the memory can be safely deallocated without any further modifications.
-///
-/// This macro defaults the error to [`Error`].
-///
-/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
-/// after the `struct` initializer to specify the error type you want to use.
-///
-/// # Examples
-///
-/// ```rust
-/// # #![feature(new_uninit)]
-/// use kernel::{init::{self, PinInit}, error::Error};
-/// #[pin_data]
-/// struct BigBuf {
-/// big: Box<[u8; 1024 * 1024 * 1024]>,
-/// small: [u8; 1024 * 1024],
-/// ptr: *mut u8,
-/// }
-///
-/// impl BigBuf {
-/// fn new() -> impl PinInit {
-/// try_pin_init!(Self {
-/// big: Box::init(init::zeroed(), GFP_KERNEL)?,
-/// small: [0; 1024 * 1024],
-/// ptr: core::ptr::null_mut(),
-/// }? Error)
-/// }
-/// }
-/// ```
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! try_pin_init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)? ),
- @fields($($fields)*),
- @error($crate::error::Error),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }? $err:ty) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)? ),
- @fields($($fields)*),
- @error($err),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
-}
-
-/// Construct an in-place initializer for `struct`s.
-///
-/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
-/// [`try_init!`].
-///
-/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
-/// - `unsafe` code must guarantee either full initialization or return an error and allow
-/// deallocation of the memory.
-/// - the fields are initialized in the order given in the initializer.
-/// - no references to fields are allowed to be created inside of the initializer.
-///
-/// This initializer is for initializing data in-place that might later be moved. If you want to
-/// pin-initialize, use [`pin_init!`].
-///
-/// [`try_init!`]: crate::try_init!
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error(::core::convert::Infallible),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
- }
-}
-
-/// Construct an in-place fallible initializer for `struct`s.
-///
-/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
-/// [`init!`].
-///
-/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
-/// append `? $type` after the `struct` initializer.
-/// The safety caveats from [`try_pin_init!`] also apply:
-/// - `unsafe` code must guarantee either full initialization or return an error and allow
-/// deallocation of the memory.
-/// - the fields are initialized in the order given in the initializer.
-/// - no references to fields are allowed to be created inside of the initializer.
-///
-/// # Examples
-///
-/// ```rust
-/// use kernel::{init::{PinInit, zeroed}, error::Error};
-/// struct BigBuf {
-/// big: Box<[u8; 1024 * 1024 * 1024]>,
-/// small: [u8; 1024 * 1024],
-/// }
-///
-/// impl BigBuf {
-/// fn new() -> impl Init {
-/// try_init!(Self {
-/// big: Box::init(zeroed(), GFP_KERNEL)?,
-/// small: [0; 1024 * 1024],
-/// }? Error)
-/// }
-/// }
-/// ```
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! try_init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error($crate::error::Error),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
- };
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }? $err:ty) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error($err),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
- };
-}
-
-/// A pin-initializer for the type `T`.
-///
-/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
-/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the
-/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc`] on this.
-///
-/// Also see the [module description](self).
-///
-/// # Safety
-///
-/// When implementing this trait you will need to take great care. Also there are probably very few
-/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
-///
-/// The [`PinInit::__pinned_init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// [`Arc`]: crate::sync::Arc
-/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
-#[must_use = "An initializer must be used in order to create its value."]
-pub unsafe trait PinInit: Sized {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- /// - `slot` will not move until it is dropped, i.e. it will be pinned.
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
-
- /// First initializes the value using `self` then calls the function `f` with the initialized
- /// value.
- ///
- /// If `f` returns an error the value is dropped and the initializer will forward the error.
- ///
- /// # Examples
- ///
- /// ```rust
- /// # #![allow(clippy::disallowed_names)]
- /// use kernel::{types::Opaque, init::pin_init_from_closure};
- /// #[repr(C)]
- /// struct RawFoo([u8; 16]);
- /// extern {
- /// fn init_foo(_: *mut RawFoo);
- /// }
- ///
- /// #[pin_data]
- /// struct Foo {
- /// #[pin]
- /// raw: Opaque,
- /// }
- ///
- /// impl Foo {
- /// fn setup(self: Pin<&mut Self>) {
- /// pr_info!("Setting up foo");
- /// }
- /// }
- ///
- /// let foo = pin_init!(Foo {
- /// raw <- unsafe {
- /// Opaque::ffi_init(|s| {
- /// init_foo(s);
- /// })
- /// },
- /// }).pin_chain(|foo| {
- /// foo.setup();
- /// Ok(())
- /// });
- /// ```
- fn pin_chain(self, f: F) -> ChainPinInit
- where
- F: FnOnce(Pin<&mut T>) -> Result<(), E>,
- {
- ChainPinInit(self, f, PhantomData)
- }
-}
-
-/// An initializer returned by [`PinInit::pin_chain`].
-pub struct ChainPinInit(I, F, __internal::Invariant<(E, Box)>);
-
-// SAFETY: The `__pinned_init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
-// - considers `slot` pinned.
-unsafe impl PinInit for ChainPinInit
-where
- I: PinInit,
- F: FnOnce(Pin<&mut T>) -> Result<(), E>,
-{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- let val = unsafe { &mut *slot };
- // SAFETY: `slot` is considered pinned.
- let val = unsafe { Pin::new_unchecked(val) };
- (self.1)(val).map_err(|e| {
- // SAFETY: `slot` was initialized above.
- unsafe { core::ptr::drop_in_place(slot) };
- e
- })
- }
-}
-
-/// An initializer for `T`.
-///
-/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
-/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the
-/// [`InPlaceInit::init`] function of a smart pointer like [`Arc`] on this. Because
-/// [`PinInit`] is a super trait, you can use every function that takes it as well.
-///
-/// Also see the [module description](self).
-///
-/// # Safety
-///
-/// When implementing this trait you will need to take great care. Also there are probably very few
-/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
-///
-/// The [`Init::__init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
-/// code as `__init`.
-///
-/// Contrary to its supertype [`PinInit`] the caller is allowed to
-/// move the pointee after initialization.
-///
-/// [`Arc`]: crate::sync::Arc
-#[must_use = "An initializer must be used in order to create its value."]
-pub unsafe trait Init: PinInit {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
-
- /// First initializes the value using `self` then calls the function `f` with the initialized
- /// value.
- ///
- /// If `f` returns an error the value is dropped and the initializer will forward the error.
- ///
- /// # Examples
- ///
- /// ```rust
- /// # #![allow(clippy::disallowed_names)]
- /// use kernel::{types::Opaque, init::{self, init_from_closure}};
- /// struct Foo {
- /// buf: [u8; 1_000_000],
- /// }
- ///
- /// impl Foo {
- /// fn setup(&mut self) {
- /// pr_info!("Setting up foo");
- /// }
- /// }
- ///
- /// let foo = init!(Foo {
- /// buf <- init::zeroed()
- /// }).chain(|foo| {
- /// foo.setup();
- /// Ok(())
- /// });
- /// ```
- fn chain(self, f: F) -> ChainInit
- where
- F: FnOnce(&mut T) -> Result<(), E>,
- {
- ChainInit(self, f, PhantomData)
- }
-}
-
-/// An initializer returned by [`Init::chain`].
-pub struct ChainInit(I, F, __internal::Invariant<(E, Box)>);
-
-// SAFETY: The `__init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
-unsafe impl Init for ChainInit
-where
- I: Init,
- F: FnOnce(&mut T) -> Result<(), E>,
-{
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- (self.1)(unsafe { &mut *slot }).map_err(|e| {
- // SAFETY: `slot` was initialized above.
- unsafe { core::ptr::drop_in_place(slot) };
- e
- })
- }
-}
-
-// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
-unsafe impl PinInit for ChainInit
-where
- I: Init,
- F: FnOnce(&mut T) -> Result<(), E>,
-{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
- unsafe { self.__init(slot) }
- }
-}
-
-/// Creates a new [`PinInit`] from the given closure.
-///
-/// # Safety
-///
-/// The closure:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - may assume that the `slot` does not move if `T: !Unpin`,
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-#[inline]
-pub const unsafe fn pin_init_from_closure(
- f: impl FnOnce(*mut T) -> Result<(), E>,
-) -> impl PinInit {
- __internal::InitClosure(f, PhantomData)
-}
-
-/// Creates a new [`Init`] from the given closure.
-///
-/// # Safety
-///
-/// The closure:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - the `slot` may move after initialization.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-#[inline]
-pub const unsafe fn init_from_closure(
- f: impl FnOnce(*mut T) -> Result<(), E>,
-) -> impl Init {
- __internal::InitClosure(f, PhantomData)
-}
-
-/// An initializer that leaves the memory uninitialized.
-///
-/// The initializer is a no-op. The `slot` memory is not changed.
-#[inline]
-pub fn uninit() -> impl Init, E> {
- // SAFETY: The memory is allowed to be uninitialized.
- unsafe { init_from_closure(|_| Ok(())) }
-}
-
-/// Initializes an array by initializing each element via the provided initializer.
-///
-/// # Examples
-///
-/// ```rust
-/// use kernel::{error::Error, init::init_array_from_fn};
-/// let array: Box<[usize; 1_000]> = Box::init::(init_array_from_fn(|i| i), GFP_KERNEL).unwrap();
-/// assert_eq!(array.len(), 1_000);
-/// ```
-pub fn init_array_from_fn(
- mut make_init: impl FnMut(usize) -> I,
-) -> impl Init<[T; N], E>
-where
- I: Init,
-{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::();
- // Counts the number of initialized elements and when dropped drops that many elements from
- // `slot`.
- let mut init_count = ScopeGuard::new_with_data(0, |i| {
- // We now free every element that has been initialized before.
- // SAFETY: The loop initialized exactly the values from 0..i and since we
- // return `Err` below, the caller will consider the memory at `slot` as
- // uninitialized.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- });
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- unsafe { init.__init(ptr) }?;
- *init_count += 1;
- }
- init_count.dismiss();
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { init_from_closure(init) }
-}
-
-/// Initializes an array by initializing each element via the provided initializer.
-///
-/// # Examples
-///
-/// ```rust
-/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
-/// let array: Arc<[Mutex; 1_000]> =
-/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap();
-/// assert_eq!(array.len(), 1_000);
-/// ```
-pub fn pin_init_array_from_fn(
- mut make_init: impl FnMut(usize) -> I,
-) -> impl PinInit<[T; N], E>
-where
- I: PinInit,
-{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::();
- // Counts the number of initialized elements and when dropped drops that many elements from
- // `slot`.
- let mut init_count = ScopeGuard::new_with_data(0, |i| {
- // We now free every element that has been initialized before.
- // SAFETY: The loop initialized exactly the values from 0..i and since we
- // return `Err` below, the caller will consider the memory at `slot` as
- // uninitialized.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- });
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- unsafe { init.__pinned_init(ptr) }?;
- *init_count += 1;
- }
- init_count.dismiss();
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { pin_init_from_closure(init) }
-}
-
-// SAFETY: Every type can be initialized by-value.
-unsafe impl Init for T {
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- unsafe { slot.write(self) };
- Ok(())
- }
-}
-
-// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
-unsafe impl PinInit for T {
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- unsafe { self.__init(slot) }
- }
-}
-
-/// Smart pointer that can initialize memory in-place.
-pub trait InPlaceInit: Sized {
- /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- /// type.
- ///
- /// If `T: !Unpin` it will not be able to move afterwards.
- fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E>
- where
- E: From;
-
- /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- /// type.
- ///
- /// If `T: !Unpin` it will not be able to move afterwards.
- fn pin_init(init: impl PinInit, flags: Flags) -> error::Result>
- where
- Error: From,
- {
- // SAFETY: We delegate to `init` and only change the error type.
- let init = unsafe {
- pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- };
- Self::try_pin_init(init, flags)
- }
-
- /// Use the given initializer to in-place initialize a `T`.
- fn try_init(init: impl Init, flags: Flags) -> Result
- where
- E: From;
-
- /// Use the given initializer to in-place initialize a `T`.
- fn init(init: impl Init, flags: Flags) -> error::Result
- where
- Error: From,
- {
- // SAFETY: We delegate to `init` and only change the error type.
- let init = unsafe {
- init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- };
- Self::try_init(init, flags)
- }
-}
-
-impl InPlaceInit for Box {
- #[inline]
- fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E>
- where
- E: From,
- {
- let mut this = as BoxExt<_>>::new_uninit(flags)?;
- let slot = this.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { this.assume_init() }.into())
- }
-
- #[inline]
- fn try_init(init: impl Init, flags: Flags) -> Result
- where
- E: From,
- {
- let mut this = as BoxExt<_>>::new_uninit(flags)?;
- let slot = this.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid.
- unsafe { init.__init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { this.assume_init() })
- }
-}
-
-impl InPlaceInit for UniqueArc {
- #[inline]
- fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E>
- where
- E: From,
- {
- let mut this = UniqueArc::new_uninit(flags)?;
- let slot = this.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { this.assume_init() }.into())
- }
-
- #[inline]
- fn try_init(init: impl Init, flags: Flags) -> Result
- where
- E: From,
- {
- let mut this = UniqueArc::new_uninit(flags)?;
- let slot = this.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid.
- unsafe { init.__init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { this.assume_init() })
- }
-}
-
-/// Trait facilitating pinned destruction.
-///
-/// Use [`pinned_drop`] to implement this trait safely:
-///
-/// ```rust
-/// # use kernel::sync::Mutex;
-/// use kernel::macros::pinned_drop;
-/// use core::pin::Pin;
-/// #[pin_data(PinnedDrop)]
-/// struct Foo {
-/// #[pin]
-/// mtx: Mutex,
-/// }
-///
-/// #[pinned_drop]
-/// impl PinnedDrop for Foo {
-/// fn drop(self: Pin<&mut Self>) {
-/// pr_info!("Foo is being dropped!");
-/// }
-/// }
-/// ```
-///
-/// # Safety
-///
-/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
-///
-/// [`pinned_drop`]: kernel::macros::pinned_drop
-pub unsafe trait PinnedDrop: __internal::HasPinData {
- /// Executes the pinned destructor of this type.
- ///
- /// While this function is marked safe, it is actually unsafe to call it manually. For this
- /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
- /// and thus prevents this function from being called where it should not.
- ///
- /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
- /// automatically.
- fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
-}
-
-/// Marker trait for types that can be initialized by writing just zeroes.
-///
-/// # Safety
-///
-/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
-/// this is not UB:
-///
-/// ```rust,ignore
-/// let val: Self = unsafe { core::mem::zeroed() };
-/// ```
-pub unsafe trait Zeroable {}
-
-/// Create a new zeroed T.
-///
-/// The returned initializer will write `0x00` to every byte of the given `slot`.
-#[inline]
-pub fn zeroed() -> impl Init {
- // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
- // and because we write all zeroes, the memory is initialized.
- unsafe {
- init_from_closure(|slot: *mut T| {
- slot.write_bytes(0, 1);
- Ok(())
- })
- }
-}
-
-macro_rules! impl_zeroable {
- ($($({$($generics:tt)*})? $t:ty, )*) => {
- $(unsafe impl$($($generics)*)? Zeroable for $t {})*
- };
-}
-
-impl_zeroable! {
- // SAFETY: All primitives that are allowed to be zero.
- bool,
- char,
- u8, u16, u32, u64, u128, usize,
- i8, i16, i32, i64, i128, isize,
- f32, f64,
-
- // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
- // creating an instance of an uninhabited type is immediate undefined behavior. For more on
- // uninhabited/empty types, consult The Rustonomicon:
- // . The Rust Reference
- // also has information on undefined behavior:
- // .
- //
- // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
- {} PhantomData, core::marker::PhantomPinned, (),
-
- // SAFETY: Type is allowed to take any value, including all zeros.
- {} MaybeUninit,
- // SAFETY: Type is allowed to take any value, including all zeros.
- {} Opaque,
-
- // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
- {} UnsafeCell,
-
- // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- Option, Option, Option, Option,
- Option, Option,
- Option, Option, Option, Option,
- Option, Option,
-
- // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- //
- // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
- {} Option>,
- {} Option>,
-
- // SAFETY: `null` pointer is valid.
- //
- // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
- // null.
- //
- // When `Pointee` gets stabilized, we could use
- // `T: ?Sized where ::Metadata: Zeroable`
- {} *mut T, {} *const T,
-
- // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
- // zero.
- {} *mut [T], {} *const [T], *mut str, *const str,
-
- // SAFETY: `T` is `Zeroable`.
- {} [T; N], {} Wrapping,
-}
-
-macro_rules! impl_tuple_zeroable {
- ($(,)?) => {};
- ($first:ident, $($t:ident),* $(,)?) => {
- // SAFETY: All elements are zeroable and padding can be zero.
- unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
- impl_tuple_zeroable!($($t),* ,);
- }
-}
-
-impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs
deleted file mode 100644
--- a/rust/kernel/init/__internal.rs
+++ /dev/null
@@ -1,230 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-//! This module contains API-internal items for pin-init.
-//!
-//! These items must not be used outside of
-//! - `kernel/init.rs`
-//! - `macros/pin_data.rs`
-//! - `macros/pinned_drop.rs`
-
-use super::*;
-
-/// See the [nomicon] for what subtyping is. See also [this table].
-///
-/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
-/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
-pub(super) type Invariant = PhantomData *mut T>;
-
-/// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this
-/// type, since the closure needs to fulfill the same safety requirement as the
-/// `__pinned_init`/`__init` functions.
-pub(crate) struct InitClosure(pub(crate) F, pub(crate) Invariant<(E, T)>);
-
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__init` invariants.
-unsafe impl Init for InitClosure
-where
- F: FnOnce(*mut T) -> Result<(), E>,
-{
- #[inline]
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
-}
-
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__pinned_init` invariants.
-unsafe impl PinInit for InitClosure
-where
- F: FnOnce(*mut T) -> Result<(), E>,
-{
- #[inline]
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
-}
-
-/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate
-/// the pin projections within the initializers.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait HasPinData {
- type PinData: PinData;
-
- unsafe fn __pin_data() -> Self::PinData;
-}
-
-/// Marker trait for pinning data of structs.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait PinData: Copy {
- type Datee: ?Sized + HasPinData;
-
- /// Type inference helper function.
- fn make_closure(self, f: F) -> F
- where
- F: FnOnce(*mut Self::Datee) -> Result