4.18的kernel上无法卸载内核驱动模块

本人的另外一个博客地址: https://www.joeys.top,欢迎大家光临，哈哈哈

问题描述： module在4.18的kernel上面无法在rmmod的时候会如下报错

[root@node-1 zjp]# insmod hello7.ko [root@node-1 zjp]# rmmod hello7.ko rmmod: ERROR: could not remove 'hello7': Device or resource busy rmmod: ERROR: could not remove module hello7.ko: Device or resource busy

在rmmod卸载module的时候会调用delete_module，在return

EBUSY The module is not "live" (i.e., it is still being initialized or is already marked for removal); or, the module has an init function but has no exit function, and O_TRUNC was not specified in flags.

调查过程

查看已经安装的module信息

[root@node-1 zjp]# cat /proc/modules | grep hello7 hello7 262144 0 [permanent], Live 0xffff4f26408b0000 (OE)

这个打印的代码

1005 if (mod->init != NULL && mod->exit == NULL) { 1006 printed_something = 1; 1007 seq_printf(m, "[permanent],"); 1008 } 1009

卸载的时候

948 /* If it has an init func, it must have an exit func to unload */ 949 if (mod->init && !mod->exit) { 950 forced = try_force_unload(flags); 951 if (!forced) { 952 /* This module can't be removed */ 953 ret = -EBUSY; 954 goto out; 955 } 956 }

代码回去强制卸载如果调用 try_force_unload函数

811 #ifdef CONFIG_MODULE_FORCE_UNLOAD 812 static inline int try_force_unload(unsigned int flags) 813 { 814 int ret = (flags & O_TRUNC); 815 if (ret) 816 add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE); 817 return ret; 818 } 819 #else 820 static inline int try_force_unload(unsigned int flags) 821 { 822 return 0; 823 } 824 #endif /* CONFIG_MODULE_FORCE_UNLOAD */

4.18的系统没有配置这个

[root@node-1 zjp]# grep -r "CONFIG_MODULE_FORCE_UNLOAD" /boot/config-4.18.0-80.11.2.el8.es.2.aarch64 # CONFIG_MODULE_FORCE_UNLOAD is not set

根本原因是 mod->exit为NULL LKM cleanup_module() not called

在测试的module的Makefile上添加 KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO

LINUX_KERNEL_PATH := /lib/modules/$(shell uname -r)/build #Linux内核源代码的绝对路径 KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO all: make -C $(LINUX_KERNEL_PATH) M=$(CURRENT_PATH) modules #编译模块了 clean: make -C $(LINUX_KERNEL_PATH) M=$(CURRENT_PATH) clean #清理 ⋊> ~/work-dir vim kernel-4.18.0/linux-4.18.0-80.1.2.el8_0/Makefile 500 # check for 'asm goto' 501 ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y) 502 CC_HAVE_ASM_GOTO := 1 503 KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO 504 KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO 505 endif

4.14的Makefile文件

# check for 'asm goto' ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y) KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO endif

/scripts/gcc-goto.sh内容

1 #!/bin/sh 2 # SPDX-License-Identifier: GPL-2.0 3 # Test for gcc 'asm goto' support 4 # Copyright (C) 2010, Jason Baron <jbaron@redhat.com> 5 6 cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" 7 int main(void) 8 { 9 #if defined(__arm__) || defined(__aarch64__) 10 /* 11 * Not related to asm goto, but used by jump label 12 * and broken on some ARM GCC versions (see GCC Bug 48637). 13 */ 14 static struct { int dummy; int state; } tp; 15 asm (".long %c0" :: "i" (&tp.state)); 16 #endif 17 18 entry: 19 asm goto ("" :::: entry); 20 return 0; 21 } 22 END

抓取Makefile的执行过程 make V=1 SHELL=’sh -x’ > node-1-build.log 2>&1

+ /usr/bin/sh /usr/src/kernels/4.18.0-80.11.2.es.2.aarch64/scripts/gcc-goto.sh gcc -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -fno-common -fshort-wchar \ -Werror-implicit-function-declaration \ -Wno-format-security \ -std=gnu89

输出内容报错

<stdin>: In function ‘main’: <stdin>:9:2: error: invalid 'asm': invalid operand prefix '%c'

结果 得知4.18的编译环境是centos8的mock环境,4,18版本kernel两者的gcc版本差异很大,测试在centos8的mock环境编译的module是带有DCC_HAVE_ASM_GOTO,而4.14在build module的时候是没有带有 DCC_HAVE_ASM_GOTO;

参考链接：[v1,5/7] arm64, jump label: detect %c support for ARM64LKM cleanup_module() not called

编译参数打印

+ gcc -Wp,-MD,/root/zjp/zjp/.hello9.mod.o.d -nostdinc -isystem /usr/lib/gcc/aarch64-redhat-linux/4.8.5/include -I./arch/arm64/include -I./arch/arm64/include/generated -I./include -I./arch/arm64/include/uapi -I./arch/arm64/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -include ./include/linux/compiler_types.h -D__KERNEL__ -mlittle-endian -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -fshort-wchar -Werror-implicit-function-declaration -Wno-format-security -std=gnu89 -fno-PIE -fno-PIE -mgeneral-regs-only -DCONFIG_AS_LSE=1 -fno-asynchronous-unwind-tables -fno-delete-null-pointer-checks -O2 -Wno-maybe-uninitialized --param=allow-store-data-races=0 -Wframe-larger-than=2048 -fstack-protector-strong -Wno-unused-but-set-variable -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-var-tracking-assignments -g -gdwarf-4 -pg -fno-inline-functions-called-once -Wdeclaration-after-statement -Wno-pointer-sign -fno-strict-overflow -fno-merge-all-constants -fmerge-constants -fno-stack-check -fconserve-stack -Werror=implicit-int -Werror=strict-prototypes '-DKBUILD_BASENAME="hello9.mod"' '-DKBUILD_MODNAME="hello9"' -DMODULE -c -o /root/zjp/zjp/hello9.mod.o /root/zjp/zjp/hello9.mod.c + scripts/basic/fixdep /root/zjp/zjp/.hello9.mod.o.d /root/zjp/zjp/hello9.mod.o 'gcc -Wp,-MD,/root/zjp/zjp/.hello9.mod.o.d -nostdinc -isystem /usr/lib/gcc/aarch64-redhat-linux/4.8.5/include -I./arch/arm64/include -I./arch/arm64/include/generated -I./include -I./arch/arm64/include/uapi -I./arch/arm64/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -include ./include/linux/compiler_types.h -D__KERNEL__ -mlittle-endian -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -fshort-wchar -Werror-implicit-function-declaration -Wno-format-security -std=gnu89 -fno-PIE -fno-PIE -mgeneral-regs-only -DCONFIG_AS_LSE=1 -fno-asynchronous-unwind-tables -fno-delete-null-pointer-checks -O2 -Wno-maybe-uninitialized --param=allow-store-data-races=0 -Wframe-larger-than=2048 -fstack-protector-strong -Wno-unused-but-set-variable -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-var-tracking-assignments -g -gdwarf-4 -pg -fno-inline-functions-called-once -Wdeclaration-after-statement -Wno-pointer-sign -fno-strict-overflow -fno-merge-all-constants -fmerge-constants -fno-stack-check -fconserve-stack -Werror=implicit-int -Werror=strict-prototypes -DKBUILD_BASENAME='\''"hello9.mod"'\'' -DKBUILD_MODNAME='\''"hello9"'\'' -DMODULE -c -o /root/zjp/zjp/hello9.mod.o /root/zjp/zjp/hello9.mod.c'