diff options
Diffstat (limited to 'arch')
609 files changed, 8011 insertions, 7378 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index 17fe351cdde0..516f2b05bd66 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -540,11 +540,17 @@ config HAVE_CONTEXT_TRACKING help Provide kernel/user boundaries probes necessary for subsystems that need it, such as userspace RCU extended quiescent state. - Syscalls need to be wrapped inside user_exit()-user_enter() through - the slow path using TIF_NOHZ flag. Exceptions handlers must be - wrapped as well. Irqs are already protected inside - rcu_irq_enter/rcu_irq_exit() but preemption or signal handling on - irq exit still need to be protected. + Syscalls need to be wrapped inside user_exit()-user_enter(), either + optimized behind static key or through the slow path using TIF_NOHZ + flag. Exceptions handlers must be wrapped as well. Irqs are already + protected inside rcu_irq_enter/rcu_irq_exit() but preemption or signal + handling on irq exit still need to be protected. + +config HAVE_TIF_NOHZ + bool + help + Arch relies on TIF_NOHZ and syscall slow path to implement context + tracking calls to user_enter()/user_exit(). config HAVE_VIRT_CPU_ACCOUNTING bool diff --git a/arch/alpha/include/asm/futex.h b/arch/alpha/include/asm/futex.h index bfd3c01038f8..da67afd578fd 100644 --- a/arch/alpha/include/asm/futex.h +++ b/arch/alpha/include/asm/futex.h @@ -31,7 +31,8 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, { int oldval = 0, ret; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -53,8 +54,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/alpha/kernel/irq_alpha.c b/arch/alpha/kernel/irq_alpha.c index da3e10d5f7fe..d17e44c99df9 100644 --- a/arch/alpha/kernel/irq_alpha.c +++ b/arch/alpha/kernel/irq_alpha.c @@ -213,32 +213,13 @@ process_mcheck_info(unsigned long vector, unsigned long la_ptr, * The special RTC interrupt type. The interrupt itself was * processed by PALcode, and comes in via entInt vector 1. */ - -struct irqaction timer_irqaction = { - .handler = rtc_timer_interrupt, - .name = "timer", -}; - void __init -init_rtc_irq(void) +init_rtc_irq(irq_handler_t handler) { irq_set_chip_and_handler_name(RTC_IRQ, &dummy_irq_chip, handle_percpu_irq, "RTC"); - setup_irq(RTC_IRQ, &timer_irqaction); + if (!handler) + handler = rtc_timer_interrupt; + if (request_irq(RTC_IRQ, handler, 0, "timer", NULL)) + pr_err("Failed to register timer interrupt\n"); } - -/* Dummy irqactions. */ -struct irqaction isa_cascade_irqaction = { - .handler = no_action, - .name = "isa-cascade" -}; - -struct irqaction timer_cascade_irqaction = { - .handler = no_action, - .name = "timer-cascade" -}; - -struct irqaction halt_switch_irqaction = { - .handler = no_action, - .name = "halt-switch" -}; diff --git a/arch/alpha/kernel/irq_i8259.c b/arch/alpha/kernel/irq_i8259.c index 5d54c076a8ae..1dcf0d9038fd 100644 --- a/arch/alpha/kernel/irq_i8259.c +++ b/arch/alpha/kernel/irq_i8259.c @@ -82,11 +82,6 @@ struct irq_chip i8259a_irq_type = { void __init init_i8259a_irqs(void) { - static struct irqaction cascade = { - .handler = no_action, - .name = "cascade", - }; - long i; outb(0xff, 0x21); /* mask all of 8259A-1 */ @@ -96,7 +91,8 @@ init_i8259a_irqs(void) irq_set_chip_and_handler(i, &i8259a_irq_type, handle_level_irq); } - setup_irq(2, &cascade); + if (request_irq(2, no_action, 0, "cascade", NULL)) + pr_err("Failed to request irq 2 (cascade)\n"); } diff --git a/arch/alpha/kernel/irq_impl.h b/arch/alpha/kernel/irq_impl.h index 16f2b0276f3a..fbf21892e66d 100644 --- a/arch/alpha/kernel/irq_impl.h +++ b/arch/alpha/kernel/irq_impl.h @@ -21,14 +21,9 @@ extern void isa_no_iack_sc_device_interrupt(unsigned long); extern void srm_device_interrupt(unsigned long); extern void pyxis_device_interrupt(unsigned long); -extern struct irqaction timer_irqaction; -extern struct irqaction isa_cascade_irqaction; -extern struct irqaction timer_cascade_irqaction; -extern struct irqaction halt_switch_irqaction; - extern void init_srm_irqs(long, unsigned long); extern void init_pyxis_irqs(unsigned long); -extern void init_rtc_irq(void); +extern void init_rtc_irq(irq_handler_t handler); extern void common_init_isa_dma(void); diff --git a/arch/alpha/kernel/irq_pyxis.c b/arch/alpha/kernel/irq_pyxis.c index a968b10e687d..27070b5bd33e 100644 --- a/arch/alpha/kernel/irq_pyxis.c +++ b/arch/alpha/kernel/irq_pyxis.c @@ -107,5 +107,6 @@ init_pyxis_irqs(unsigned long ignore_mask) irq_set_status_flags(i, IRQ_LEVEL); } - setup_irq(16+7, &isa_cascade_irqaction); + if (request_irq(16 + 7, no_action, 0, "isa-cascade", NULL)) + pr_err("Failed to register isa-cascade interrupt\n"); } diff --git a/arch/alpha/kernel/sys_alcor.c b/arch/alpha/kernel/sys_alcor.c index e56efd5b855f..ce5430056f65 100644 --- a/arch/alpha/kernel/sys_alcor.c +++ b/arch/alpha/kernel/sys_alcor.c @@ -133,7 +133,8 @@ alcor_init_irq(void) init_i8259a_irqs(); common_init_isa_dma(); - setup_irq(16+31, &isa_cascade_irqaction); + if (request_irq(16 + 31, no_action, 0, "isa-cascade", NULL)) + pr_err("Failed to register isa-cascade interrupt\n"); } diff --git a/arch/alpha/kernel/sys_cabriolet.c b/arch/alpha/kernel/sys_cabriolet.c index 10bc46a4ec40..0aa6a27d0e2f 100644 --- a/arch/alpha/kernel/sys_cabriolet.c +++ b/arch/alpha/kernel/sys_cabriolet.c @@ -112,7 +112,8 @@ common_init_irq(void (*srm_dev_int)(unsigned long v)) } common_init_isa_dma(); - setup_irq(16+4, &isa_cascade_irqaction); + if (request_irq(16 + 4, no_action, 0, "isa-cascade", NULL)) + pr_err("Failed to register isa-cascade interrupt\n"); } #ifndef CONFIG_ALPHA_PC164 diff --git a/arch/alpha/kernel/sys_eb64p.c b/arch/alpha/kernel/sys_eb64p.c index 5251937ec1b4..1cdfe55fb987 100644 --- a/arch/alpha/kernel/sys_eb64p.c +++ b/arch/alpha/kernel/sys_eb64p.c @@ -123,7 +123,8 @@ eb64p_init_irq(void) } common_init_isa_dma(); - setup_irq(16+5, &isa_cascade_irqaction); + if (request_irq(16 + 5, no_action, 0, "isa-cascade", NULL)) + pr_err("Failed to register isa-cascade interrupt\n"); } /* diff --git a/arch/alpha/kernel/sys_marvel.c b/arch/alpha/kernel/sys_marvel.c index 8d34cf6e002a..533899a4a1a1 100644 --- a/arch/alpha/kernel/sys_marvel.c +++ b/arch/alpha/kernel/sys_marvel.c @@ -397,7 +397,7 @@ marvel_init_pci(void) static void __init marvel_init_rtc(void) { - init_rtc_irq(); + init_rtc_irq(NULL); } static void diff --git a/arch/alpha/kernel/sys_miata.c b/arch/alpha/kernel/sys_miata.c index 6fa07dc5339d..702292af2225 100644 --- a/arch/alpha/kernel/sys_miata.c +++ b/arch/alpha/kernel/sys_miata.c @@ -81,8 +81,10 @@ miata_init_irq(void) init_pyxis_irqs(0x63b0000); common_init_isa_dma(); - setup_irq(16+2, &halt_switch_irqaction); /* SRM only? */ - setup_irq(16+6, &timer_cascade_irqaction); + if (request_irq(16 + 2, no_action, 0, "halt-switch", NULL)) + pr_err("Failed to register halt-switch interrupt\n"); + if (request_irq(16 + 6, no_action, 0, "timer-cascade", NULL)) + pr_err("Failed to register timer-cascade interrupt\n"); } diff --git a/arch/alpha/kernel/sys_ruffian.c b/arch/alpha/kernel/sys_ruffian.c index 07830cccabf9..d33074011960 100644 --- a/arch/alpha/kernel/sys_ruffian.c +++ b/arch/alpha/kernel/sys_ruffian.c @@ -82,7 +82,8 @@ ruffian_init_rtc(void) outb(0x31, 0x42); outb(0x13, 0x42); - setup_irq(0, &timer_irqaction); + if (request_irq(0, rtc_timer_interrupt, 0, "timer", NULL)) + pr_err("Failed to request irq 0 (timer)\n"); } static void diff --git a/arch/alpha/kernel/sys_rx164.c b/arch/alpha/kernel/sys_rx164.c index a3db719d3c38..4d85eaeb44aa 100644 --- a/arch/alpha/kernel/sys_rx164.c +++ b/arch/alpha/kernel/sys_rx164.c @@ -106,7 +106,8 @@ rx164_init_irq(void) init_i8259a_irqs(); common_init_isa_dma(); - setup_irq(16+20, &isa_cascade_irqaction); + if (request_irq(16 + 20, no_action, 0, "isa-cascade", NULL)) + pr_err("Failed to register isa-cascade interrupt\n"); } diff --git a/arch/alpha/kernel/sys_sx164.c b/arch/alpha/kernel/sys_sx164.c index 1ec638a2746a..17cc203176c8 100644 --- a/arch/alpha/kernel/sys_sx164.c +++ b/arch/alpha/kernel/sys_sx164.c @@ -54,7 +54,8 @@ sx164_init_irq(void) else init_pyxis_irqs(0xff00003f0000UL); - setup_irq(16+6, &timer_cascade_irqaction); + if (request_irq(16 + 6, no_action, 0, "timer-cascade", NULL)) + pr_err("Failed to register timer-cascade interrupt\n"); } /* diff --git a/arch/alpha/kernel/sys_wildfire.c b/arch/alpha/kernel/sys_wildfire.c index 8e64052811ab..2191bde161fd 100644 --- a/arch/alpha/kernel/sys_wildfire.c +++ b/arch/alpha/kernel/sys_wildfire.c @@ -156,10 +156,6 @@ static void __init wildfire_init_irq_per_pca(int qbbno, int pcano) { int i, irq_bias; - static struct irqaction isa_enable = { - .handler = no_action, - .name = "isa_enable", - }; irq_bias = qbbno * (WILDFIRE_PCA_PER_QBB * WILDFIRE_IRQ_PER_PCA) + pcano * WILDFIRE_IRQ_PER_PCA; @@ -198,7 +194,8 @@ wildfire_init_irq_per_pca(int qbbno, int pcano) irq_set_status_flags(i + irq_bias, IRQ_LEVEL); } - setup_irq(32+irq_bias, &isa_enable); + if (request_irq(32 + irq_bias, no_action, 0, "isa_enable", NULL)) + pr_err("Failed to register isa_enable interrupt\n"); } static void __init diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c index 0069360697ee..4d01c392ab14 100644 --- a/arch/alpha/kernel/time.c +++ b/arch/alpha/kernel/time.c @@ -242,7 +242,7 @@ common_init_rtc(void) outb(0x31, 0x42); outb(0x13, 0x42); - init_rtc_irq(); + init_rtc_irq(NULL); } @@ -396,9 +396,7 @@ time_init(void) if (alpha_using_qemu) { clocksource_register_hz(&qemu_cs, NSEC_PER_SEC); init_qemu_clockevent(); - - timer_irqaction.handler = qemu_timer_interrupt; - init_rtc_irq(); + init_rtc_irq(qemu_timer_interrupt); return; } diff --git a/arch/arc/include/asm/futex.h b/arch/arc/include/asm/futex.h index 9d0d070e6c22..607d1c16d4dd 100644 --- a/arch/arc/include/asm/futex.h +++ b/arch/arc/include/asm/futex.h @@ -75,10 +75,12 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, { int oldval = 0, ret; + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + #ifndef CONFIG_ARC_HAS_LLSC preempt_disable(); /* to guarantee atomic r-m-w of futex op */ #endif - pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -101,7 +103,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, ret = -ENOSYS; } - pagefault_enable(); #ifndef CONFIG_ARC_HAS_LLSC preempt_enable(); #endif diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 97864aabc2a6..03bbfc312fe7 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -3,7 +3,6 @@ config ARM bool default y select ARCH_32BIT_OFF_T - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEVMEM_IS_ALLOWED diff --git a/arch/arm/boot/compressed/efi-header.S b/arch/arm/boot/compressed/efi-header.S index a5983588f96b..62286da318e7 100644 --- a/arch/arm/boot/compressed/efi-header.S +++ b/arch/arm/boot/compressed/efi-header.S @@ -60,7 +60,7 @@ optional_header: .long __pecoff_code_size @ SizeOfCode .long __pecoff_data_size @ SizeOfInitializedData .long 0 @ SizeOfUninitializedData - .long efi_stub_entry - start @ AddressOfEntryPoint + .long efi_entry - start @ AddressOfEntryPoint .long start_offset @ BaseOfCode .long __pecoff_data_start - start @ BaseOfData @@ -70,8 +70,8 @@ extra_header_fields: .long SZ_512 @ FileAlignment .short 0 @ MajorOsVersion .short 0 @ MinorOsVersion - .short 0 @ MajorImageVersion - .short 0 @ MinorImageVersion + .short LINUX_EFISTUB_MAJOR_VERSION @ MajorImageVersion + .short LINUX_EFISTUB_MINOR_VERSION @ MinorImageVersion .short 0 @ MajorSubsystemVersion .short 0 @ MinorSubsystemVersion .long 0 @ Win32VersionValue diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index 088b0a060876..04f77214f050 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -1437,29 +1437,25 @@ __enter_kernel: reloc_code_end: #ifdef CONFIG_EFI_STUB - .align 2 -_start: .long start - . - -ENTRY(efi_stub_entry) - @ allocate space on stack for passing current zImage address - @ and for the EFI stub to return of new entry point of - @ zImage, as EFI stub may copy the kernel. Pointer address - @ is passed in r2. r0 and r1 are passed through from the - @ EFI firmware to efi_entry - adr ip, _start - ldr r3, [ip] - add r3, r3, ip - stmfd sp!, {r3, lr} - mov r2, sp @ pass zImage address in r2 - bl efi_entry - - @ Check for error return from EFI stub. r0 has FDT address - @ or error code. - cmn r0, #1 - beq efi_load_fail - - @ Preserve return value of efi_entry() in r4 - mov r4, r0 +ENTRY(efi_enter_kernel) + mov r7, r0 @ preserve image base + mov r4, r1 @ preserve DT pointer + + mov r0, r4 @ DT start + add r1, r4, r2 @ DT end + bl cache_clean_flush + + mov r0, r7 @ relocated zImage + ldr r1, =_edata @ size of zImage + add r1, r1, r0 @ end of zImage + bl cache_clean_flush + + @ The PE/COFF loader might not have cleaned the code we are + @ running beyond the PoU, and so calling cache_off below from + @ inside the PE/COFF loader allocated region is unsafe unless + @ we explicitly clean it to the PoC. + adr r0, call_cache_fn @ region of code we will + adr r1, 0f @ run with MMU off bl cache_clean_flush bl cache_off @@ -1469,18 +1465,10 @@ ENTRY(efi_stub_entry) mov r0, #0 mov r1, #0xFFFFFFFF mov r2, r4 - - @ Branch to (possibly) relocated zImage that is in [sp] - ldr lr, [sp] - ldr ip, =start_offset - add lr, lr, ip - mov pc, lr @ no mode switch - -efi_load_fail: - @ Return EFI_LOAD_ERROR to EFI firmware on error. - ldr r0, =0x80000001 - ldmfd sp!, {ip, pc} -ENDPROC(efi_stub_entry) + add r7, r7, #(__efi_start - start) + mov pc, r7 @ no mode switch +ENDPROC(efi_enter_kernel) +0: #endif .align diff --git a/arch/arm/boot/dts/aspeed-g4.dtsi b/arch/arm/boot/dts/aspeed-g4.dtsi index 807a0fc20670..8e04303e8514 100644 --- a/arch/arm/boot/dts/aspeed-g4.dtsi +++ b/arch/arm/boot/dts/aspeed-g4.dtsi @@ -164,6 +164,8 @@ reg = <0x1e6a0000 0x300>; interrupts = <5>; clocks = <&syscon ASPEED_CLK_GATE_USBPORT1CLK>; + aspeed,vhub-downstream-ports = <5>; + aspeed,vhub-generic-endpoints = <15>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb2d_default>; status = "disabled"; diff --git a/arch/arm/boot/dts/aspeed-g5.dtsi b/arch/arm/boot/dts/aspeed-g5.dtsi index ebec0fa8baa7..f12ec04d3cbc 100644 --- a/arch/arm/boot/dts/aspeed-g5.dtsi +++ b/arch/arm/boot/dts/aspeed-g5.dtsi @@ -195,6 +195,8 @@ reg = <0x1e6a0000 0x300>; interrupts = <5>; clocks = <&syscon ASPEED_CLK_GATE_USBPORT1CLK>; + aspeed,vhub-downstream-ports = <5>; + aspeed,vhub-generic-endpoints = <15>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb2ad_default>; status = "disabled"; diff --git a/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi b/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi index 045ce66ca876..7028e21bdd98 100644 --- a/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi +++ b/arch/arm/boot/dts/aspeed-g6-pinctrl.dtsi @@ -1112,6 +1112,31 @@ groups = "UART9"; }; + pinctrl_usb2ah_default: usb2ah_default { + function = "USB2AH"; + groups = "USBA"; + }; + + pinctrl_usb2ad_default: usb2ad_default { + function = "USB2AD"; + groups = "USBA"; + }; + + pinctrl_usb2bh_default: usb2bh_default { + function = "USB2BH"; + groups = "USBB"; + }; + + pinctrl_usb2bd_default: usb2bd_default { + function = "USB2BD"; + groups = "USBB"; + }; + + pinctrl_usb11bhid_default: usb11bhid_default { + function = "USB11BHID"; + groups = "USBB"; + }; + pinctrl_vb_default: vb_default { function = "VB"; groups = "VB"; diff --git a/arch/arm/boot/dts/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed-g6.dtsi index 796976d275e1..0a29b3b57a9d 100644 --- a/arch/arm/boot/dts/aspeed-g6.dtsi +++ b/arch/arm/boot/dts/aspeed-g6.dtsi @@ -245,6 +245,51 @@ status = "disabled"; }; + ehci0: usb@1e6a1000 { + compatible = "aspeed,ast2600-ehci", "generic-ehci"; + reg = <0x1e6a1000 0x100>; + interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon ASPEED_CLK_GATE_USBPORT1CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb2ah_default>; + status = "disabled"; + }; + + ehci1: usb@1e6a3000 { + compatible = "aspeed,ast2600-ehci", "generic-ehci"; + reg = <0x1e6a3000 0x100>; + interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon ASPEED_CLK_GATE_USBPORT2CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb2bh_default>; + status = "disabled"; + }; + + uhci: usb@1e6b0000 { + compatible = "aspeed,ast2600-uhci", "generic-uhci"; + reg = <0x1e6b0000 0x100>; + interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>; + #ports = <2>; + clocks = <&syscon ASPEED_CLK_GATE_USBUHCICLK>; + status = "disabled"; + /* + * No default pinmux, it will follow EHCI, use an + * explicit pinmux override if EHCI is not enabled. + */ + }; + + vhub: usb-vhub@1e6a0000 { + compatible = "aspeed,ast2600-usb-vhub"; + reg = <0x1e6a0000 0x350>; + interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon ASPEED_CLK_GATE_USBPORT1CLK>; + aspeed,vhub-downstream-ports = <7>; + aspeed,vhub-generic-endpoints = <21>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb2ad_default>; + status = "disabled"; + }; + apb { compatible = "simple-bus"; #address-cells = <1>; diff --git a/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts b/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts index b75af21069f9..4c3f606e5b8d 100644 --- a/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts +++ b/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts @@ -112,6 +112,7 @@ &sdhci { #address-cells = <1>; #size-cells = <0>; + pinctrl-names = "default"; pinctrl-0 = <&emmc_gpio34 &gpclk2_gpio43>; bus-width = <4>; mmc-pwrseq = <&wifi_pwrseq>; diff --git a/arch/arm/boot/dts/bcm2835-rpi.dtsi b/arch/arm/boot/dts/bcm2835-rpi.dtsi index 394c8a71b13b..fd2c766e0f71 100644 --- a/arch/arm/boot/dts/bcm2835-rpi.dtsi +++ b/arch/arm/boot/dts/bcm2835-rpi.dtsi @@ -15,6 +15,7 @@ firmware: firmware { compatible = "raspberrypi,bcm2835-firmware", "simple-bus"; mboxes = <&mailbox>; + dma-ranges; }; power: power { diff --git a/arch/arm/boot/dts/dm8148-evm.dts b/arch/arm/boot/dts/dm8148-evm.dts index 3931fb068ff0..91d1018ab75f 100644 --- a/arch/arm/boot/dts/dm8148-evm.dts +++ b/arch/arm/boot/dts/dm8148-evm.dts @@ -24,12 +24,12 @@ &cpsw_emac0 { phy-handle = <ðphy0>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; }; &cpsw_emac1 { phy-handle = <ðphy1>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; }; &davinci_mdio { diff --git a/arch/arm/boot/dts/dm8148-t410.dts b/arch/arm/boot/dts/dm8148-t410.dts index 9e43d5ec0bb2..79ccdd4470f4 100644 --- a/arch/arm/boot/dts/dm8148-t410.dts +++ b/arch/arm/boot/dts/dm8148-t410.dts @@ -33,12 +33,12 @@ &cpsw_emac0 { phy-handle = <ðphy0>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; }; &cpsw_emac1 { phy-handle = <ðphy1>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; }; &davinci_mdio { diff --git a/arch/arm/boot/dts/dra62x-j5eco-evm.dts b/arch/arm/boot/dts/dra62x-j5eco-evm.dts index 861ab90a3f3a..c16e183822be 100644 --- a/arch/arm/boot/dts/dra62x-j5eco-evm.dts +++ b/arch/arm/boot/dts/dra62x-j5eco-evm.dts @@ -24,12 +24,12 @@ &cpsw_emac0 { phy-handle = <ðphy0>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; }; &cpsw_emac1 { phy-handle = <ðphy1>; - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; }; &davinci_mdio { diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index 4305051bb769..5f5ee16f07a3 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -148,6 +148,7 @@ #address-cells = <1>; #size-cells = <1>; ranges = <0x0 0x0 0x0 0xc0000000>; + dma-ranges = <0x80000000 0x0 0x80000000 0x80000000>; ti,hwmods = "l3_main_1", "l3_main_2"; reg = <0x0 0x44000000 0x0 0x1000000>, <0x0 0x45000000 0x0 0x1000>; diff --git a/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi b/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi index 31719c079d67..44f97546dd0a 100644 --- a/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi +++ b/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi @@ -33,7 +33,7 @@ }; }; - lcd_vdd3_reg: voltage-regulator-6 { + lcd_vdd3_reg: voltage-regulator-7 { compatible = "regulator-fixed"; regulator-name = "LCD_VDD_2.2V"; regulator-min-microvolt = <2200000>; @@ -42,7 +42,7 @@ enable-active-high; }; - ps_als_reg: voltage-regulator-7 { + ps_als_reg: voltage-regulator-8 { compatible = "regulator-fixed"; regulator-name = "LED_A_3.0V"; regulator-min-microvolt = <3000000>; diff --git a/arch/arm/boot/dts/exynos4412-n710x.dts b/arch/arm/boot/dts/exynos4412-n710x.dts index 98cd1284cd90..4189e1fb204c 100644 --- a/arch/arm/boot/dts/exynos4412-n710x.dts +++ b/arch/arm/boot/dts/exynos4412-n710x.dts @@ -13,7 +13,7 @@ /* bootargs are passed in by bootloader */ - cam_vdda_reg: voltage-regulator-6 { + cam_vdda_reg: voltage-regulator-7 { compatible = "regulator-fixed"; regulator-name = "CAM_SENSOR_CORE_1.2V"; regulator-min-microvolt = <1200000>; diff --git a/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi index 4d18952658f8..77d871340eb7 100644 --- a/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi +++ b/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi @@ -112,7 +112,7 @@ regulators { vdd_arm: buck1 { regulator-name = "vdd_arm"; - regulator-min-microvolt = <730000>; + regulator-min-microvolt = <925000>; regulator-max-microvolt = <1380000>; regulator-initial-mode = <DA9063_BUCK_MODE_SYNC>; regulator-always-on; @@ -120,7 +120,7 @@ vdd_soc: buck2 { regulator-name = "vdd_soc"; - regulator-min-microvolt = <730000>; + regulator-min-microvolt = <1150000>; regulator-max-microvolt = <1380000>; regulator-initial-mode = <DA9063_BUCK_MODE_SYNC>; regulator-always-on; diff --git a/arch/arm/boot/dts/motorola-mapphone-common.dtsi b/arch/arm/boot/dts/motorola-mapphone-common.dtsi index b6e82b165f5c..9067e0ef4240 100644 --- a/arch/arm/boot/dts/motorola-mapphone-common.dtsi +++ b/arch/arm/boot/dts/motorola-mapphone-common.dtsi @@ -429,7 +429,7 @@ reset-gpios = <&gpio6 13 GPIO_ACTIVE_HIGH>; /* gpio173 */ /* gpio_183 with sys_nirq2 pad as wakeup */ - interrupts-extended = <&gpio6 23 IRQ_TYPE_EDGE_FALLING>, + interrupts-extended = <&gpio6 23 IRQ_TYPE_LEVEL_LOW>, <&omap4_pmx_core 0x160>; interrupt-names = "irq", "wakeup"; wakeup-source; diff --git a/arch/arm/boot/dts/omap3-n900.dts b/arch/arm/boot/dts/omap3-n900.dts index c3c6d7d04a76..4089d97405c9 100644 --- a/arch/arm/boot/dts/omap3-n900.dts +++ b/arch/arm/boot/dts/omap3-n900.dts @@ -854,34 +854,46 @@ compatible = "ti,omap2-onenand"; reg = <0 0 0x20000>; /* CS0, offset 0, IO size 128K */ + /* + * These timings are based on CONFIG_OMAP_GPMC_DEBUG=y reported + * bootloader set values when booted with v5.1 + * (OneNAND Manufacturer: Samsung): + * + * cs0 GPMC_CS_CONFIG1: 0xfb001202 + * cs0 GPMC_CS_CONFIG2: 0x00111100 + * cs0 GPMC_CS_CONFIG3: 0x00020200 + * cs0 GPMC_CS_CONFIG4: 0x11001102 + * cs0 GPMC_CS_CONFIG5: 0x03101616 + * cs0 GPMC_CS_CONFIG6: 0x90060000 + */ gpmc,sync-read; gpmc,sync-write; gpmc,burst-length = <16>; gpmc,burst-read; gpmc,burst-wrap; gpmc,burst-write; - gpmc,device-width = <2>; /* GPMC_DEVWIDTH_16BIT */ - gpmc,mux-add-data = <2>; /* GPMC_MUX_AD */ + gpmc,device-width = <2>; + gpmc,mux-add-data = <2>; gpmc,cs-on-ns = <0>; - gpmc,cs-rd-off-ns = <87>; - gpmc,cs-wr-off-ns = <87>; + gpmc,cs-rd-off-ns = <102>; + gpmc,cs-wr-off-ns = <102>; gpmc,adv-on-ns = <0>; - gpmc,adv-rd-off-ns = <10>; - gpmc,adv-wr-off-ns = <10>; - gpmc,oe-on-ns = <15>; - gpmc,oe-off-ns = <87>; + gpmc,adv-rd-off-ns = <12>; + gpmc,adv-wr-off-ns = <12>; + gpmc,oe-on-ns = <12>; + gpmc,oe-off-ns = <102>; gpmc,we-on-ns = <0>; - gpmc,we-off-ns = <87>; - gpmc,rd-cycle-ns = <112>; - gpmc,wr-cycle-ns = <112>; - gpmc,access-ns = <81>; - gpmc,page-burst-access-ns = <15>; + gpmc,we-off-ns = <102>; + gpmc,rd-cycle-ns = <132>; + gpmc,wr-cycle-ns = <132>; + gpmc,access-ns = <96>; + gpmc,page-burst-access-ns = <18>; gpmc,bus-turnaround-ns = <0>; gpmc,cycle2cycle-delay-ns = <0>; gpmc,wait-monitoring-ns = <0>; - gpmc,clk-activation-ns = <5>; - gpmc,wr-data-mux-bus-ns = <30>; - gpmc,wr-access-ns = <81>; + gpmc,clk-activation-ns = <6>; + gpmc,wr-data-mux-bus-ns = <36>; + gpmc,wr-access-ns = <96>; gpmc,sync-clk-ps = <15000>; /* diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi index d0ecf54d5a23..a7562d3deb1a 100644 --- a/arch/arm/boot/dts/omap5.dtsi +++ b/arch/arm/boot/dts/omap5.dtsi @@ -143,6 +143,7 @@ #address-cells = <1>; #size-cells = <1>; ranges = <0 0 0 0xc0000000>; + dma-ranges = <0x80000000 0x0 0x80000000 0x80000000>; ti,hwmods = "l3_main_1", "l3_main_2", "l3_main_3"; reg = <0 0x44000000 0 0x2000>, <0 0x44800000 0 0x3000>, diff --git a/arch/arm/boot/dts/ox810se.dtsi b/arch/arm/boot/dts/ox810se.dtsi index 9f6c2b660ed3..0755e5864c4a 100644 --- a/arch/arm/boot/dts/ox810se.dtsi +++ b/arch/arm/boot/dts/ox810se.dtsi @@ -323,8 +323,8 @@ interrupt-controller; reg = <0 0x200>; #interrupt-cells = <1>; - valid-mask = <0xFFFFFFFF>; - clear-mask = <0>; + valid-mask = <0xffffffff>; + clear-mask = <0xffffffff>; }; timer0: timer@200 { diff --git a/arch/arm/boot/dts/ox820.dtsi b/arch/arm/boot/dts/ox820.dtsi index c9b327732063..90846a7655b4 100644 --- a/arch/arm/boot/dts/ox820.dtsi +++ b/arch/arm/boot/dts/ox820.dtsi @@ -240,8 +240,8 @@ reg = <0 0x200>; interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; - valid-mask = <0xFFFFFFFF>; - clear-mask = <0>; + valid-mask = <0xffffffff>; + clear-mask = <0xffffffff>; }; timer0: timer@200 { diff --git a/arch/arm/boot/dts/sun8i-a33.dtsi b/arch/arm/boot/dts/sun8i-a33.dtsi index 1532a0e59af4..a2c37adacf77 100644 --- a/arch/arm/boot/dts/sun8i-a33.dtsi +++ b/arch/arm/boot/dts/sun8i-a33.dtsi @@ -215,7 +215,7 @@ }; crypto: crypto-engine@1c15000 { - compatible = "allwinner,sun4i-a10-crypto"; + compatible = "allwinner,sun8i-a33-crypto"; reg = <0x01c15000 0x1000>; interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>; clocks = <&ccu CLK_BUS_SS>, <&ccu CLK_SS>; diff --git a/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts b/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts index 2fd31a0a0b34..e8b3669e0e5d 100644 --- a/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts +++ b/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts @@ -374,8 +374,8 @@ }; ®_dldo3 { - regulator-min-microvolt = <2800000>; - regulator-max-microvolt = <2800000>; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; regulator-name = "vdd-csi"; }; @@ -498,7 +498,8 @@ }; &usbphy { - usb0_id_det-gpios = <&pio 7 11 GPIO_ACTIVE_HIGH>; /* PH11 */ + usb0_id_det-gpios = <&pio 7 11 (GPIO_ACTIVE_HIGH | GPIO_PULL_UP)>; /* PH11 */ + usb0_vbus_power-supply = <&usb_power_supply>; usb0_vbus-supply = <®_drivevbus>; usb1_vbus-supply = <®_vmain>; usb2_vbus-supply = <®_vmain>; diff --git a/arch/arm/boot/dts/sun8i-a83t.dtsi b/arch/arm/boot/dts/sun8i-a83t.dtsi index 74ac7ee9383c..e7b9bef1be6b 100644 --- a/arch/arm/boot/dts/sun8i-a83t.dtsi +++ b/arch/arm/boot/dts/sun8i-a83t.dtsi @@ -1006,10 +1006,10 @@ reg = <0x01c30000 0x104>; interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq"; - resets = <&ccu CLK_BUS_EMAC>; - reset-names = "stmmaceth"; - clocks = <&ccu RST_BUS_EMAC>; + clocks = <&ccu CLK_BUS_EMAC>; clock-names = "stmmaceth"; + resets = <&ccu RST_BUS_EMAC>; + reset-names = "stmmaceth"; status = "disabled"; mdio: mdio { diff --git a/arch/arm/boot/dts/sun8i-r40.dtsi b/arch/arm/boot/dts/sun8i-r40.dtsi index 8f09a24b36ec..a9d5d6ddbd71 100644 --- a/arch/arm/boot/dts/sun8i-r40.dtsi +++ b/arch/arm/boot/dts/sun8i-r40.dtsi @@ -181,6 +181,32 @@ interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>; }; + spi0: spi@1c05000 { + compatible = "allwinner,sun8i-r40-spi", + "allwinner,sun8i-h3-spi"; + reg = <0x01c05000 0x1000>; + interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_SPI0>, <&ccu CLK_SPI0>; + clock-names = "ahb", "mod"; + resets = <&ccu RST_BUS_SPI0>; + status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; + }; + + spi1: spi@1c06000 { + compatible = "allwinner,sun8i-r40-spi", + "allwinner,sun8i-h3-spi"; + reg = <0x01c06000 0x1000>; + interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_SPI1>, <&ccu CLK_SPI1>; + clock-names = "ahb", "mod"; + resets = <&ccu RST_BUS_SPI1>; + status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; + }; + csi0: csi@1c09000 { compatible = "allwinner,sun8i-r40-csi0", "allwinner,sun7i-a20-csi0"; @@ -290,6 +316,29 @@ resets = <&ccu RST_BUS_CE>; }; + spi2: spi@1c17000 { + compatible = "allwinner,sun8i-r40-spi", + "allwinner,sun8i-h3-spi"; + reg = <0x01c17000 0x1000>; + interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_SPI2>, <&ccu CLK_SPI2>; + clock-names = "ahb", "mod"; + resets = <&ccu RST_BUS_SPI2>; + status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; + }; + + ahci: sata@1c18000 { + compatible = "allwinner,sun8i-r40-ahci"; + reg = <0x01c18000 0x1000>; + interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_SATA>, <&ccu CLK_SATA>; + resets = <&ccu RST_BUS_SATA>; + reset-names = "ahci"; + status = "disabled"; + }; + ehci1: usb@1c19000 { compatible = "allwinner,sun8i-r40-ehci", "generic-ehci"; reg = <0x01c19000 0x100>; @@ -336,6 +385,19 @@ status = "disabled"; }; + spi3: spi@1c1f000 { + compatible = "allwinner,sun8i-r40-spi", + "allwinner,sun8i-h3-spi"; + reg = <0x01c1f000 0x1000>; + interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_SPI3>, <&ccu CLK_SPI3>; + clock-names = "ahb", "mod"; + resets = <&ccu RST_BUS_SPI3>; + status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; + }; + ccu: clock@1c20000 { compatible = "allwinner,sun8i-r40-ccu"; reg = <0x01c20000 0x400>; @@ -653,69 +715,6 @@ #size-cells = <0>; }; - spi0: spi@1c05000 { - compatible = "allwinner,sun8i-r40-spi", - "allwinner,sun8i-h3-spi"; - reg = <0x01c05000 0x1000>; - interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&ccu CLK_BUS_SPI0>, <&ccu CLK_SPI0>; - clock-names = "ahb", "mod"; - resets = <&ccu RST_BUS_SPI0>; - status = "disabled"; - #address-cells = <1>; - #size-cells = <0>; - }; - - spi1: spi@1c06000 { - compatible = "allwinner,sun8i-r40-spi", - "allwinner,sun8i-h3-spi"; - reg = <0x01c06000 0x1000>; - interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&ccu CLK_BUS_SPI1>, <&ccu CLK_SPI1>; - clock-names = "ahb", "mod"; - resets = <&ccu RST_BUS_SPI1>; - status = "disabled"; - #address-cells = <1>; - #size-cells = <0>; - }; - - spi2: spi@1c07000 { - compatible = "allwinner,sun8i-r40-spi", - "allwinner,sun8i-h3-spi"; - reg = <0x01c07000 0x1000>; - interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&ccu CLK_BUS_SPI2>, <&ccu CLK_SPI2>; - clock-names = "ahb", "mod"; - resets = <&ccu RST_BUS_SPI2>; - status = "disabled"; - #address-cells = <1>; - #size-cells = <0>; - }; - - spi3: spi@1c0f000 { - compatible = "allwinner,sun8i-r40-spi", - "allwinner,sun8i-h3-spi"; - reg = <0x01c0f000 0x1000>; - interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&ccu CLK_BUS_SPI3>, <&ccu CLK_SPI3>; - clock-names = "ahb", "mod"; - resets = <&ccu RST_BUS_SPI3>; - status = "disabled"; - #address-cells = <1>; - #size-cells = <0>; - }; - - ahci: sata@1c18000 { - compatible = "allwinner,sun8i-r40-ahci"; - reg = <0x01c18000 0x1000>; - interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&ccu CLK_BUS_SATA>, <&ccu CLK_SATA>; - resets = <&ccu RST_BUS_SATA>; - reset-names = "ahci"; - status = "disabled"; - - }; - gmac: ethernet@1c50000 { compatible = "allwinner,sun8i-r40-gmac"; syscon = <&ccu>; diff --git a/arch/arm/common/sa1111.c b/arch/arm/common/sa1111.c index 947ef7981d92..c98ebae1aeac 100644 --- a/arch/arm/common/sa1111.c +++ b/arch/arm/common/sa1111.c @@ -302,10 +302,13 @@ static int sa1111_retrigger_irq(struct irq_data *d) break; } - if (i == 8) + if (i == 8) { pr_err("Danger Will Robinson: failed to re-trigger IRQ%d\n", d->irq); - return i == 8 ? -1 : 0; + return 0; + } + + return 1; } static int sa1111_type_irq(struct irq_data *d, unsigned int flags) diff --git a/arch/arm/include/asm/clocksource.h b/arch/arm/include/asm/clocksource.h index 0b350a7e26f3..13651c731a81 100644 --- a/arch/arm/include/asm/clocksource.h +++ b/arch/arm/include/asm/clocksource.h @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_CLOCKSOURCE_H #define _ASM_CLOCKSOURCE_H -struct arch_clocksource_data { - bool vdso_direct; /* Usable for direct VDSO access? */ -}; +#include <asm/vdso/clocksource.h> -#endif +#endif /* _ASM_CLOCKSOURCE_H */ diff --git a/arch/arm/include/asm/cp15.h b/arch/arm/include/asm/cp15.h index d2453e2d3f1f..a54230e65647 100644 --- a/arch/arm/include/asm/cp15.h +++ b/arch/arm/include/asm/cp15.h @@ -50,25 +50,7 @@ #ifdef CONFIG_CPU_CP15 -#define __ACCESS_CP15(CRn, Op1, CRm, Op2) \ - "mrc", "mcr", __stringify(p15, Op1, %0, CRn, CRm, Op2), u32 -#define __ACCESS_CP15_64(Op1, CRm) \ - "mrrc", "mcrr", __stringify(p15, Op1, %Q0, %R0, CRm), u64 - -#define __read_sysreg(r, w, c, t) ({ \ - t __val; \ - asm volatile(r " " c : "=r" (__val)); \ - __val; \ -}) -#define read_sysreg(...) __read_sysreg(__VA_ARGS__) - -#define __write_sysreg(v, r, w, c, t) asm volatile(w " " c : : "r" ((t)(v))) -#define write_sysreg(v, ...) __write_sysreg(v, __VA_ARGS__) - -#define BPIALL __ACCESS_CP15(c7, 0, c5, 6) -#define ICIALLU __ACCESS_CP15(c7, 0, c5, 0) - -#define CNTVCT __ACCESS_CP15_64(1, c14) +#include <asm/vdso/cp15.h> extern unsigned long cr_alignment; /* defined in entry-armv.S */ diff --git a/arch/arm/include/asm/floppy.h b/arch/arm/include/asm/floppy.h index f4fe4d02cef2..79fa327238e8 100644 --- a/arch/arm/include/asm/floppy.h +++ b/arch/arm/include/asm/floppy.h @@ -8,16 +8,18 @@ */ #ifndef __ASM_ARM_FLOPPY_H #define __ASM_ARM_FLOPPY_H -#if 0 -#include <mach/floppy.h> -#endif -#define fd_outb(val,port) \ - do { \ - if ((port) == (u32)FD_DOR) \ - fd_setdor((val)); \ - else \ - outb((val),(port)); \ +#define fd_outb(val,port) \ + do { \ + int new_val = (val); \ + if (((port) & 7) == FD_DOR) { \ + if (new_val & 0xf0) \ + new_val = (new_val & 0x0c) | \ + floppy_selects[new_val & 3]; \ + else \ + new_val &= 0x0c; \ + } \ + outb(new_val, (port)); \ } while(0) #define fd_inb(port) inb((port)) @@ -53,69 +55,7 @@ static inline int fd_dma_setup(void *data, unsigned int length, * to a non-zero track, and then restoring it to track 0. If an error occurs, * then there is no floppy drive present. [to be put back in again] */ -static unsigned char floppy_selects[2][4] = -{ - { 0x10, 0x21, 0x23, 0x33 }, - { 0x10, 0x21, 0x23, 0x33 } -}; - -#define fd_setdor(dor) \ -do { \ - int new_dor = (dor); \ - if (new_dor & 0xf0) \ - new_dor = (new_dor & 0x0c) | floppy_selects[fdc][new_dor & 3]; \ - else \ - new_dor &= 0x0c; \ - outb(new_dor, FD_DOR); \ -} while (0) - -/* - * Someday, we'll automatically detect which drives are present... - */ -static inline void fd_scandrives (void) -{ -#if 0 - int floppy, drive_count; - - fd_disable_irq(); - raw_cmd = &default_raw_cmd; - raw_cmd->flags = FD_RAW_SPIN | FD_RAW_NEED_SEEK; - raw_cmd->track = 0; - raw_cmd->rate = ?; - drive_count = 0; - for (floppy = 0; floppy < 4; floppy ++) { - current_drive = drive_count; - /* - * Turn on floppy motor - */ - if (start_motor(redo_fd_request)) - continue; - /* - * Set up FDC - */ - fdc_specify(); - /* - * Tell FDC to recalibrate - */ - output_byte(FD_RECALIBRATE); - LAST_OUT(UNIT(floppy)); - /* wait for command to complete */ - if (!successful) { - int i; - for (i = drive_count; i < 3; i--) - floppy_selects[fdc][i] = floppy_selects[fdc][i + 1]; - floppy_selects[fdc][3] = 0; - floppy -= 1; - } else - drive_count++; - } -#else - floppy_selects[0][0] = 0x10; - floppy_selects[0][1] = 0x21; - floppy_selects[0][2] = 0x23; - floppy_selects[0][3] = 0x33; -#endif -} +static unsigned char floppy_selects[4] = { 0x10, 0x21, 0x23, 0x33 }; #define FDC1 (0x3f0) @@ -135,9 +75,7 @@ static inline void fd_scandrives (void) */ static void driveswap(int *ints, int dummy, int dummy2) { - floppy_selects[0][0] ^= floppy_selects[0][1]; - floppy_selects[0][1] ^= floppy_selects[0][0]; - floppy_selects[0][0] ^= floppy_selects[0][1]; + swap(floppy_selects[0], floppy_selects[1]); } #define EXTRA_FLOPPY_PARAMS ,{ "driveswap", &driveswap, NULL, 0, 0 } diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 83c391b597d4..e133da303a98 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -134,10 +134,12 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret, tmp; + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + #ifndef CONFIG_SMP preempt_disable(); #endif - pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -159,7 +161,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); #ifndef CONFIG_SMP preempt_enable(); #endif diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index 614bf829e454..b9241051e5cb 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -14,6 +14,7 @@ #include <asm/ptrace.h> #include <asm/types.h> #include <asm/unified.h> +#include <asm/vdso/processor.h> #ifdef __KERNEL__ #define STACK_TOP ((current->personality & ADDR_LIMIT_32BIT) ? \ @@ -85,16 +86,6 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); -#if __LINUX_ARM_ARCH__ == 6 || defined(CONFIG_ARM_ERRATA_754327) -#define cpu_relax() \ - do { \ - smp_mb(); \ - __asm__ __volatile__("nop; nop; nop; nop; nop; nop; nop; nop; nop; nop;"); \ - } while (0) -#else -#define cpu_relax() barrier() -#endif - #define task_pt_regs(p) \ ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 0d0d5178e2c3..3609a6980c34 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -141,7 +141,6 @@ extern int vfp_restore_user_hwstate(struct user_vfp *, #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ #define TIF_SECCOMP 7 /* seccomp syscall filtering active */ -#define TIF_NOHZ 12 /* in adaptive nohz mode */ #define TIF_USING_IWMMXT 17 #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ #define TIF_RESTORE_SIGMASK 20 diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index 8a0fae94d45e..435aba289fc5 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -16,6 +16,9 @@ /* Enable topology flag updates */ #define arch_update_cpu_topology topology_update_cpu_topology +/* Replace task scheduler's default thermal pressure retrieve API */ +#define arch_scale_thermal_pressure topology_get_thermal_pressure + #else static inline void init_cpu_topology(void) { } diff --git a/arch/arm/include/asm/vdso/clocksource.h b/arch/arm/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..50c0b19fb755 --- /dev/null +++ b/arch/arm/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif /* __ASM_VDSOCLOCKSOURCE_H */ diff --git a/arch/arm/include/asm/vdso/cp15.h b/arch/arm/include/asm/vdso/cp15.h new file mode 100644 index 000000000000..bed16fa1865e --- /dev/null +++ b/arch/arm/include/asm/vdso/cp15.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_CP15_H +#define __ASM_VDSO_CP15_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_CPU_CP15 + +#include <linux/stringify.h> + +#define __ACCESS_CP15(CRn, Op1, CRm, Op2) \ + "mrc", "mcr", __stringify(p15, Op1, %0, CRn, CRm, Op2), u32 +#define __ACCESS_CP15_64(Op1, CRm) \ + "mrrc", "mcrr", __stringify(p15, Op1, %Q0, %R0, CRm), u64 + +#define __read_sysreg(r, w, c, t) ({ \ + t __val; \ + asm volatile(r " " c : "=r" (__val)); \ + __val; \ +}) +#define read_sysreg(...) __read_sysreg(__VA_ARGS__) + +#define __write_sysreg(v, r, w, c, t) asm volatile(w " " c : : "r" ((t)(v))) +#define write_sysreg(v, ...) __write_sysreg(v, __VA_ARGS__) + +#define BPIALL __ACCESS_CP15(c7, 0, c5, 6) +#define ICIALLU __ACCESS_CP15(c7, 0, c5, 0) + +#define CNTVCT __ACCESS_CP15_64(1, c14) + +#endif /* CONFIG_CPU_CP15 */ + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CP15_H */ diff --git a/arch/arm/include/asm/vdso/gettimeofday.h b/arch/arm/include/asm/vdso/gettimeofday.h index fe6e1f65932d..36dc18553ed8 100644 --- a/arch/arm/include/asm/vdso/gettimeofday.h +++ b/arch/arm/include/asm/vdso/gettimeofday.h @@ -7,9 +7,9 @@ #ifndef __ASSEMBLY__ -#include <asm/barrier.h> -#include <asm/cp15.h> +#include <asm/errno.h> #include <asm/unistd.h> +#include <asm/vdso/cp15.h> #include <uapi/linux/time.h> #define VDSO_HAS_CLOCK_GETRES 1 @@ -106,20 +106,32 @@ static __always_inline int clock_getres32_fallback( return ret; } +static inline bool arm_vdso_hres_capable(void) +{ + return IS_ENABLED(CONFIG_ARM_ARCH_TIMER); +} +#define __arch_vdso_hres_capable arm_vdso_hres_capable + static __always_inline u64 __arch_get_hw_counter(int clock_mode) { #ifdef CONFIG_ARM_ARCH_TIMER u64 cycle_now; - if (!clock_mode) - return -EINVAL; + /* + * Core checks for mode already, so this raced against a concurrent + * update. Return something. Core will do another round and then + * see the mode change and fallback to the syscall. + */ + if (clock_mode == VDSO_CLOCKMODE_NONE) + return 0; isb(); cycle_now = read_sysreg(CNTVCT); return cycle_now; #else - return -EINVAL; /* use fallback */ + /* Make GCC happy. This is compiled out anyway */ + return 0; #endif } diff --git a/arch/arm/include/asm/vdso/processor.h b/arch/arm/include/asm/vdso/processor.h new file mode 100644 index 000000000000..45efb3ff511c --- /dev/null +++ b/arch/arm/include/asm/vdso/processor.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#if __LINUX_ARM_ARCH__ == 6 || defined(CONFIG_ARM_ERRATA_754327) +#define cpu_relax() \ + do { \ + smp_mb(); \ + __asm__ __volatile__("nop; nop; nop; nop; nop; nop; nop; nop; nop; nop;"); \ + } while (0) +#else +#define cpu_relax() barrier() +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/arm/include/asm/vdso/vsyscall.h b/arch/arm/include/asm/vdso/vsyscall.h index cff87d8d30da..47e41ae8ccd0 100644 --- a/arch/arm/include/asm/vdso/vsyscall.h +++ b/arch/arm/include/asm/vdso/vsyscall.h @@ -11,18 +11,6 @@ extern struct vdso_data *vdso_data; extern bool cntvct_ok; -static __always_inline -bool tk_is_cntvct(const struct timekeeper *tk) -{ - if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER)) - return false; - - if (!tk->tkr_mono.clock->archdata.vdso_direct) - return false; - - return true; -} - /* * Update the vDSO data page to keep in sync with kernel timekeeping. */ @@ -34,29 +22,6 @@ struct vdso_data *__arm_get_k_vdso_data(void) #define __arch_get_k_vdso_data __arm_get_k_vdso_data static __always_inline -bool __arm_update_vdso_data(void) -{ - return cntvct_ok; -} -#define __arch_update_vdso_data __arm_update_vdso_data - -static __always_inline -int __arm_get_clock_mode(struct timekeeper *tk) -{ - u32 __tk_is_cntvct = tk_is_cntvct(tk); - - return __tk_is_cntvct; -} -#define __arch_get_clock_mode __arm_get_clock_mode - -static __always_inline -int __arm_use_vsyscall(struct vdso_data *vdata) -{ - return vdata[CS_HRES_COARSE].clock_mode; -} -#define __arch_use_vsyscall __arm_use_vsyscall - -static __always_inline void __arm_sync_vdso_data(struct vdso_data *vdata) { flush_dcache_page(virt_to_page(vdata)); diff --git a/arch/arm/kernel/reboot.c b/arch/arm/kernel/reboot.c index bb18ed0539f4..0ce388f15422 100644 --- a/arch/arm/kernel/reboot.c +++ b/arch/arm/kernel/reboot.c @@ -88,11 +88,11 @@ void soft_restart(unsigned long addr) * to execute e.g. a RAM-based pin loop is not sufficient. This allows the * kexec'd kernel to use any and all RAM as it sees fit, without having to * avoid any code or data used by any SW CPU pin loop. The CPU hotplug - * functionality embodied in disable_nonboot_cpus() to achieve this. + * functionality embodied in smp_shutdown_nonboot_cpus() to achieve this. */ void machine_shutdown(void) { - disable_nonboot_cpus(); + smp_shutdown_nonboot_cpus(reboot_cpu); } /* diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 0b30e884e088..6e41c4b62607 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -9,7 +9,6 @@ config ARM64 select ACPI_MCFG if (ACPI && PCI) select ACPI_SPCR_TABLE if ACPI select ACPI_PPTT if ACPI - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEVMEM_IS_ALLOWED select ARCH_HAS_DMA_PREP_COHERENT @@ -118,6 +117,7 @@ config ARM64 select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_BITREVERSE + select HAVE_ARCH_COMPILER_H select HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE @@ -281,6 +281,9 @@ config ZONE_DMA32 config ARCH_ENABLE_MEMORY_HOTPLUG def_bool y +config ARCH_ENABLE_MEMORY_HOTREMOVE + def_bool y + config SMP def_bool y @@ -952,11 +955,11 @@ config HOTPLUG_CPU # Common NUMA Features config NUMA - bool "Numa Memory Allocation and Scheduler Support" + bool "NUMA Memory Allocation and Scheduler Support" select ACPI_NUMA if ACPI select OF_NUMA help - Enable NUMA (Non Uniform Memory Access) support. + Enable NUMA (Non-Uniform Memory Access) support. The kernel will try to allocate memory used by a CPU on the local memory of the CPU and add some more @@ -1498,6 +1501,9 @@ config ARM64_PTR_AUTH bool "Enable support for pointer authentication" default y depends on !KVM || ARM64_VHE + depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC + depends on CC_IS_GCC || (CC_IS_CLANG && AS_HAS_CFI_NEGATE_RA_STATE) + depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) help Pointer authentication (part of the ARMv8.3 Extensions) provides instructions for signing and authenticating pointers against secret @@ -1505,16 +1511,72 @@ config ARM64_PTR_AUTH and other attacks. This option enables these instructions at EL0 (i.e. for userspace). - Choosing this option will cause the kernel to initialise secret keys for each process at exec() time, with these keys being context-switched along with the process. + If the compiler supports the -mbranch-protection or + -msign-return-address flag (e.g. GCC 7 or later), then this option + will also cause the kernel itself to be compiled with return address + protection. In this case, and if the target hardware is known to + support pointer authentication, then CONFIG_STACKPROTECTOR can be + disabled with minimal loss of protection. + The feature is detected at runtime. If the feature is not present in hardware it will not be advertised to userspace/KVM guest nor will it be enabled. However, KVM guest also require VHE mode and hence CONFIG_ARM64_VHE=y option to use this feature. + If the feature is present on the boot CPU but not on a late CPU, then + the late CPU will be parked. Also, if the boot CPU does not have + address auth and the late CPU has then the late CPU will still boot + but with the feature disabled. On such a system, this option should + not be selected. + + This feature works with FUNCTION_GRAPH_TRACER option only if + DYNAMIC_FTRACE_WITH_REGS is enabled. + +config CC_HAS_BRANCH_PROT_PAC_RET + # GCC 9 or later, clang 8 or later + def_bool $(cc-option,-mbranch-protection=pac-ret+leaf) + +config CC_HAS_SIGN_RETURN_ADDRESS + # GCC 7, 8 + def_bool $(cc-option,-msign-return-address=all) + +config AS_HAS_PAC + def_bool $(as-option,-Wa$(comma)-march=armv8.3-a) + +config AS_HAS_CFI_NEGATE_RA_STATE + def_bool $(as-instr,.cfi_startproc\n.cfi_negate_ra_state\n.cfi_endproc\n) + +endmenu + +menu "ARMv8.4 architectural features" + +config ARM64_AMU_EXTN + bool "Enable support for the Activity Monitors Unit CPU extension" + default y + help + The activity monitors extension is an optional extension introduced + by the ARMv8.4 CPU architecture. This enables support for version 1 + of the activity monitors architecture, AMUv1. + + To enable the use of this extension on CPUs that implement it, say Y. + + Note that for architectural reasons, firmware _must_ implement AMU + support when running on CPUs that present the activity monitors + extension. The required support is present in: + * Version 1.5 and later of the ARM Trusted Firmware + + For kernels that have this configuration enabled but boot with broken + firmware, you may need to say N here until the firmware is fixed. + Otherwise you may experience firmware panics or lockups when + accessing the counter registers. Even if you are not observing these + symptoms, the values returned by the register reads might not + correctly reflect reality. Most commonly, the value read will be 0, + indicating that the counter is not enabled. + endmenu menu "ARMv8.5 architectural features" diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index dca1a97751ab..f15f92ba53e6 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -65,6 +65,17 @@ stack_protector_prepare: prepare0 include/generated/asm-offsets.h)) endif +ifeq ($(CONFIG_ARM64_PTR_AUTH),y) +branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address=all +branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf +# -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the +# compiler to generate them and consequently to break the single image contract +# we pass it only to the assembler. This option is utilized only in case of non +# integrated assemblers. +branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a +KBUILD_CFLAGS += $(branch-prot-flags-y) +endif + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__ diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-kii-pro.dts b/arch/arm64/boot/dts/amlogic/meson-gxbb-kii-pro.dts index 2f1f829450a2..6c9cc45fb417 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxbb-kii-pro.dts +++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-kii-pro.dts @@ -76,3 +76,7 @@ }; }; }; + +&ir { + linux,rc-map-name = "rc-videostrong-kii-pro"; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi index 0bf375ec959b..55b71bb4baf8 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi @@ -53,7 +53,7 @@ * PSCI node is not added default, U-boot will add missing * parts if it determines to use PSCI. */ - entry-method = "arm,psci"; + entry-method = "psci"; CPU_PW20: cpu-pw20 { compatible = "arm,idle-state"; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts index 4223a2352d45..dde50c88f5e3 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts @@ -119,12 +119,12 @@ ethernet@e4000 { phy-handle = <&rgmii_phy1>; - phy-connection-type = "rgmii-txid"; + phy-connection-type = "rgmii-id"; }; ethernet@e6000 { phy-handle = <&rgmii_phy2>; - phy-connection-type = "rgmii-txid"; + phy-connection-type = "rgmii-id"; }; ethernet@e8000 { diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts index dbc23d6cd3b4..d53ccc56bb63 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts @@ -131,12 +131,12 @@ &fman0 { ethernet@e4000 { phy-handle = <&rgmii_phy1>; - phy-connection-type = "rgmii"; + phy-connection-type = "rgmii-id"; }; ethernet@e6000 { phy-handle = <&rgmii_phy2>; - phy-connection-type = "rgmii"; + phy-connection-type = "rgmii-id"; }; ethernet@e8000 { diff --git a/arch/arm64/boot/dts/sprd/sc9863a.dtsi b/arch/arm64/boot/dts/sprd/sc9863a.dtsi index cd80756c888d..2c590ca1d079 100644 --- a/arch/arm64/boot/dts/sprd/sc9863a.dtsi +++ b/arch/arm64/boot/dts/sprd/sc9863a.dtsi @@ -108,7 +108,7 @@ }; idle-states { - entry-method = "arm,psci"; + entry-method = "psci"; CORE_PD: core-pd { compatible = "arm,idle-state"; entry-latency-us = <4000>; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 4db223dbc549..e7573289a66f 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -62,6 +62,7 @@ CONFIG_ARCH_ZX=y CONFIG_ARCH_ZYNQMP=y CONFIG_ARM64_VA_BITS_48=y CONFIG_SCHED_MC=y +CONFIG_SCHED_SMT=y CONFIG_NUMA=y CONFIG_SECCOMP=y CONFIG_KEXEC=y diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S index 45062553467f..1dc5bbbfeed2 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/arch/arm64/crypto/aes-ce.S @@ -9,8 +9,8 @@ #include <linux/linkage.h> #include <asm/assembler.h> -#define AES_ENTRY(func) SYM_FUNC_START(ce_ ## func) -#define AES_ENDPROC(func) SYM_FUNC_END(ce_ ## func) +#define AES_FUNC_START(func) SYM_FUNC_START(ce_ ## func) +#define AES_FUNC_END(func) SYM_FUNC_END(ce_ ## func) .arch armv8-a+crypto diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 8a2faa42b57e..cf618d8f6cec 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -51,7 +51,7 @@ SYM_FUNC_END(aes_decrypt_block5x) * int blocks) */ -AES_ENTRY(aes_ecb_encrypt) +AES_FUNC_START(aes_ecb_encrypt) stp x29, x30, [sp, #-16]! mov x29, sp @@ -79,10 +79,10 @@ ST5( st1 {v4.16b}, [x0], #16 ) .Lecbencout: ldp x29, x30, [sp], #16 ret -AES_ENDPROC(aes_ecb_encrypt) +AES_FUNC_END(aes_ecb_encrypt) -AES_ENTRY(aes_ecb_decrypt) +AES_FUNC_START(aes_ecb_decrypt) stp x29, x30, [sp, #-16]! mov x29, sp @@ -110,7 +110,7 @@ ST5( st1 {v4.16b}, [x0], #16 ) .Lecbdecout: ldp x29, x30, [sp], #16 ret -AES_ENDPROC(aes_ecb_decrypt) +AES_FUNC_END(aes_ecb_decrypt) /* @@ -126,7 +126,7 @@ AES_ENDPROC(aes_ecb_decrypt) * u32 const rk2[]); */ -AES_ENTRY(aes_essiv_cbc_encrypt) +AES_FUNC_START(aes_essiv_cbc_encrypt) ld1 {v4.16b}, [x5] /* get iv */ mov w8, #14 /* AES-256: 14 rounds */ @@ -135,7 +135,7 @@ AES_ENTRY(aes_essiv_cbc_encrypt) enc_switch_key w3, x2, x6 b .Lcbcencloop4x -AES_ENTRY(aes_cbc_encrypt) +AES_FUNC_START(aes_cbc_encrypt) ld1 {v4.16b}, [x5] /* get iv */ enc_prepare w3, x2, x6 @@ -167,10 +167,10 @@ AES_ENTRY(aes_cbc_encrypt) .Lcbcencout: st1 {v4.16b}, [x5] /* return iv */ ret -AES_ENDPROC(aes_cbc_encrypt) -AES_ENDPROC(aes_essiv_cbc_encrypt) +AES_FUNC_END(aes_cbc_encrypt) +AES_FUNC_END(aes_essiv_cbc_encrypt) -AES_ENTRY(aes_essiv_cbc_decrypt) +AES_FUNC_START(aes_essiv_cbc_decrypt) stp x29, x30, [sp, #-16]! mov x29, sp @@ -181,7 +181,7 @@ AES_ENTRY(aes_essiv_cbc_decrypt) encrypt_block cbciv, w8, x6, x7, w9 b .Lessivcbcdecstart -AES_ENTRY(aes_cbc_decrypt) +AES_FUNC_START(aes_cbc_decrypt) stp x29, x30, [sp, #-16]! mov x29, sp @@ -238,8 +238,8 @@ ST5( st1 {v4.16b}, [x0], #16 ) st1 {cbciv.16b}, [x5] /* return iv */ ldp x29, x30, [sp], #16 ret -AES_ENDPROC(aes_cbc_decrypt) -AES_ENDPROC(aes_essiv_cbc_decrypt) +AES_FUNC_END(aes_cbc_decrypt) +AES_FUNC_END(aes_essiv_cbc_decrypt) /* @@ -249,7 +249,7 @@ AES_ENDPROC(aes_essiv_cbc_decrypt) * int rounds, int bytes, u8 const iv[]) */ -AES_ENTRY(aes_cbc_cts_encrypt) +AES_FUNC_START(aes_cbc_cts_encrypt) adr_l x8, .Lcts_permute_table sub x4, x4, #16 add x9, x8, #32 @@ -276,9 +276,9 @@ AES_ENTRY(aes_cbc_cts_encrypt) st1 {v0.16b}, [x4] /* overlapping stores */ st1 {v1.16b}, [x0] ret -AES_ENDPROC(aes_cbc_cts_encrypt) +AES_FUNC_END(aes_cbc_cts_encrypt) -AES_ENTRY(aes_cbc_cts_decrypt) +AES_FUNC_START(aes_cbc_cts_decrypt) adr_l x8, .Lcts_permute_table sub x4, x4, #16 add x9, x8, #32 @@ -305,7 +305,7 @@ AES_ENTRY(aes_cbc_cts_decrypt) st1 {v2.16b}, [x4] /* overlapping stores */ st1 {v0.16b}, [x0] ret -AES_ENDPROC(aes_cbc_cts_decrypt) +AES_FUNC_END(aes_cbc_cts_decrypt) .section ".rodata", "a" .align 6 @@ -324,7 +324,7 @@ AES_ENDPROC(aes_cbc_cts_decrypt) * int blocks, u8 ctr[]) */ -AES_ENTRY(aes_ctr_encrypt) +AES_FUNC_START(aes_ctr_encrypt) stp x29, x30, [sp, #-16]! mov x29, sp @@ -409,7 +409,7 @@ ST5( st1 {v4.16b}, [x0], #16 ) rev x7, x7 ins vctr.d[0], x7 b .Lctrcarrydone -AES_ENDPROC(aes_ctr_encrypt) +AES_FUNC_END(aes_ctr_encrypt) /* @@ -433,7 +433,7 @@ AES_ENDPROC(aes_ctr_encrypt) uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s .endm -AES_ENTRY(aes_xts_encrypt) +AES_FUNC_START(aes_xts_encrypt) stp x29, x30, [sp, #-16]! mov x29, sp @@ -518,9 +518,9 @@ AES_ENTRY(aes_xts_encrypt) st1 {v2.16b}, [x4] /* overlapping stores */ mov w4, wzr b .Lxtsencctsout -AES_ENDPROC(aes_xts_encrypt) +AES_FUNC_END(aes_xts_encrypt) -AES_ENTRY(aes_xts_decrypt) +AES_FUNC_START(aes_xts_decrypt) stp x29, x30, [sp, #-16]! mov x29, sp @@ -612,13 +612,13 @@ AES_ENTRY(aes_xts_decrypt) st1 {v2.16b}, [x4] /* overlapping stores */ mov w4, wzr b .Lxtsdecctsout -AES_ENDPROC(aes_xts_decrypt) +AES_FUNC_END(aes_xts_decrypt) /* * aes_mac_update(u8 const in[], u32 const rk[], int rounds, * int blocks, u8 dg[], int enc_before, int enc_after) */ -AES_ENTRY(aes_mac_update) +AES_FUNC_START(aes_mac_update) frame_push 6 mov x19, x0 @@ -676,4 +676,4 @@ AES_ENTRY(aes_mac_update) ld1 {v0.16b}, [x23] /* get dg */ enc_prepare w21, x20, x0 b .Lmacloop4x -AES_ENDPROC(aes_mac_update) +AES_FUNC_END(aes_mac_update) diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S index 247d34ddaab0..e47d3ec2cfb4 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/arch/arm64/crypto/aes-neon.S @@ -8,8 +8,8 @@ #include <linux/linkage.h> #include <asm/assembler.h> -#define AES_ENTRY(func) SYM_FUNC_START(neon_ ## func) -#define AES_ENDPROC(func) SYM_FUNC_END(neon_ ## func) +#define AES_FUNC_START(func) SYM_FUNC_START(neon_ ## func) +#define AES_FUNC_END(func) SYM_FUNC_END(neon_ ## func) xtsmask .req v7 cbciv .req v7 diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index c1f9660d104c..37ca3e889848 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -55,10 +55,10 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, break; } chacha_4block_xor_neon(state, dst, src, nrounds, l); - bytes -= CHACHA_BLOCK_SIZE * 5; - src += CHACHA_BLOCK_SIZE * 5; - dst += CHACHA_BLOCK_SIZE * 5; - state[12] += 5; + bytes -= l; + src += l; + dst += l; + state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } } diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 084c6a30b03a..6b958dcdf136 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -587,20 +587,20 @@ CPU_LE( rev w8, w8 ) * struct ghash_key const *k, u64 dg[], u8 ctr[], * int rounds, u8 tag) */ -ENTRY(pmull_gcm_encrypt) +SYM_FUNC_START(pmull_gcm_encrypt) pmull_gcm_do_crypt 1 -ENDPROC(pmull_gcm_encrypt) +SYM_FUNC_END(pmull_gcm_encrypt) /* * void pmull_gcm_decrypt(int blocks, u8 dst[], const u8 src[], * struct ghash_key const *k, u64 dg[], u8 ctr[], * int rounds, u8 tag) */ -ENTRY(pmull_gcm_decrypt) +SYM_FUNC_START(pmull_gcm_decrypt) pmull_gcm_do_crypt 0 -ENDPROC(pmull_gcm_decrypt) +SYM_FUNC_END(pmull_gcm_decrypt) -pmull_gcm_ghash_4x: +SYM_FUNC_START_LOCAL(pmull_gcm_ghash_4x) movi MASK.16b, #0xe1 shl MASK.2d, MASK.2d, #57 @@ -681,9 +681,9 @@ pmull_gcm_ghash_4x: eor XL.16b, XL.16b, T2.16b ret -ENDPROC(pmull_gcm_ghash_4x) +SYM_FUNC_END(pmull_gcm_ghash_4x) -pmull_gcm_enc_4x: +SYM_FUNC_START_LOCAL(pmull_gcm_enc_4x) ld1 {KS0.16b}, [x5] // load upper counter sub w10, w8, #4 sub w11, w8, #3 @@ -746,7 +746,7 @@ pmull_gcm_enc_4x: eor INP3.16b, INP3.16b, KS3.16b ret -ENDPROC(pmull_gcm_enc_4x) +SYM_FUNC_END(pmull_gcm_enc_4x) .section ".rodata", "a" .align 6 diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index 324e7d5ab37e..5e5dc05d63a0 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -221,7 +221,7 @@ alternative_endif .macro user_alt, label, oldinstr, newinstr, cond 9999: alternative_insn "\oldinstr", "\newinstr", \cond - _ASM_EXTABLE 9999b, \label + _asm_extable 9999b, \label .endm /* diff --git a/arch/arm64/include/asm/asm_pointer_auth.h b/arch/arm64/include/asm/asm_pointer_auth.h new file mode 100644 index 000000000000..ce2a8486992b --- /dev/null +++ b/arch/arm64/include/asm/asm_pointer_auth.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_ASM_POINTER_AUTH_H +#define __ASM_ASM_POINTER_AUTH_H + +#include <asm/alternative.h> +#include <asm/asm-offsets.h> +#include <asm/cpufeature.h> +#include <asm/sysreg.h> + +#ifdef CONFIG_ARM64_PTR_AUTH +/* + * thread.keys_user.ap* as offset exceeds the #imm offset range + * so use the base value of ldp as thread.keys_user and offset as + * thread.keys_user.ap*. + */ + .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 + mov \tmp1, #THREAD_KEYS_USER + add \tmp1, \tsk, \tmp1 +alternative_if_not ARM64_HAS_ADDRESS_AUTH + b .Laddr_auth_skip_\@ +alternative_else_nop_endif + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA] + msr_s SYS_APIAKEYLO_EL1, \tmp2 + msr_s SYS_APIAKEYHI_EL1, \tmp3 + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIB] + msr_s SYS_APIBKEYLO_EL1, \tmp2 + msr_s SYS_APIBKEYHI_EL1, \tmp3 + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDA] + msr_s SYS_APDAKEYLO_EL1, \tmp2 + msr_s SYS_APDAKEYHI_EL1, \tmp3 + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDB] + msr_s SYS_APDBKEYLO_EL1, \tmp2 + msr_s SYS_APDBKEYHI_EL1, \tmp3 +.Laddr_auth_skip_\@: +alternative_if ARM64_HAS_GENERIC_AUTH + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APGA] + msr_s SYS_APGAKEYLO_EL1, \tmp2 + msr_s SYS_APGAKEYHI_EL1, \tmp3 +alternative_else_nop_endif + .endm + + .macro ptrauth_keys_install_kernel tsk, sync, tmp1, tmp2, tmp3 +alternative_if ARM64_HAS_ADDRESS_AUTH + mov \tmp1, #THREAD_KEYS_KERNEL + add \tmp1, \tsk, \tmp1 + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_KERNEL_KEY_APIA] + msr_s SYS_APIAKEYLO_EL1, \tmp2 + msr_s SYS_APIAKEYHI_EL1, \tmp3 + .if \sync == 1 + isb + .endif +alternative_else_nop_endif + .endm + +#else /* CONFIG_ARM64_PTR_AUTH */ + + .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 + .endm + + .macro ptrauth_keys_install_kernel tsk, sync, tmp1, tmp2, tmp3 + .endm + +#endif /* CONFIG_ARM64_PTR_AUTH */ + +#endif /* __ASM_ASM_POINTER_AUTH_H */ diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index aca337d79d12..0bff325117b4 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -257,12 +257,6 @@ alternative_endif .endm /* - * mmid - get context id from mm pointer (mm->context.id) - */ - .macro mmid, rd, rn - ldr \rd, [\rn, #MM_CONTEXT_ID] - .endm -/* * read_ctr - read CTR_EL0. If the system has mismatched register fields, * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val */ @@ -431,6 +425,16 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU .endm /* + * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present + */ + .macro reset_amuserenr_el0, tmpreg + mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 + ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4 + cbz \tmpreg, .Lskip_\@ // Skip if no AMU present + msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 +.Lskip_\@: + .endm +/* * copy_page - copy src to dest using temp registers t1-t8 */ .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h index 8d2a7de39744..b6f7bc6da5fb 100644 --- a/arch/arm64/include/asm/checksum.h +++ b/arch/arm64/include/asm/checksum.h @@ -5,7 +5,12 @@ #ifndef __ASM_CHECKSUM_H #define __ASM_CHECKSUM_H -#include <linux/types.h> +#include <linux/in6.h> + +#define _HAVE_ARCH_IPV6_CSUM +__sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum sum); static inline __sum16 csum_fold(__wsum csum) { diff --git a/arch/arm64/include/asm/clocksource.h b/arch/arm64/include/asm/clocksource.h index 0ece64a26c8c..482185566b0c 100644 --- a/arch/arm64/include/asm/clocksource.h +++ b/arch/arm64/include/asm/clocksource.h @@ -2,8 +2,6 @@ #ifndef _ASM_CLOCKSOURCE_H #define _ASM_CLOCKSOURCE_H -struct arch_clocksource_data { - bool vdso_direct; /* Usable for direct VDSO access? */ -}; +#include <asm/vdso/clocksource.h> #endif diff --git a/arch/arm64/include/asm/compiler.h b/arch/arm64/include/asm/compiler.h new file mode 100644 index 000000000000..eece20d2c55f --- /dev/null +++ b/arch/arm64/include/asm/compiler.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_COMPILER_H +#define __ASM_COMPILER_H + +#if defined(CONFIG_ARM64_PTR_AUTH) + +/* + * The EL0/EL1 pointer bits used by a pointer authentication code. + * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply. + */ +#define ptrauth_user_pac_mask() GENMASK_ULL(54, vabits_actual) +#define ptrauth_kernel_pac_mask() GENMASK_ULL(63, vabits_actual) + +/* Valid for EL0 TTBR0 and EL1 TTBR1 instruction pointers */ +#define ptrauth_clear_pac(ptr) \ + ((ptr & BIT_ULL(55)) ? (ptr | ptrauth_kernel_pac_mask()) : \ + (ptr & ~ptrauth_user_pac_mask())) + +#define __builtin_return_address(val) \ + (void *)(ptrauth_clear_pac((unsigned long)__builtin_return_address(val))) + +#endif /* CONFIG_ARM64_PTR_AUTH */ + +#endif /* __ASM_COMPILER_H */ diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 86aabf1e0199..d28e8f37d3b4 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -55,12 +55,12 @@ struct cpu_operations { #endif }; -extern const struct cpu_operations *cpu_ops[NR_CPUS]; -int __init cpu_read_ops(int cpu); +int __init init_cpu_ops(int cpu); +extern const struct cpu_operations *get_cpu_ops(int cpu); -static inline void __init cpu_read_bootcpu_ops(void) +static inline void __init init_bootcpu_ops(void) { - cpu_read_ops(0); + init_cpu_ops(0); } #endif /* ifndef __ASM_CPU_OPS_H */ diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 865e0253fc1e..8eb5a088ae65 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -58,7 +58,10 @@ #define ARM64_WORKAROUND_SPECULATIVE_AT_NVHE 48 #define ARM64_HAS_E0PD 49 #define ARM64_HAS_RNG 50 +#define ARM64_HAS_AMU_EXTN 51 +#define ARM64_HAS_ADDRESS_AUTH 52 +#define ARM64_HAS_GENERIC_AUTH 53 -#define ARM64_NCAPS 51 +#define ARM64_NCAPS 54 #endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 2a746b99e937..afe08251ff95 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -208,6 +208,10 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; * In some non-typical cases either both (a) and (b), or neither, * should be permitted. This can be described by including neither * or both flags in the capability's type field. + * + * In case of a conflict, the CPU is prevented from booting. If the + * ARM64_CPUCAP_PANIC_ON_CONFLICT flag is specified for the capability, + * then a kernel panic is triggered. */ @@ -240,6 +244,8 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; #define ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU ((u16)BIT(4)) /* Is it safe for a late CPU to miss this capability when system has it */ #define ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU ((u16)BIT(5)) +/* Panic when a conflict is detected */ +#define ARM64_CPUCAP_PANIC_ON_CONFLICT ((u16)BIT(6)) /* * CPU errata workarounds that need to be enabled at boot time if one or @@ -279,9 +285,20 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; /* * CPU feature used early in the boot based on the boot CPU. All secondary - * CPUs must match the state of the capability as detected by the boot CPU. + * CPUs must match the state of the capability as detected by the boot CPU. In + * case of a conflict, a kernel panic is triggered. + */ +#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE \ + (ARM64_CPUCAP_SCOPE_BOOT_CPU | ARM64_CPUCAP_PANIC_ON_CONFLICT) + +/* + * CPU feature used early in the boot based on the boot CPU. It is safe for a + * late CPU to have this feature even though the boot CPU hasn't enabled it, + * although the feature will not be used by Linux in this case. If the boot CPU + * has enabled this feature already, then every late CPU must have it. */ -#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE ARM64_CPUCAP_SCOPE_BOOT_CPU +#define ARM64_CPUCAP_BOOT_CPU_FEATURE \ + (ARM64_CPUCAP_SCOPE_BOOT_CPU | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU) struct arm64_cpu_capabilities { const char *desc; @@ -340,18 +357,6 @@ static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap) return cap->type & ARM64_CPUCAP_SCOPE_MASK; } -static inline bool -cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap) -{ - return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU); -} - -static inline bool -cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap) -{ - return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU); -} - /* * Generic helper for handling capabilties with multiple (match,enable) pairs * of call backs, sharing the same capability bit. @@ -390,14 +395,16 @@ unsigned long cpu_get_elf_hwcap2(void); #define cpu_set_named_feature(name) cpu_set_feature(cpu_feature(name)) #define cpu_have_named_feature(name) cpu_have_feature(cpu_feature(name)) -/* System capability check for constant caps */ -static __always_inline bool __cpus_have_const_cap(int num) +static __always_inline bool system_capabilities_finalized(void) { - if (num >= ARM64_NCAPS) - return false; - return static_branch_unlikely(&cpu_hwcap_keys[num]); + return static_branch_likely(&arm64_const_caps_ready); } +/* + * Test for a capability with a runtime check. + * + * Before the capability is detected, this returns false. + */ static inline bool cpus_have_cap(unsigned int num) { if (num >= ARM64_NCAPS) @@ -405,14 +412,53 @@ static inline bool cpus_have_cap(unsigned int num) return test_bit(num, cpu_hwcaps); } +/* + * Test for a capability without a runtime check. + * + * Before capabilities are finalized, this returns false. + * After capabilities are finalized, this is patched to avoid a runtime check. + * + * @num must be a compile-time constant. + */ +static __always_inline bool __cpus_have_const_cap(int num) +{ + if (num >= ARM64_NCAPS) + return false; + return static_branch_unlikely(&cpu_hwcap_keys[num]); +} + +/* + * Test for a capability, possibly with a runtime check. + * + * Before capabilities are finalized, this behaves as cpus_have_cap(). + * After capabilities are finalized, this is patched to avoid a runtime check. + * + * @num must be a compile-time constant. + */ static __always_inline bool cpus_have_const_cap(int num) { - if (static_branch_likely(&arm64_const_caps_ready)) + if (system_capabilities_finalized()) return __cpus_have_const_cap(num); else return cpus_have_cap(num); } +/* + * Test for a capability without a runtime check. + * + * Before capabilities are finalized, this will BUG(). + * After capabilities are finalized, this is patched to avoid a runtime check. + * + * @num must be a compile-time constant. + */ +static __always_inline bool cpus_have_final_cap(int num) +{ + if (system_capabilities_finalized()) + return __cpus_have_const_cap(num); + else + BUG(); +} + static inline void cpus_set_cap(unsigned int num) { if (num >= ARM64_NCAPS) { @@ -447,6 +493,29 @@ cpuid_feature_extract_unsigned_field(u64 features, int field) return cpuid_feature_extract_unsigned_field_width(features, field, 4); } +/* + * Fields that identify the version of the Performance Monitors Extension do + * not follow the standard ID scheme. See ARM DDI 0487E.a page D13-2825, + * "Alternative ID scheme used for the Performance Monitors Extension version". + */ +static inline u64 __attribute_const__ +cpuid_feature_cap_perfmon_field(u64 features, int field, u64 cap) +{ + u64 val = cpuid_feature_extract_unsigned_field(features, field); + u64 mask = GENMASK_ULL(field + 3, field); + + /* Treat IMPLEMENTATION DEFINED functionality as unimplemented */ + if (val == 0xf) + val = 0; + + if (val > cap) { + features &= ~mask; + features |= (cap << field) & mask; + } + + return features; +} + static inline u64 arm64_ftr_mask(const struct arm64_ftr_bits *ftrp) { return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift); @@ -590,15 +659,13 @@ static __always_inline bool system_supports_cnp(void) static inline bool system_supports_address_auth(void) { return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && - (cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH_ARCH) || - cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF)); + cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH); } static inline bool system_supports_generic_auth(void) { return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && - (cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH_ARCH) || - cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF)); + cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH); } static inline bool system_uses_irq_prio_masking(void) @@ -613,11 +680,6 @@ static inline bool system_has_prio_mask_debugging(void) system_uses_irq_prio_masking(); } -static inline bool system_capabilities_finalized(void) -{ - return static_branch_likely(&arm64_const_caps_ready); -} - #define ARM64_BP_HARDEN_UNKNOWN -1 #define ARM64_BP_HARDEN_WA_NEEDED 0 #define ARM64_BP_HARDEN_NOT_REQUIRED 1 @@ -678,6 +740,11 @@ static inline bool cpu_has_hw_af(void) ID_AA64MMFR1_HADBS_SHIFT); } +#ifdef CONFIG_ARM64_AMU_EXTN +/* Check whether the cpu supports the Activity Monitors Unit (AMU) */ +extern bool cpu_has_amu_feat(int cpu); +#endif + #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 44531a69d32b..45e821222774 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -58,13 +58,6 @@ efi_status_t __efi_rt_asm_wrapper(void *, const char *, ...); /* arch specific definitions used by the stub code */ /* - * AArch64 requires the DTB to be 8-byte aligned in the first 512MiB from - * start of kernel and may not cross a 2MiB boundary. We set alignment to - * 2MiB so we know it won't cross a 2MiB boundary. - */ -#define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */ - -/* * In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the * kernel need greater alignment than we require the segments to be padded to. */ @@ -107,9 +100,6 @@ static inline void free_screen_info(struct screen_info *si) { } -/* redeclare as 'hidden' so the compiler will generate relative references */ -extern struct screen_info screen_info __attribute__((__visibility__("hidden"))); - static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt) { } diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index cb29253ae86b..6a395a7e6707 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -60,7 +60,7 @@ #define ESR_ELx_EC_BKPT32 (0x38) /* Unallocated EC: 0x39 */ #define ESR_ELx_EC_VECTOR32 (0x3A) /* EL2 only */ -/* Unallocted EC: 0x3B */ +/* Unallocated EC: 0x3B */ #define ESR_ELx_EC_BRK64 (0x3C) /* Unallocated EC: 0x3D - 0x3F */ #define ESR_ELx_EC_MAX (0x3F) diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 6cc26a127819..97f6a63810ec 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -48,7 +48,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr) int oldval = 0, ret, tmp; u32 __user *uaddr = __uaccess_mask_ptr(_uaddr); - pagefault_disable(); + if (!access_ok(_uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -75,8 +76,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 6e5d839f42b5..51c1d9918999 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -267,6 +267,7 @@ /* Hyp Coprocessor Trap Register */ #define CPTR_EL2_TCPAC (1 << 31) +#define CPTR_EL2_TAM (1 << 30) #define CPTR_EL2_TTA (1 << 20) #define CPTR_EL2_TFP (1 << CPTR_EL2_TFP_SHIFT) #define CPTR_EL2_TZ (1 << 8) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 44a243754c1b..7c7eeeaab9fa 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -36,6 +36,8 @@ */ #define KVM_VECTOR_PREAMBLE (2 * AARCH64_INSN_SIZE) +#define __SMCCC_WORKAROUND_1_SMC_SZ 36 + #ifndef __ASSEMBLY__ #include <linux/mm.h> @@ -75,6 +77,8 @@ extern void __vgic_v3_init_lrs(void); extern u32 __kvm_get_mdcr_el2(void); +extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; + /* Home-grown __this_cpu_{ptr,read} variants that always work at HYP */ #define __hyp_this_cpu_ptr(sym) \ ({ \ diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 785762860c63..30b0e8d6b895 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -481,7 +481,7 @@ static inline void *kvm_get_hyp_vector(void) int slot = -1; if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR) && data->fn) { - vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs_start)); + vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); slot = data->hyp_vectors_slot; } @@ -510,14 +510,13 @@ static inline int kvm_map_vectors(void) * HBP + HEL2 -> use hardened vertors and use exec mapping */ if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR)) { - __kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs_start); + __kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs); __kvm_bp_vect_base = kern_hyp_va(__kvm_bp_vect_base); } if (cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) { - phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs_start); - unsigned long size = (__bp_harden_hyp_vecs_end - - __bp_harden_hyp_vecs_start); + phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs); + unsigned long size = __BP_HARDEN_HYP_VECS_SZ; /* * Always allocate a spare vector slot, as we don't diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 4d94676e5a8b..2be67b232499 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -54,6 +54,7 @@ #define MODULES_VADDR (BPF_JIT_REGION_END) #define MODULES_VSIZE (SZ_128M) #define VMEMMAP_START (-VMEMMAP_SIZE - SZ_2M) +#define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE) #define PCI_IO_END (VMEMMAP_START - SZ_2M) #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) #define FIXADDR_TOP (PCI_IO_START - SZ_2M) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index d79ce6df9e12..68140fdd89d6 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -13,6 +13,7 @@ #define TTBR_ASID_MASK (UL(0xffff) << 48) #define BP_HARDEN_EL2_SLOTS 4 +#define __BP_HARDEN_HYP_VECS_SZ (BP_HARDEN_EL2_SLOTS * SZ_2K) #ifndef __ASSEMBLY__ @@ -23,9 +24,9 @@ typedef struct { } mm_context_t; /* - * This macro is only used by the TLBI code, which cannot race with an - * ASID change and therefore doesn't need to reload the counter using - * atomic64_read. + * This macro is only used by the TLBI and low-level switch_mm() code, + * neither of which can race with an ASID change. We therefore don't + * need to reload the counter using atomic64_read(). */ #define ASID(mm) ((mm)->context.id.counter & 0xffff) @@ -43,7 +44,8 @@ struct bp_hardening_data { #if (defined(CONFIG_HARDEN_BRANCH_PREDICTOR) || \ defined(CONFIG_HARDEN_EL2_VECTORS)) -extern char __bp_harden_hyp_vecs_start[], __bp_harden_hyp_vecs_end[]; + +extern char __bp_harden_hyp_vecs[]; extern atomic_t arm64_el2_vector_last_slot; #endif /* CONFIG_HARDEN_BRANCH_PREDICTOR || CONFIG_HARDEN_EL2_VECTORS */ diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 3827ff4040a3..ab46187c6300 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -46,6 +46,8 @@ static inline void cpu_set_reserved_ttbr0(void) isb(); } +void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); + static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) { BUG_ON(pgd == swapper_pg_dir); diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index d39ddb258a04..75d6cd23a679 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -21,6 +21,10 @@ extern void __cpu_copy_user_page(void *to, const void *from, extern void copy_page(void *to, const void *from); extern void clear_page(void *to); +#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ + alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE + #define clear_user_page(addr,vaddr,pg) __cpu_clear_user_page(addr, vaddr) #define copy_user_page(to,from,vaddr,pg) __cpu_copy_user_page(to, from, vaddr) diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h index 2bdbc79bbd01..e7765b62c712 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -176,9 +176,10 @@ #define ARMV8_PMU_PMCR_X (1 << 4) /* Export to ETM */ #define ARMV8_PMU_PMCR_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ #define ARMV8_PMU_PMCR_LC (1 << 6) /* Overflow on 64 bit cycle counter */ +#define ARMV8_PMU_PMCR_LP (1 << 7) /* Long event counter enable */ #define ARMV8_PMU_PMCR_N_SHIFT 11 /* Number of counters supported */ #define ARMV8_PMU_PMCR_N_MASK 0x1f -#define ARMV8_PMU_PMCR_MASK 0x7f /* Mask for writable bits */ +#define ARMV8_PMU_PMCR_MASK 0xff /* Mask for writable bits */ /* * PMOVSR: counters overflow flag status reg diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h index 7a24bad1a58b..70c47156e54b 100644 --- a/arch/arm64/include/asm/pointer_auth.h +++ b/arch/arm64/include/asm/pointer_auth.h @@ -22,7 +22,7 @@ struct ptrauth_key { * We give each process its own keys, which are shared by all threads. The keys * are inherited upon fork(), and reinitialised upon exec*(). */ -struct ptrauth_keys { +struct ptrauth_keys_user { struct ptrauth_key apia; struct ptrauth_key apib; struct ptrauth_key apda; @@ -30,7 +30,11 @@ struct ptrauth_keys { struct ptrauth_key apga; }; -static inline void ptrauth_keys_init(struct ptrauth_keys *keys) +struct ptrauth_keys_kernel { + struct ptrauth_key apia; +}; + +static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys) { if (system_supports_address_auth()) { get_random_bytes(&keys->apia, sizeof(keys->apia)); @@ -50,48 +54,38 @@ do { \ write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \ } while (0) -static inline void ptrauth_keys_switch(struct ptrauth_keys *keys) +static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys) { - if (system_supports_address_auth()) { - __ptrauth_key_install(APIA, keys->apia); - __ptrauth_key_install(APIB, keys->apib); - __ptrauth_key_install(APDA, keys->apda); - __ptrauth_key_install(APDB, keys->apdb); - } + if (system_supports_address_auth()) + get_random_bytes(&keys->apia, sizeof(keys->apia)); +} - if (system_supports_generic_auth()) - __ptrauth_key_install(APGA, keys->apga); +static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys) +{ + if (system_supports_address_auth()) + __ptrauth_key_install(APIA, keys->apia); } extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg); -/* - * The EL0 pointer bits used by a pointer authentication code. - * This is dependent on TBI0 being enabled, or bits 63:56 would also apply. - */ -#define ptrauth_user_pac_mask() GENMASK(54, vabits_actual) - -/* Only valid for EL0 TTBR0 instruction pointers */ static inline unsigned long ptrauth_strip_insn_pac(unsigned long ptr) { - return ptr & ~ptrauth_user_pac_mask(); + return ptrauth_clear_pac(ptr); } #define ptrauth_thread_init_user(tsk) \ -do { \ - struct task_struct *__ptiu_tsk = (tsk); \ - ptrauth_keys_init(&__ptiu_tsk->thread.keys_user); \ - ptrauth_keys_switch(&__ptiu_tsk->thread.keys_user); \ -} while (0) - -#define ptrauth_thread_switch(tsk) \ - ptrauth_keys_switch(&(tsk)->thread.keys_user) + ptrauth_keys_init_user(&(tsk)->thread.keys_user) +#define ptrauth_thread_init_kernel(tsk) \ + ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel) +#define ptrauth_thread_switch_kernel(tsk) \ + ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel) #else /* CONFIG_ARM64_PTR_AUTH */ #define ptrauth_prctl_reset_keys(tsk, arg) (-EINVAL) #define ptrauth_strip_insn_pac(lr) (lr) #define ptrauth_thread_init_user(tsk) -#define ptrauth_thread_switch(tsk) +#define ptrauth_thread_init_kernel(tsk) +#define ptrauth_thread_switch_kernel(tsk) #endif /* CONFIG_ARM64_PTR_AUTH */ #endif /* __ASM_POINTER_AUTH_H */ diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h index a2ce65a0c1fa..0d5d1f0525eb 100644 --- a/arch/arm64/include/asm/proc-fns.h +++ b/arch/arm64/include/asm/proc-fns.h @@ -13,11 +13,9 @@ #include <asm/page.h> -struct mm_struct; struct cpu_suspend_ctx; extern void cpu_do_idle(void); -extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr); extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr); diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 5ba63204d078..240fe5e5b720 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -28,6 +28,8 @@ #include <linux/string.h> #include <linux/thread_info.h> +#include <vdso/processor.h> + #include <asm/alternative.h> #include <asm/cpufeature.h> #include <asm/hw_breakpoint.h> @@ -146,7 +148,8 @@ struct thread_struct { unsigned long fault_code; /* ESR_EL1 value */ struct debug_info debug; /* debugging */ #ifdef CONFIG_ARM64_PTR_AUTH - struct ptrauth_keys keys_user; + struct ptrauth_keys_user keys_user; + struct ptrauth_keys_kernel keys_kernel; #endif }; @@ -256,11 +259,6 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); -static inline void cpu_relax(void) -{ - asm volatile("yield" ::: "memory"); -} - /* Thread switching */ extern struct task_struct *cpu_switch_to(struct task_struct *prev, struct task_struct *next); diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index a0c8a0b65259..40d5ba029615 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -23,6 +23,14 @@ #define CPU_STUCK_REASON_52_BIT_VA (UL(1) << CPU_STUCK_REASON_SHIFT) #define CPU_STUCK_REASON_NO_GRAN (UL(2) << CPU_STUCK_REASON_SHIFT) +/* Possible options for __cpu_setup */ +/* Option to setup primary cpu */ +#define ARM64_CPU_BOOT_PRIMARY (1) +/* Option to setup secondary cpus */ +#define ARM64_CPU_BOOT_SECONDARY (2) +/* Option to setup cpus for different cpu run time services */ +#define ARM64_CPU_RUNTIME (3) + #ifndef __ASSEMBLY__ #include <asm/percpu.h> @@ -30,6 +38,7 @@ #include <linux/threads.h> #include <linux/cpumask.h> #include <linux/thread_info.h> +#include <asm/pointer_auth.h> DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number); @@ -87,6 +96,9 @@ asmlinkage void secondary_start_kernel(void); struct secondary_data { void *stack; struct task_struct *task; +#ifdef CONFIG_ARM64_PTR_AUTH + struct ptrauth_keys_kernel ptrauth_key; +#endif long status; }; diff --git a/arch/arm64/include/asm/stackprotector.h b/arch/arm64/include/asm/stackprotector.h index 5884a2b02827..7263e0bac680 100644 --- a/arch/arm64/include/asm/stackprotector.h +++ b/arch/arm64/include/asm/stackprotector.h @@ -15,6 +15,7 @@ #include <linux/random.h> #include <linux/version.h> +#include <asm/pointer_auth.h> extern unsigned long __stack_chk_guard; @@ -26,6 +27,7 @@ extern unsigned long __stack_chk_guard; */ static __always_inline void boot_init_stack_canary(void) { +#if defined(CONFIG_STACKPROTECTOR) unsigned long canary; /* Try to get a semi random initial value. */ @@ -36,6 +38,9 @@ static __always_inline void boot_init_stack_canary(void) current->stack_canary = canary; if (!IS_ENABLED(CONFIG_STACKPROTECTOR_PER_TASK)) __stack_chk_guard = current->stack_canary; +#endif + ptrauth_thread_init_kernel(current); + ptrauth_thread_switch_kernel(current); } #endif /* _ASM_STACKPROTECTOR_H */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index b91570ff9db1..ebc622432831 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -386,6 +386,42 @@ #define SYS_TPIDR_EL0 sys_reg(3, 3, 13, 0, 2) #define SYS_TPIDRRO_EL0 sys_reg(3, 3, 13, 0, 3) +/* Definitions for system register interface to AMU for ARMv8.4 onwards */ +#define SYS_AM_EL0(crm, op2) sys_reg(3, 3, 13, (crm), (op2)) +#define SYS_AMCR_EL0 SYS_AM_EL0(2, 0) +#define SYS_AMCFGR_EL0 SYS_AM_EL0(2, 1) +#define SYS_AMCGCR_EL0 SYS_AM_EL0(2, 2) +#define SYS_AMUSERENR_EL0 SYS_AM_EL0(2, 3) +#define SYS_AMCNTENCLR0_EL0 SYS_AM_EL0(2, 4) +#define SYS_AMCNTENSET0_EL0 SYS_AM_EL0(2, 5) +#define SYS_AMCNTENCLR1_EL0 SYS_AM_EL0(3, 0) +#define SYS_AMCNTENSET1_EL0 SYS_AM_EL0(3, 1) + +/* + * Group 0 of activity monitors (architected): + * op0 op1 CRn CRm op2 + * Counter: 11 011 1101 010:n<3> n<2:0> + * Type: 11 011 1101 011:n<3> n<2:0> + * n: 0-15 + * + * Group 1 of activity monitors (auxiliary): + * op0 op1 CRn CRm op2 + * Counter: 11 011 1101 110:n<3> n<2:0> + * Type: 11 011 1101 111:n<3> n<2:0> + * n: 0-15 + */ + +#define SYS_AMEVCNTR0_EL0(n) SYS_AM_EL0(4 + ((n) >> 3), (n) & 7) +#define SYS_AMEVTYPE0_EL0(n) SYS_AM_EL0(6 + ((n) >> 3), (n) & 7) +#define SYS_AMEVCNTR1_EL0(n) SYS_AM_EL0(12 + ((n) >> 3), (n) & 7) +#define SYS_AMEVTYPE1_EL0(n) SYS_AM_EL0(14 + ((n) >> 3), (n) & 7) + +/* AMU v1: Fixed (architecturally defined) activity monitors */ +#define SYS_AMEVCNTR0_CORE_EL0 SYS_AMEVCNTR0_EL0(0) +#define SYS_AMEVCNTR0_CONST_EL0 SYS_AMEVCNTR0_EL0(1) +#define SYS_AMEVCNTR0_INST_RET_EL0 SYS_AMEVCNTR0_EL0(2) +#define SYS_AMEVCNTR0_MEM_STALL SYS_AMEVCNTR0_EL0(3) + #define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0) #define SYS_CNTP_TVAL_EL0 sys_reg(3, 3, 14, 2, 0) @@ -598,6 +634,7 @@ #define ID_AA64PFR0_CSV3_SHIFT 60 #define ID_AA64PFR0_CSV2_SHIFT 56 #define ID_AA64PFR0_DIT_SHIFT 48 +#define ID_AA64PFR0_AMU_SHIFT 44 #define ID_AA64PFR0_SVE_SHIFT 32 #define ID_AA64PFR0_RAS_SHIFT 28 #define ID_AA64PFR0_GIC_SHIFT 24 @@ -608,6 +645,7 @@ #define ID_AA64PFR0_EL1_SHIFT 4 #define ID_AA64PFR0_EL0_SHIFT 0 +#define ID_AA64PFR0_AMU 0x1 #define ID_AA64PFR0_SVE 0x1 #define ID_AA64PFR0_RAS_V1 0x1 #define ID_AA64PFR0_FP_NI 0xf @@ -702,6 +740,16 @@ #define ID_AA64DFR0_TRACEVER_SHIFT 4 #define ID_AA64DFR0_DEBUGVER_SHIFT 0 +#define ID_AA64DFR0_PMUVER_8_0 0x1 +#define ID_AA64DFR0_PMUVER_8_1 0x4 +#define ID_AA64DFR0_PMUVER_8_4 0x5 +#define ID_AA64DFR0_PMUVER_8_5 0x6 +#define ID_AA64DFR0_PMUVER_IMP_DEF 0xf + +#define ID_DFR0_PERFMON_SHIFT 24 + +#define ID_DFR0_PERFMON_8_1 0x4 + #define ID_ISAR5_RDM_SHIFT 24 #define ID_ISAR5_CRC32_SHIFT 16 #define ID_ISAR5_SHA2_SHIFT 12 diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index f0cec4160136..512174a8e789 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -63,7 +63,6 @@ void arch_release_task_struct(struct task_struct *tsk); #define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */ #define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */ #define TIF_FSCHECK 5 /* Check FS is USER_DS on return */ -#define TIF_NOHZ 7 #define TIF_SYSCALL_TRACE 8 /* syscall trace active */ #define TIF_SYSCALL_AUDIT 9 /* syscall auditing */ #define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */ @@ -83,7 +82,6 @@ void arch_release_task_struct(struct task_struct *tsk); #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) #define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE) -#define _TIF_NOHZ (1 << TIF_NOHZ) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) @@ -100,7 +98,7 @@ void arch_release_task_struct(struct task_struct *tsk); #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \ - _TIF_NOHZ | _TIF_SYSCALL_EMU) + _TIF_SYSCALL_EMU) #define INIT_THREAD_INFO(tsk) \ { \ diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index a4d945db95a2..0cc835ddfcd1 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -16,6 +16,15 @@ int pcibus_to_node(struct pci_bus *bus); #include <linux/arch_topology.h> +#ifdef CONFIG_ARM64_AMU_EXTN +/* + * Replace task scheduler's default counter-based + * frequency-invariance scale factor setting. + */ +void topology_scale_freq_tick(void); +#define arch_scale_freq_tick topology_scale_freq_tick +#endif /* CONFIG_ARM64_AMU_EXTN */ + /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_capacity topology_get_freq_scale @@ -25,6 +34,9 @@ int pcibus_to_node(struct pci_bus *bus); /* Enable topology flag updates */ #define arch_update_cpu_topology topology_update_cpu_topology +/* Replace task scheduler's default thermal pressure retrieve API */ +#define arch_scale_thermal_pressure topology_get_thermal_pressure + #include <asm-generic/topology.h> #endif /* _ASM_ARM_TOPOLOGY_H */ diff --git a/arch/arm64/include/asm/vdso/clocksource.h b/arch/arm64/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..df6ea65c1dec --- /dev/null +++ b/arch/arm64/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/arm64/include/asm/vdso/compat_gettimeofday.h b/arch/arm64/include/asm/vdso/compat_gettimeofday.h index 537b1e695365..b6907ae78e53 100644 --- a/arch/arm64/include/asm/vdso/compat_gettimeofday.h +++ b/arch/arm64/include/asm/vdso/compat_gettimeofday.h @@ -8,12 +8,10 @@ #ifndef __ASSEMBLY__ #include <asm/unistd.h> -#include <uapi/linux/time.h> +#include <asm/errno.h> #include <asm/vdso/compat_barrier.h> -#define __VDSO_USE_SYSCALL ULLONG_MAX - #define VDSO_HAS_CLOCK_GETRES 1 #define BUILD_VDSO32 1 @@ -78,10 +76,6 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) register long ret asm ("r0"); register long nr asm("r7") = __NR_compat_clock_getres_time64; - /* The checks below are required for ABI consistency with arm */ - if ((_clkid >= MAX_CLOCKS) && (_ts == NULL)) - return -EINVAL; - asm volatile( " swi #0\n" : "=r" (ret) @@ -99,10 +93,6 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) register long ret asm ("r0"); register long nr asm("r7") = __NR_compat_clock_getres; - /* The checks below are required for ABI consistency with arm */ - if ((_clkid >= MAX_CLOCKS) && (_ts == NULL)) - return -EINVAL; - asm volatile( " swi #0\n" : "=r" (ret) @@ -117,11 +107,12 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) u64 res; /* - * clock_mode == 0 implies that vDSO are enabled otherwise - * fallback on syscall. + * Core checks for mode already, so this raced against a concurrent + * update. Return something. Core will do another round and then + * see the mode change and fallback to the syscall. */ - if (clock_mode) - return __VDSO_USE_SYSCALL; + if (clock_mode == VDSO_CLOCKMODE_NONE) + return 0; /* * This isb() is required to prevent that the counter value diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h index b08f476b72b4..afba6ba332f8 100644 --- a/arch/arm64/include/asm/vdso/gettimeofday.h +++ b/arch/arm64/include/asm/vdso/gettimeofday.h @@ -8,9 +8,6 @@ #ifndef __ASSEMBLY__ #include <asm/unistd.h> -#include <uapi/linux/time.h> - -#define __VDSO_USE_SYSCALL ULLONG_MAX #define VDSO_HAS_CLOCK_GETRES 1 @@ -71,11 +68,12 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) u64 res; /* - * clock_mode == 0 implies that vDSO are enabled otherwise - * fallback on syscall. + * Core checks for mode already, so this raced against a concurrent + * update. Return something. Core will do another round and then + * see the mode change and fallback to the syscall. */ - if (clock_mode) - return __VDSO_USE_SYSCALL; + if (clock_mode == VDSO_CLOCKMODE_NONE) + return 0; /* * This isb() is required to prevent that the counter value diff --git a/arch/arm64/include/asm/vdso/processor.h b/arch/arm64/include/asm/vdso/processor.h new file mode 100644 index 000000000000..ff830b766ad2 --- /dev/null +++ b/arch/arm64/include/asm/vdso/processor.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +static inline void cpu_relax(void) +{ + asm volatile("yield" ::: "memory"); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/arm64/include/asm/vdso/vsyscall.h b/arch/arm64/include/asm/vdso/vsyscall.h index 0c20a7c1bee5..f94b1457c117 100644 --- a/arch/arm64/include/asm/vdso/vsyscall.h +++ b/arch/arm64/include/asm/vdso/vsyscall.h @@ -22,15 +22,6 @@ struct vdso_data *__arm64_get_k_vdso_data(void) #define __arch_get_k_vdso_data __arm64_get_k_vdso_data static __always_inline -int __arm64_get_clock_mode(struct timekeeper *tk) -{ - u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct; - - return use_syscall; -} -#define __arch_get_clock_mode __arm64_get_clock_mode - -static __always_inline void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk) { vdata[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK; diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fc6488660f64..4e5b8ee31442 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -21,7 +21,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o -extra-$(CONFIG_EFI) := efi-entry.o +targets += efi-entry.o OBJCOPYFLAGS := --prefix-symbols=__efistub_ $(obj)/%.stub.o: $(obj)/%.o FORCE diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index 7832b3216370..4cc581af2d96 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -630,7 +630,7 @@ static int __init armv8_deprecated_init(void) register_insn_emulation(&cp15_barrier_ops); if (IS_ENABLED(CONFIG_SETEND_EMULATION)) { - if(system_supports_mixed_endian_el0()) + if (system_supports_mixed_endian_el0()) register_insn_emulation(&setend_ops); else pr_info("setend instruction emulation is not supported on this system\n"); diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index a5bdce8af65b..9981a0a5a87f 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -40,6 +40,10 @@ int main(void) #endif BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); +#ifdef CONFIG_ARM64_PTR_AUTH + DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user)); + DEFINE(THREAD_KEYS_KERNEL, offsetof(struct task_struct, thread.keys_kernel)); +#endif BLANK(); DEFINE(S_X0, offsetof(struct pt_regs, regs[0])); DEFINE(S_X2, offsetof(struct pt_regs, regs[2])); @@ -88,6 +92,9 @@ int main(void) BLANK(); DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack)); DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task)); +#ifdef CONFIG_ARM64_PTR_AUTH + DEFINE(CPU_BOOT_PTRAUTH_KEY, offsetof(struct secondary_data, ptrauth_key)); +#endif BLANK(); #ifdef CONFIG_KVM_ARM_HOST DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt)); @@ -128,5 +135,14 @@ int main(void) DEFINE(SDEI_EVENT_INTREGS, offsetof(struct sdei_registered_event, interrupted_regs)); DEFINE(SDEI_EVENT_PRIORITY, offsetof(struct sdei_registered_event, priority)); #endif +#ifdef CONFIG_ARM64_PTR_AUTH + DEFINE(PTRAUTH_USER_KEY_APIA, offsetof(struct ptrauth_keys_user, apia)); + DEFINE(PTRAUTH_USER_KEY_APIB, offsetof(struct ptrauth_keys_user, apib)); + DEFINE(PTRAUTH_USER_KEY_APDA, offsetof(struct ptrauth_keys_user, apda)); + DEFINE(PTRAUTH_USER_KEY_APDB, offsetof(struct ptrauth_keys_user, apdb)); + DEFINE(PTRAUTH_USER_KEY_APGA, offsetof(struct ptrauth_keys_user, apga)); + DEFINE(PTRAUTH_KERNEL_KEY_APIA, offsetof(struct ptrauth_keys_kernel, apia)); + BLANK(); +#endif return 0; } diff --git a/arch/arm64/kernel/cpu-reset.S b/arch/arm64/kernel/cpu-reset.S index 32c7bf858dd9..38087b4c0432 100644 --- a/arch/arm64/kernel/cpu-reset.S +++ b/arch/arm64/kernel/cpu-reset.S @@ -32,7 +32,7 @@ ENTRY(__cpu_soft_restart) /* Clear sctlr_el1 flags. */ mrs x12, sctlr_el1 - ldr x13, =SCTLR_ELx_FLAGS + mov_q x13, SCTLR_ELx_FLAGS bic x12, x12, x13 pre_disable_mmu_workaround msr sctlr_el1, x12 diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 703ad0a84f99..df56d2295d16 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -11,6 +11,7 @@ #include <asm/cpu.h> #include <asm/cputype.h> #include <asm/cpufeature.h> +#include <asm/kvm_asm.h> #include <asm/smp_plat.h> static bool __maybe_unused @@ -113,13 +114,10 @@ atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1); DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); #ifdef CONFIG_KVM_INDIRECT_VECTORS -extern char __smccc_workaround_1_smc_start[]; -extern char __smccc_workaround_1_smc_end[]; - static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, const char *hyp_vecs_end) { - void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K); + void *dst = lm_alias(__bp_harden_hyp_vecs + slot * SZ_2K); int i; for (i = 0; i < SZ_2K; i += 0x80) @@ -163,9 +161,6 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn, raw_spin_unlock(&bp_lock); } #else -#define __smccc_workaround_1_smc_start NULL -#define __smccc_workaround_1_smc_end NULL - static void install_bp_hardening_cb(bp_hardening_cb_t fn, const char *hyp_vecs_start, const char *hyp_vecs_end) @@ -176,7 +171,7 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn, #include <linux/arm-smccc.h> -static void call_smc_arch_workaround_1(void) +static void __maybe_unused call_smc_arch_workaround_1(void) { arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } @@ -239,11 +234,14 @@ static int detect_harden_bp_fw(void) smccc_end = NULL; break; +#if IS_ENABLED(CONFIG_KVM_ARM_HOST) case SMCCC_CONDUIT_SMC: cb = call_smc_arch_workaround_1; - smccc_start = __smccc_workaround_1_smc_start; - smccc_end = __smccc_workaround_1_smc_end; + smccc_start = __smccc_workaround_1_smc; + smccc_end = __smccc_workaround_1_smc + + __SMCCC_WORKAROUND_1_SMC_SZ; break; +#endif default: return -1; diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index 7e07072757af..e133011f64b5 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -15,10 +15,12 @@ #include <asm/smp_plat.h> extern const struct cpu_operations smp_spin_table_ops; +#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL extern const struct cpu_operations acpi_parking_protocol_ops; +#endif extern const struct cpu_operations cpu_psci_ops; -const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; +static const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; static const struct cpu_operations *const dt_supported_cpu_ops[] __initconst = { &smp_spin_table_ops, @@ -94,7 +96,7 @@ static const char *__init cpu_read_enable_method(int cpu) /* * Read a cpu's enable method and record it in cpu_ops. */ -int __init cpu_read_ops(int cpu) +int __init init_cpu_ops(int cpu) { const char *enable_method = cpu_read_enable_method(cpu); @@ -109,3 +111,8 @@ int __init cpu_read_ops(int cpu) return 0; } + +const struct cpu_operations *get_cpu_ops(int cpu) +{ + return cpu_ops[cpu]; +} diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 0b6715625cf6..9fac745aa7bb 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -116,6 +116,8 @@ cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused) static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap); +static bool __system_matches_cap(unsigned int n); + /* * NOTE: Any changes to the visibility of features should be kept in * sync with the documentation of the CPU feature register ABI. @@ -163,6 +165,7 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_DIT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_AMU_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0), @@ -551,7 +554,7 @@ static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new) BUG_ON(!reg); - for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { + for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { u64 ftr_mask = arm64_ftr_mask(ftrp); s64 ftr_new = arm64_ftr_value(ftrp, new); @@ -1222,6 +1225,57 @@ static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap, #endif +#ifdef CONFIG_ARM64_AMU_EXTN + +/* + * The "amu_cpus" cpumask only signals that the CPU implementation for the + * flagged CPUs supports the Activity Monitors Unit (AMU) but does not provide + * information regarding all the events that it supports. When a CPU bit is + * set in the cpumask, the user of this feature can only rely on the presence + * of the 4 fixed counters for that CPU. But this does not guarantee that the + * counters are enabled or access to these counters is enabled by code + * executed at higher exception levels (firmware). + */ +static struct cpumask amu_cpus __read_mostly; + +bool cpu_has_amu_feat(int cpu) +{ + return cpumask_test_cpu(cpu, &amu_cpus); +} + +/* Initialize the use of AMU counters for frequency invariance */ +extern void init_cpu_freq_invariance_counters(void); + +static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap) +{ + if (has_cpuid_feature(cap, SCOPE_LOCAL_CPU)) { + pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n", + smp_processor_id()); + cpumask_set_cpu(smp_processor_id(), &amu_cpus); + init_cpu_freq_invariance_counters(); + } +} + +static bool has_amu(const struct arm64_cpu_capabilities *cap, + int __unused) +{ + /* + * The AMU extension is a non-conflicting feature: the kernel can + * safely run a mix of CPUs with and without support for the + * activity monitors extension. Therefore, unconditionally enable + * the capability to allow any late CPU to use the feature. + * + * With this feature unconditionally enabled, the cpu_enable + * function will be called for all CPUs that match the criteria, + * including secondary and hotplugged, marking this feature as + * present on that respective CPU. The enable function will also + * print a detection message. + */ + + return true; +} +#endif + #ifdef CONFIG_ARM64_VHE static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused) { @@ -1316,10 +1370,18 @@ static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused) #endif /* CONFIG_ARM64_RAS_EXTN */ #ifdef CONFIG_ARM64_PTR_AUTH -static void cpu_enable_address_auth(struct arm64_cpu_capabilities const *cap) +static bool has_address_auth(const struct arm64_cpu_capabilities *entry, + int __unused) { - sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | - SCTLR_ELx_ENDA | SCTLR_ELx_ENDB); + return __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_ARCH) || + __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF); +} + +static bool has_generic_auth(const struct arm64_cpu_capabilities *entry, + int __unused) +{ + return __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH) || + __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF); } #endif /* CONFIG_ARM64_PTR_AUTH */ @@ -1347,6 +1409,25 @@ static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry, } #endif +/* Internal helper functions to match cpu capability type */ +static bool +cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap) +{ + return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU); +} + +static bool +cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap) +{ + return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU); +} + +static bool +cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) +{ + return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT); +} + static const struct arm64_cpu_capabilities arm64_features[] = { { .desc = "GIC system register CPU interface", @@ -1499,6 +1580,24 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_clear_disr, }, #endif /* CONFIG_ARM64_RAS_EXTN */ +#ifdef CONFIG_ARM64_AMU_EXTN + { + /* + * The feature is enabled by default if CONFIG_ARM64_AMU_EXTN=y. + * Therefore, don't provide .desc as we don't want the detection + * message to be shown until at least one CPU is detected to + * support the feature. + */ + .capability = ARM64_HAS_AMU_EXTN, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + .matches = has_amu, + .sys_reg = SYS_ID_AA64PFR0_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64PFR0_AMU_SHIFT, + .min_field_value = ID_AA64PFR0_AMU, + .cpu_enable = cpu_amu_enable, + }, +#endif /* CONFIG_ARM64_AMU_EXTN */ { .desc = "Data cache clean to the PoU not required for I/D coherence", .capability = ARM64_HAS_CACHE_IDC, @@ -1592,24 +1691,27 @@ static const struct arm64_cpu_capabilities arm64_features[] = { { .desc = "Address authentication (architected algorithm)", .capability = ARM64_HAS_ADDRESS_AUTH_ARCH, - .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_APA_SHIFT, .min_field_value = ID_AA64ISAR1_APA_ARCHITECTED, .matches = has_cpuid_feature, - .cpu_enable = cpu_enable_address_auth, }, { .desc = "Address authentication (IMP DEF algorithm)", .capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF, - .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, .sys_reg = SYS_ID_AA64ISAR1_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64ISAR1_API_SHIFT, .min_field_value = ID_AA64ISAR1_API_IMP_DEF, .matches = has_cpuid_feature, - .cpu_enable = cpu_enable_address_auth, + }, + { + .capability = ARM64_HAS_ADDRESS_AUTH, + .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, + .matches = has_address_auth, }, { .desc = "Generic authentication (architected algorithm)", @@ -1631,6 +1733,11 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .min_field_value = ID_AA64ISAR1_GPI_IMP_DEF, .matches = has_cpuid_feature, }, + { + .capability = ARM64_HAS_GENERIC_AUTH, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_generic_auth, + }, #endif /* CONFIG_ARM64_PTR_AUTH */ #ifdef CONFIG_ARM64_PSEUDO_NMI { @@ -1980,10 +2087,8 @@ static void __init enable_cpu_capabilities(u16 scope_mask) * Run through the list of capabilities to check for conflicts. * If the system has already detected a capability, take necessary * action on this CPU. - * - * Returns "false" on conflicts. */ -static bool verify_local_cpu_caps(u16 scope_mask) +static void verify_local_cpu_caps(u16 scope_mask) { int i; bool cpu_has_cap, system_has_cap; @@ -2028,10 +2133,12 @@ static bool verify_local_cpu_caps(u16 scope_mask) pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n", smp_processor_id(), caps->capability, caps->desc, system_has_cap, cpu_has_cap); - return false; - } - return true; + if (cpucap_panic_on_conflict(caps)) + cpu_panic_kernel(); + else + cpu_die_early(); + } } /* @@ -2041,12 +2148,8 @@ static bool verify_local_cpu_caps(u16 scope_mask) static void check_early_cpu_features(void) { verify_cpu_asid_bits(); - /* - * Early features are used by the kernel already. If there - * is a conflict, we cannot proceed further. - */ - if (!verify_local_cpu_caps(SCOPE_BOOT_CPU)) - cpu_panic_kernel(); + + verify_local_cpu_caps(SCOPE_BOOT_CPU); } static void @@ -2094,8 +2197,7 @@ static void verify_local_cpu_capabilities(void) * check_early_cpu_features(), as they need to be verified * on all secondary CPUs. */ - if (!verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU)) - cpu_die_early(); + verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU); verify_local_elf_hwcaps(arm64_elf_hwcaps); @@ -2146,6 +2248,23 @@ bool this_cpu_has_cap(unsigned int n) return false; } +/* + * This helper function is used in a narrow window when, + * - The system wide safe registers are set with all the SMP CPUs and, + * - The SYSTEM_FEATURE cpu_hwcaps may not have been set. + * In all other cases cpus_have_{const_}cap() should be used. + */ +static bool __system_matches_cap(unsigned int n) +{ + if (n < ARM64_NCAPS) { + const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n]; + + if (cap) + return cap->matches(cap, SCOPE_SYSTEM); + } + return false; +} + void cpu_set_feature(unsigned int num) { WARN_ON(num >= MAX_CPU_FEATURES); @@ -2218,7 +2337,7 @@ void __init setup_cpu_features(void) static bool __maybe_unused cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused) { - return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO)); + return (__system_matches_cap(ARM64_HAS_PAN) && !__system_matches_cap(ARM64_HAS_UAO)); } static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap) diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index e4d6af2fdec7..b512b5503f6e 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -18,11 +18,11 @@ int arm_cpuidle_init(unsigned int cpu) { + const struct cpu_operations *ops = get_cpu_ops(cpu); int ret = -EOPNOTSUPP; - if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_suspend && - cpu_ops[cpu]->cpu_init_idle) - ret = cpu_ops[cpu]->cpu_init_idle(cpu); + if (ops && ops->cpu_suspend && ops->cpu_init_idle) + ret = ops->cpu_init_idle(cpu); return ret; } @@ -37,8 +37,9 @@ int arm_cpuidle_init(unsigned int cpu) int arm_cpuidle_suspend(int index) { int cpu = smp_processor_id(); + const struct cpu_operations *ops = get_cpu_ops(cpu); - return cpu_ops[cpu]->cpu_suspend(index); + return ops->cpu_suspend(index); } #ifdef CONFIG_ACPI diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S index 304d5b02ca67..1a03618df0df 100644 --- a/arch/arm64/kernel/efi-entry.S +++ b/arch/arm64/kernel/efi-entry.S @@ -10,81 +10,35 @@ #include <asm/assembler.h> -#define EFI_LOAD_ERROR 0x8000000000000001 - __INIT - /* - * We arrive here from the EFI boot manager with: - * - * * CPU in little-endian mode - * * MMU on with identity-mapped RAM - * * Icache and Dcache on - * - * We will most likely be running from some place other than where - * we want to be. The kernel image wants to be placed at TEXT_OFFSET - * from start of RAM. - */ -ENTRY(entry) - /* - * Create a stack frame to save FP/LR with extra space - * for image_addr variable passed to efi_entry(). - */ - stp x29, x30, [sp, #-32]! - mov x29, sp - - /* - * Call efi_entry to do the real work. - * x0 and x1 are already set up by firmware. Current runtime - * address of image is calculated and passed via *image_addr. - * - * unsigned long efi_entry(void *handle, - * efi_system_table_t *sys_table, - * unsigned long *image_addr) ; - */ - adr_l x8, _text - add x2, sp, 16 - str x8, [x2] - bl efi_entry - cmn x0, #1 - b.eq efi_load_fail - +SYM_CODE_START(efi_enter_kernel) /* * efi_entry() will have copied the kernel image if necessary and we - * return here with device tree address in x0 and the kernel entry - * point stored at *image_addr. Save those values in registers which - * are callee preserved. - */ - mov x20, x0 // DTB address - ldr x0, [sp, #16] // relocated _text address - ldr w21, =stext_offset - add x21, x0, x21 - - /* - * Calculate size of the kernel Image (same for original and copy). + * end up here with device tree address in x1 and the kernel entry + * point stored in x0. Save those values in registers which are + * callee preserved. */ - adr_l x1, _text - adr_l x2, _edata - sub x1, x2, x1 + ldr w2, =stext_offset + add x19, x0, x2 // relocated Image entrypoint + mov x20, x1 // DTB address /* - * Flush the copied Image to the PoC, and ensure it is not shadowed by + * Clean the copied Image to the PoC, and ensure it is not shadowed by * stale icache entries from before relocation. */ - bl __flush_dcache_area + ldr w1, =kernel_size + bl __clean_dcache_area_poc ic ialluis /* - * Ensure that the rest of this function (in the original Image) is - * visible when the caches are disabled. The I-cache can't have stale - * entries for the VA range of the current image, so no maintenance is - * necessary. + * Clean the remainder of this routine to the PoC + * so that we can safely disable the MMU and caches. */ - adr x0, entry - adr x1, entry_end - sub x1, x1, x0 - bl __flush_dcache_area - + adr x0, 0f + ldr w1, 3f + bl __clean_dcache_area_poc +0: /* Turn off Dcache and MMU */ mrs x0, CurrentEL cmp x0, #CurrentEL_EL2 @@ -109,12 +63,6 @@ ENTRY(entry) mov x1, xzr mov x2, xzr mov x3, xzr - br x21 - -efi_load_fail: - mov x0, #EFI_LOAD_ERROR - ldp x29, x30, [sp], #32 - ret - -entry_end: -ENDPROC(entry) + br x19 +SYM_CODE_END(efi_enter_kernel) +3: .long . - 0b diff --git a/arch/arm64/kernel/efi-header.S b/arch/arm64/kernel/efi-header.S index a7cfacce3e15..914999ccaf8a 100644 --- a/arch/arm64/kernel/efi-header.S +++ b/arch/arm64/kernel/efi-header.S @@ -27,7 +27,7 @@ optional_header: .long __initdata_begin - efi_header_end // SizeOfCode .long __pecoff_data_size // SizeOfInitializedData .long 0 // SizeOfUninitializedData - .long __efistub_entry - _head // AddressOfEntryPoint + .long __efistub_efi_entry - _head // AddressOfEntryPoint .long efi_header_end - _head // BaseOfCode extra_header_fields: @@ -36,8 +36,8 @@ extra_header_fields: .long PECOFF_FILE_ALIGNMENT // FileAlignment .short 0 // MajorOperatingSystemVersion .short 0 // MinorOperatingSystemVersion - .short 0 // MajorImageVersion - .short 0 // MinorImageVersion + .short LINUX_EFISTUB_MAJOR_VERSION // MajorImageVersion + .short LINUX_EFISTUB_MINOR_VERSION // MinorImageVersion .short 0 // MajorSubsystemVersion .short 0 // MinorSubsystemVersion .long 0 // Win32VersionValue diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index fde59981445c..c839b5bf1904 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -175,7 +175,7 @@ NOKPROBE_SYMBOL(el0_pc); static void notrace el0_sp(struct pt_regs *regs, unsigned long esr) { user_exit_irqoff(); - local_daif_restore(DAIF_PROCCTX_NOIRQ); + local_daif_restore(DAIF_PROCCTX); do_sp_pc_abort(regs->sp, esr, regs); } NOKPROBE_SYMBOL(el0_sp); diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 7d02f9966d34..833d48c9acb5 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -75,27 +75,27 @@ add x29, sp, #S_STACKFRAME .endm -ENTRY(ftrace_regs_caller) +SYM_CODE_START(ftrace_regs_caller) ftrace_regs_entry 1 b ftrace_common -ENDPROC(ftrace_regs_caller) +SYM_CODE_END(ftrace_regs_caller) -ENTRY(ftrace_caller) +SYM_CODE_START(ftrace_caller) ftrace_regs_entry 0 b ftrace_common -ENDPROC(ftrace_caller) +SYM_CODE_END(ftrace_caller) -ENTRY(ftrace_common) +SYM_CODE_START(ftrace_common) sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn) mov x1, x9 // parent_ip (callsite's LR) ldr_l x2, function_trace_op // op mov x3, sp // regs -GLOBAL(ftrace_call) +SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) bl ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER -GLOBAL(ftrace_graph_call) // ftrace_graph_caller(); +SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); nop // If enabled, this will be replaced // "b ftrace_graph_caller" #endif @@ -122,17 +122,17 @@ ftrace_common_return: add sp, sp, #S_FRAME_SIZE + 16 ret x9 -ENDPROC(ftrace_common) +SYM_CODE_END(ftrace_common) #ifdef CONFIG_FUNCTION_GRAPH_TRACER -ENTRY(ftrace_graph_caller) +SYM_CODE_START(ftrace_graph_caller) ldr x0, [sp, #S_PC] sub x0, x0, #AARCH64_INSN_SIZE // ip (callsite's BL insn) add x1, sp, #S_LR // parent_ip (callsite's LR) ldr x2, [sp, #S_FRAME_SIZE] // parent fp (callsite's FP) bl prepare_ftrace_return b ftrace_common_return -ENDPROC(ftrace_graph_caller) +SYM_CODE_END(ftrace_graph_caller) #endif #else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ @@ -218,7 +218,7 @@ ENDPROC(ftrace_graph_caller) * - tracer function to probe instrumented function's entry, * - ftrace_graph_caller to set up an exit hook */ -ENTRY(_mcount) +SYM_FUNC_START(_mcount) mcount_enter ldr_l x2, ftrace_trace_function @@ -242,7 +242,7 @@ skip_ftrace_call: // } b.ne ftrace_graph_caller // ftrace_graph_caller(); #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ mcount_exit -ENDPROC(_mcount) +SYM_FUNC_END(_mcount) EXPORT_SYMBOL(_mcount) NOKPROBE(_mcount) @@ -253,9 +253,9 @@ NOKPROBE(_mcount) * and later on, NOP to branch to ftrace_caller() when enabled or branch to * NOP when disabled per-function base. */ -ENTRY(_mcount) +SYM_FUNC_START(_mcount) ret -ENDPROC(_mcount) +SYM_FUNC_END(_mcount) EXPORT_SYMBOL(_mcount) NOKPROBE(_mcount) @@ -268,24 +268,24 @@ NOKPROBE(_mcount) * - tracer function to probe instrumented function's entry, * - ftrace_graph_caller to set up an exit hook */ -ENTRY(ftrace_caller) +SYM_FUNC_START(ftrace_caller) mcount_enter mcount_get_pc0 x0 // function's pc mcount_get_lr x1 // function's lr -GLOBAL(ftrace_call) // tracer(pc, lr); +SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr); nop // This will be replaced with "bl xxx" // where xxx can be any kind of tracer. #ifdef CONFIG_FUNCTION_GRAPH_TRACER -GLOBAL(ftrace_graph_call) // ftrace_graph_caller(); +SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); nop // If enabled, this will be replaced // "b ftrace_graph_caller" #endif mcount_exit -ENDPROC(ftrace_caller) +SYM_FUNC_END(ftrace_caller) #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER @@ -298,20 +298,20 @@ ENDPROC(ftrace_caller) * the call stack in order to intercept instrumented function's return path * and run return_to_handler() later on its exit. */ -ENTRY(ftrace_graph_caller) +SYM_FUNC_START(ftrace_graph_caller) mcount_get_pc x0 // function's pc mcount_get_lr_addr x1 // pointer to function's saved lr mcount_get_parent_fp x2 // parent's fp bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp) mcount_exit -ENDPROC(ftrace_graph_caller) +SYM_FUNC_END(ftrace_graph_caller) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ -ENTRY(ftrace_stub) +SYM_FUNC_START(ftrace_stub) ret -ENDPROC(ftrace_stub) +SYM_FUNC_END(ftrace_stub) #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* @@ -320,7 +320,7 @@ ENDPROC(ftrace_stub) * Run ftrace_return_to_handler() before going back to parent. * @fp is checked against the value passed by ftrace_graph_caller(). */ -ENTRY(return_to_handler) +SYM_CODE_START(return_to_handler) /* save return value regs */ sub sp, sp, #64 stp x0, x1, [sp] @@ -340,5 +340,5 @@ ENTRY(return_to_handler) add sp, sp, #64 ret -END(return_to_handler) +SYM_CODE_END(return_to_handler) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 9461d812ae27..ddcde093c433 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -14,6 +14,7 @@ #include <asm/alternative.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> +#include <asm/asm_pointer_auth.h> #include <asm/cpufeature.h> #include <asm/errno.h> #include <asm/esr.h> @@ -177,6 +178,7 @@ alternative_cb_end apply_ssbd 1, x22, x23 + ptrauth_keys_install_kernel tsk, 1, x20, x22, x23 .else add x21, sp, #S_FRAME_SIZE get_current_task tsk @@ -341,6 +343,9 @@ alternative_else_nop_endif msr cntkctl_el1, x1 4: #endif + /* No kernel C function calls after this as user keys are set. */ + ptrauth_keys_install_user tsk, x0, x1, x2 + apply_ssbd 0, x0, x1 .endif @@ -465,7 +470,7 @@ alternative_endif .pushsection ".entry.text", "ax" .align 11 -ENTRY(vectors) +SYM_CODE_START(vectors) kernel_ventry 1, sync_invalid // Synchronous EL1t kernel_ventry 1, irq_invalid // IRQ EL1t kernel_ventry 1, fiq_invalid // FIQ EL1t @@ -492,7 +497,7 @@ ENTRY(vectors) kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0 kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0 #endif -END(vectors) +SYM_CODE_END(vectors) #ifdef CONFIG_VMAP_STACK /* @@ -534,57 +539,57 @@ __bad_stack: ASM_BUG() .endm -el0_sync_invalid: +SYM_CODE_START_LOCAL(el0_sync_invalid) inv_entry 0, BAD_SYNC -ENDPROC(el0_sync_invalid) +SYM_CODE_END(el0_sync_invalid) -el0_irq_invalid: +SYM_CODE_START_LOCAL(el0_irq_invalid) inv_entry 0, BAD_IRQ -ENDPROC(el0_irq_invalid) +SYM_CODE_END(el0_irq_invalid) -el0_fiq_invalid: +SYM_CODE_START_LOCAL(el0_fiq_invalid) inv_entry 0, BAD_FIQ -ENDPROC(el0_fiq_invalid) +SYM_CODE_END(el0_fiq_invalid) -el0_error_invalid: +SYM_CODE_START_LOCAL(el0_error_invalid) inv_entry 0, BAD_ERROR -ENDPROC(el0_error_invalid) +SYM_CODE_END(el0_error_invalid) #ifdef CONFIG_COMPAT -el0_fiq_invalid_compat: +SYM_CODE_START_LOCAL(el0_fiq_invalid_compat) inv_entry 0, BAD_FIQ, 32 -ENDPROC(el0_fiq_invalid_compat) +SYM_CODE_END(el0_fiq_invalid_compat) #endif -el1_sync_invalid: +SYM_CODE_START_LOCAL(el1_sync_invalid) inv_entry 1, BAD_SYNC -ENDPROC(el1_sync_invalid) +SYM_CODE_END(el1_sync_invalid) -el1_irq_invalid: +SYM_CODE_START_LOCAL(el1_irq_invalid) inv_entry 1, BAD_IRQ -ENDPROC(el1_irq_invalid) +SYM_CODE_END(el1_irq_invalid) -el1_fiq_invalid: +SYM_CODE_START_LOCAL(el1_fiq_invalid) inv_entry 1, BAD_FIQ -ENDPROC(el1_fiq_invalid) +SYM_CODE_END(el1_fiq_invalid) -el1_error_invalid: +SYM_CODE_START_LOCAL(el1_error_invalid) inv_entry 1, BAD_ERROR -ENDPROC(el1_error_invalid) +SYM_CODE_END(el1_error_invalid) /* * EL1 mode handlers. */ .align 6 -el1_sync: +SYM_CODE_START_LOCAL_NOALIGN(el1_sync) kernel_entry 1 mov x0, sp bl el1_sync_handler kernel_exit 1 -ENDPROC(el1_sync) +SYM_CODE_END(el1_sync) .align 6 -el1_irq: +SYM_CODE_START_LOCAL_NOALIGN(el1_irq) kernel_entry 1 gic_prio_irq_setup pmr=x20, tmp=x1 enable_da_f @@ -639,42 +644,42 @@ alternative_else_nop_endif #endif kernel_exit 1 -ENDPROC(el1_irq) +SYM_CODE_END(el1_irq) /* * EL0 mode handlers. */ .align 6 -el0_sync: +SYM_CODE_START_LOCAL_NOALIGN(el0_sync) kernel_entry 0 mov x0, sp bl el0_sync_handler b ret_to_user -ENDPROC(el0_sync) +SYM_CODE_END(el0_sync) #ifdef CONFIG_COMPAT .align 6 -el0_sync_compat: +SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat) kernel_entry 0, 32 mov x0, sp bl el0_sync_compat_handler b ret_to_user -ENDPROC(el0_sync_compat) +SYM_CODE_END(el0_sync_compat) .align 6 -el0_irq_compat: +SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat) kernel_entry 0, 32 b el0_irq_naked -ENDPROC(el0_irq_compat) +SYM_CODE_END(el0_irq_compat) -el0_error_compat: +SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat) kernel_entry 0, 32 b el0_error_naked -ENDPROC(el0_error_compat) +SYM_CODE_END(el0_error_compat) #endif .align 6 -el0_irq: +SYM_CODE_START_LOCAL_NOALIGN(el0_irq) kernel_entry 0 el0_irq_naked: gic_prio_irq_setup pmr=x20, tmp=x0 @@ -696,9 +701,9 @@ el0_irq_naked: bl trace_hardirqs_on #endif b ret_to_user -ENDPROC(el0_irq) +SYM_CODE_END(el0_irq) -el1_error: +SYM_CODE_START_LOCAL(el1_error) kernel_entry 1 mrs x1, esr_el1 gic_prio_kentry_setup tmp=x2 @@ -706,9 +711,9 @@ el1_error: mov x0, sp bl do_serror kernel_exit 1 -ENDPROC(el1_error) +SYM_CODE_END(el1_error) -el0_error: +SYM_CODE_START_LOCAL(el0_error) kernel_entry 0 el0_error_naked: mrs x25, esr_el1 @@ -720,7 +725,7 @@ el0_error_naked: bl do_serror enable_da_f b ret_to_user -ENDPROC(el0_error) +SYM_CODE_END(el0_error) /* * Ok, we need to do extra processing, enter the slow path. @@ -832,7 +837,7 @@ alternative_else_nop_endif .endm .align 11 -ENTRY(tramp_vectors) +SYM_CODE_START_NOALIGN(tramp_vectors) .space 0x400 tramp_ventry @@ -844,24 +849,24 @@ ENTRY(tramp_vectors) tramp_ventry 32 tramp_ventry 32 tramp_ventry 32 -END(tramp_vectors) +SYM_CODE_END(tramp_vectors) -ENTRY(tramp_exit_native) +SYM_CODE_START(tramp_exit_native) tramp_exit -END(tramp_exit_native) +SYM_CODE_END(tramp_exit_native) -ENTRY(tramp_exit_compat) +SYM_CODE_START(tramp_exit_compat) tramp_exit 32 -END(tramp_exit_compat) +SYM_CODE_END(tramp_exit_compat) .ltorg .popsection // .entry.tramp.text #ifdef CONFIG_RANDOMIZE_BASE .pushsection ".rodata", "a" .align PAGE_SHIFT - .globl __entry_tramp_data_start -__entry_tramp_data_start: +SYM_DATA_START(__entry_tramp_data_start) .quad vectors +SYM_DATA_END(__entry_tramp_data_start) .popsection // .rodata #endif /* CONFIG_RANDOMIZE_BASE */ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ @@ -874,7 +879,7 @@ __entry_tramp_data_start: * Previous and next are guaranteed not to be the same. * */ -ENTRY(cpu_switch_to) +SYM_FUNC_START(cpu_switch_to) mov x10, #THREAD_CPU_CONTEXT add x8, x0, x10 mov x9, sp @@ -895,21 +900,22 @@ ENTRY(cpu_switch_to) ldr lr, [x8] mov sp, x9 msr sp_el0, x1 + ptrauth_keys_install_kernel x1, 1, x8, x9, x10 ret -ENDPROC(cpu_switch_to) +SYM_FUNC_END(cpu_switch_to) NOKPROBE(cpu_switch_to) /* * This is how we return from a fork. */ -ENTRY(ret_from_fork) +SYM_CODE_START(ret_from_fork) bl schedule_tail cbz x19, 1f // not a kernel thread mov x0, x20 blr x19 1: get_current_task tsk b ret_to_user -ENDPROC(ret_from_fork) +SYM_CODE_END(ret_from_fork) NOKPROBE(ret_from_fork) #ifdef CONFIG_ARM_SDE_INTERFACE @@ -938,7 +944,7 @@ NOKPROBE(ret_from_fork) */ .ltorg .pushsection ".entry.tramp.text", "ax" -ENTRY(__sdei_asm_entry_trampoline) +SYM_CODE_START(__sdei_asm_entry_trampoline) mrs x4, ttbr1_el1 tbz x4, #USER_ASID_BIT, 1f @@ -960,7 +966,7 @@ ENTRY(__sdei_asm_entry_trampoline) ldr x4, =__sdei_asm_handler #endif br x4 -ENDPROC(__sdei_asm_entry_trampoline) +SYM_CODE_END(__sdei_asm_entry_trampoline) NOKPROBE(__sdei_asm_entry_trampoline) /* @@ -970,21 +976,22 @@ NOKPROBE(__sdei_asm_entry_trampoline) * x2: exit_mode * x4: struct sdei_registered_event argument from registration time. */ -ENTRY(__sdei_asm_exit_trampoline) +SYM_CODE_START(__sdei_asm_exit_trampoline) ldr x4, [x4, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)] cbnz x4, 1f tramp_unmap_kernel tmp=x4 1: sdei_handler_exit exit_mode=x2 -ENDPROC(__sdei_asm_exit_trampoline) +SYM_CODE_END(__sdei_asm_exit_trampoline) NOKPROBE(__sdei_asm_exit_trampoline) .ltorg .popsection // .entry.tramp.text #ifdef CONFIG_RANDOMIZE_BASE .pushsection ".rodata", "a" -__sdei_asm_trampoline_next_handler: +SYM_DATA_START(__sdei_asm_trampoline_next_handler) .quad __sdei_asm_handler +SYM_DATA_END(__sdei_asm_trampoline_next_handler) .popsection // .rodata #endif /* CONFIG_RANDOMIZE_BASE */ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ @@ -1002,7 +1009,7 @@ __sdei_asm_trampoline_next_handler: * follow SMC-CC. We save (or retrieve) all the registers as the handler may * want them. */ -ENTRY(__sdei_asm_handler) +SYM_CODE_START(__sdei_asm_handler) stp x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC] stp x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2] stp x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3] @@ -1085,6 +1092,6 @@ alternative_else_nop_endif tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline br x5 #endif -ENDPROC(__sdei_asm_handler) +SYM_CODE_END(__sdei_asm_handler) NOKPROBE(__sdei_asm_handler) #endif /* CONFIG_ARM_SDE_INTERFACE */ diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 989b1944cb71..57a91032b4c2 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -105,7 +105,7 @@ pe_header: * x24 __primary_switch() .. relocate_kernel() * current RELR displacement */ -ENTRY(stext) +SYM_CODE_START(stext) bl preserve_boot_args bl el2_setup // Drop to EL1, w0=cpu_boot_mode adrp x23, __PHYS_OFFSET @@ -118,14 +118,15 @@ ENTRY(stext) * On return, the CPU will be ready for the MMU to be turned on and * the TCR will have been set. */ + mov x0, #ARM64_CPU_BOOT_PRIMARY bl __cpu_setup // initialise processor b __primary_switch -ENDPROC(stext) +SYM_CODE_END(stext) /* * Preserve the arguments passed by the bootloader in x0 .. x3 */ -preserve_boot_args: +SYM_CODE_START_LOCAL(preserve_boot_args) mov x21, x0 // x21=FDT adr_l x0, boot_args // record the contents of @@ -137,7 +138,7 @@ preserve_boot_args: mov x1, #0x20 // 4 x 8 bytes b __inval_dcache_area // tail call -ENDPROC(preserve_boot_args) +SYM_CODE_END(preserve_boot_args) /* * Macro to create a table entry to the next page. @@ -275,7 +276,7 @@ ENDPROC(preserve_boot_args) * - first few MB of the kernel linear mapping to jump to once the MMU has * been enabled */ -__create_page_tables: +SYM_FUNC_START_LOCAL(__create_page_tables) mov x28, lr /* @@ -403,15 +404,14 @@ __create_page_tables: bl __inval_dcache_area ret x28 -ENDPROC(__create_page_tables) - .ltorg +SYM_FUNC_END(__create_page_tables) /* * The following fragment of code is executed with the MMU enabled. * * x0 = __PHYS_OFFSET */ -__primary_switched: +SYM_FUNC_START_LOCAL(__primary_switched) adrp x4, init_thread_union add sp, x4, #THREAD_SIZE adr_l x5, init_task @@ -456,7 +456,14 @@ __primary_switched: mov x29, #0 mov x30, #0 b start_kernel -ENDPROC(__primary_switched) +SYM_FUNC_END(__primary_switched) + + .pushsection ".rodata", "a" +SYM_DATA_START(kimage_vaddr) + .quad _text - TEXT_OFFSET +SYM_DATA_END(kimage_vaddr) +EXPORT_SYMBOL(kimage_vaddr) + .popsection /* * end early head section, begin head code that is also used for @@ -464,10 +471,6 @@ ENDPROC(__primary_switched) */ .section ".idmap.text","awx" -ENTRY(kimage_vaddr) - .quad _text - TEXT_OFFSET -EXPORT_SYMBOL(kimage_vaddr) - /* * If we're fortunate enough to boot at EL2, ensure that the world is * sane before dropping to EL1. @@ -475,7 +478,7 @@ EXPORT_SYMBOL(kimage_vaddr) * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in w0 if * booted in EL1 or EL2 respectively. */ -ENTRY(el2_setup) +SYM_FUNC_START(el2_setup) msr SPsel, #1 // We want to use SP_EL{1,2} mrs x0, CurrentEL cmp x0, #CurrentEL_EL2 @@ -599,7 +602,7 @@ set_hcr: isb ret -install_el2_stub: +SYM_INNER_LABEL(install_el2_stub, SYM_L_LOCAL) /* * When VHE is not in use, early init of EL2 and EL1 needs to be * done here. @@ -636,13 +639,13 @@ install_el2_stub: msr elr_el2, lr mov w0, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2 eret -ENDPROC(el2_setup) +SYM_FUNC_END(el2_setup) /* * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed * in w0. See arch/arm64/include/asm/virt.h for more info. */ -set_cpu_boot_mode_flag: +SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) adr_l x1, __boot_cpu_mode cmp w0, #BOOT_CPU_MODE_EL2 b.ne 1f @@ -651,7 +654,7 @@ set_cpu_boot_mode_flag: dmb sy dc ivac, x1 // Invalidate potentially stale cache line ret -ENDPROC(set_cpu_boot_mode_flag) +SYM_FUNC_END(set_cpu_boot_mode_flag) /* * These values are written with the MMU off, but read with the MMU on. @@ -667,15 +670,17 @@ ENDPROC(set_cpu_boot_mode_flag) * This is not in .bss, because we set it sufficiently early that the boot-time * zeroing of .bss would clobber it. */ -ENTRY(__boot_cpu_mode) +SYM_DATA_START(__boot_cpu_mode) .long BOOT_CPU_MODE_EL2 .long BOOT_CPU_MODE_EL1 +SYM_DATA_END(__boot_cpu_mode) /* * The booting CPU updates the failed status @__early_cpu_boot_status, * with MMU turned off. */ -ENTRY(__early_cpu_boot_status) +SYM_DATA_START(__early_cpu_boot_status) .quad 0 +SYM_DATA_END(__early_cpu_boot_status) .popsection @@ -683,7 +688,7 @@ ENTRY(__early_cpu_boot_status) * This provides a "holding pen" for platforms to hold all secondary * cores are held until we're ready for them to initialise. */ -ENTRY(secondary_holding_pen) +SYM_FUNC_START(secondary_holding_pen) bl el2_setup // Drop to EL1, w0=cpu_boot_mode bl set_cpu_boot_mode_flag mrs x0, mpidr_el1 @@ -695,31 +700,32 @@ pen: ldr x4, [x3] b.eq secondary_startup wfe b pen -ENDPROC(secondary_holding_pen) +SYM_FUNC_END(secondary_holding_pen) /* * Secondary entry point that jumps straight into the kernel. Only to * be used where CPUs are brought online dynamically by the kernel. */ -ENTRY(secondary_entry) +SYM_FUNC_START(secondary_entry) bl el2_setup // Drop to EL1 bl set_cpu_boot_mode_flag b secondary_startup -ENDPROC(secondary_entry) +SYM_FUNC_END(secondary_entry) -secondary_startup: +SYM_FUNC_START_LOCAL(secondary_startup) /* * Common entry point for secondary CPUs. */ bl __cpu_secondary_check52bitva + mov x0, #ARM64_CPU_BOOT_SECONDARY bl __cpu_setup // initialise processor adrp x1, swapper_pg_dir bl __enable_mmu ldr x8, =__secondary_switched br x8 -ENDPROC(secondary_startup) +SYM_FUNC_END(secondary_startup) -__secondary_switched: +SYM_FUNC_START_LOCAL(__secondary_switched) adr_l x5, vectors msr vbar_el1, x5 isb @@ -734,13 +740,13 @@ __secondary_switched: mov x29, #0 mov x30, #0 b secondary_start_kernel -ENDPROC(__secondary_switched) +SYM_FUNC_END(__secondary_switched) -__secondary_too_slow: +SYM_FUNC_START_LOCAL(__secondary_too_slow) wfe wfi b __secondary_too_slow -ENDPROC(__secondary_too_slow) +SYM_FUNC_END(__secondary_too_slow) /* * The booting CPU updates the failed status @__early_cpu_boot_status, @@ -772,7 +778,7 @@ ENDPROC(__secondary_too_slow) * Checks if the selected granule size is supported by the CPU. * If it isn't, park the CPU */ -ENTRY(__enable_mmu) +SYM_FUNC_START(__enable_mmu) mrs x2, ID_AA64MMFR0_EL1 ubfx x2, x2, #ID_AA64MMFR0_TGRAN_SHIFT, 4 cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED @@ -796,9 +802,9 @@ ENTRY(__enable_mmu) dsb nsh isb ret -ENDPROC(__enable_mmu) +SYM_FUNC_END(__enable_mmu) -ENTRY(__cpu_secondary_check52bitva) +SYM_FUNC_START(__cpu_secondary_check52bitva) #ifdef CONFIG_ARM64_VA_BITS_52 ldr_l x0, vabits_actual cmp x0, #52 @@ -816,9 +822,9 @@ ENTRY(__cpu_secondary_check52bitva) #endif 2: ret -ENDPROC(__cpu_secondary_check52bitva) +SYM_FUNC_END(__cpu_secondary_check52bitva) -__no_granule_support: +SYM_FUNC_START_LOCAL(__no_granule_support) /* Indicate that this CPU can't boot and is stuck in the kernel */ update_early_cpu_boot_status \ CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2 @@ -826,10 +832,10 @@ __no_granule_support: wfe wfi b 1b -ENDPROC(__no_granule_support) +SYM_FUNC_END(__no_granule_support) #ifdef CONFIG_RELOCATABLE -__relocate_kernel: +SYM_FUNC_START_LOCAL(__relocate_kernel) /* * Iterate over each entry in the relocation table, and apply the * relocations in place. @@ -931,10 +937,10 @@ __relocate_kernel: #endif ret -ENDPROC(__relocate_kernel) +SYM_FUNC_END(__relocate_kernel) #endif -__primary_switch: +SYM_FUNC_START_LOCAL(__primary_switch) #ifdef CONFIG_RANDOMIZE_BASE mov x19, x0 // preserve new SCTLR_EL1 value mrs x20, sctlr_el1 // preserve old SCTLR_EL1 value @@ -977,4 +983,4 @@ __primary_switch: ldr x8, =__primary_switched adrp x0, __PHYS_OFFSET br x8 -ENDPROC(__primary_switch) +SYM_FUNC_END(__primary_switch) diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S index 38bcd4d4e43b..6532105b3e32 100644 --- a/arch/arm64/kernel/hibernate-asm.S +++ b/arch/arm64/kernel/hibernate-asm.S @@ -110,8 +110,6 @@ ENTRY(swsusp_arch_suspend_exit) cbz x24, 3f /* Do we need to re-initialise EL2? */ hvc #0 3: ret - - .ltorg ENDPROC(swsusp_arch_suspend_exit) /* diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 590963c9c609..5b73e92c99e3 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -166,14 +166,11 @@ int arch_hibernation_header_restore(void *addr) sleep_cpu = -EINVAL; return -EINVAL; } - if (!cpu_online(sleep_cpu)) { - pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n"); - ret = cpu_up(sleep_cpu); - if (ret) { - pr_err("Failed to bring hibernate-CPU up!\n"); - sleep_cpu = -EINVAL; - return ret; - } + + ret = bringup_hibernate_cpu(sleep_cpu); + if (ret) { + sleep_cpu = -EINVAL; + return ret; } resume_hdr = *hdr; diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 73d46070b315..e473ead806ed 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -63,7 +63,7 @@ el1_sync: beq 9f // Nothing to reset! /* Someone called kvm_call_hyp() against the hyp-stub... */ - ldr x0, =HVC_STUB_ERR + mov_q x0, HVC_STUB_ERR eret 9: mov x0, xzr diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 25a2a9b479c2..7f06ad93fc95 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -12,7 +12,9 @@ #ifdef CONFIG_EFI -__efistub_stext_offset = stext - _text; +__efistub_kernel_size = _edata - _text; +__efistub_stext_offset = stext - _text; + /* * The EFI stub has its own symbol namespace prefixed by __efistub_, to @@ -33,7 +35,7 @@ __efistub_strnlen = __pi_strnlen; __efistub_strcmp = __pi_strcmp; __efistub_strncmp = __pi_strncmp; __efistub_strrchr = __pi_strrchr; -__efistub___flush_dcache_area = __pi___flush_dcache_area; +__efistub___clean_dcache_area_poc = __pi___clean_dcache_area_poc; #ifdef CONFIG_KASAN __efistub___memcpy = __pi_memcpy; @@ -45,6 +47,7 @@ __efistub__text = _text; __efistub__end = _end; __efistub__edata = _edata; __efistub_screen_info = screen_info; +__efistub__ctype = _ctype; #endif diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index dd3ae8081b38..b40c3b0def92 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -121,7 +121,7 @@ static int setup_dtb(struct kimage *image, /* add kaslr-seed */ ret = fdt_delprop(dtb, off, FDT_PROP_KASLR_SEED); - if (ret == -FDT_ERR_NOTFOUND) + if (ret == -FDT_ERR_NOTFOUND) ret = 0; else if (ret) goto out; diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index e40b65645c86..4d7879484cec 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -285,6 +285,17 @@ static struct attribute_group armv8_pmuv3_format_attr_group = { #define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \ (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1) + +/* + * We unconditionally enable ARMv8.5-PMU long event counter support + * (64-bit events) where supported. Indicate if this arm_pmu has long + * event counter support. + */ +static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu) +{ + return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5); +} + /* * We must chain two programmable counters for 64 bit events, * except when we have allocated the 64bit cycle counter (for CPU @@ -294,9 +305,11 @@ static struct attribute_group armv8_pmuv3_format_attr_group = { static inline bool armv8pmu_event_is_chained(struct perf_event *event) { int idx = event->hw.idx; + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); return !WARN_ON(idx < 0) && armv8pmu_event_is_64bit(event) && + !armv8pmu_has_long_event(cpu_pmu) && (idx != ARMV8_IDX_CYCLE_COUNTER); } @@ -345,7 +358,7 @@ static inline void armv8pmu_select_counter(int idx) isb(); } -static inline u32 armv8pmu_read_evcntr(int idx) +static inline u64 armv8pmu_read_evcntr(int idx) { armv8pmu_select_counter(idx); return read_sysreg(pmxevcntr_el0); @@ -362,6 +375,44 @@ static inline u64 armv8pmu_read_hw_counter(struct perf_event *event) return val; } +/* + * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP + * is set the event counters also become 64-bit counters. Unless the + * user has requested a long counter (attr.config1) then we want to + * interrupt upon 32-bit overflow - we achieve this by applying a bias. + */ +static bool armv8pmu_event_needs_bias(struct perf_event *event) +{ + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + if (armv8pmu_event_is_64bit(event)) + return false; + + if (armv8pmu_has_long_event(cpu_pmu) || + idx == ARMV8_IDX_CYCLE_COUNTER) + return true; + + return false; +} + +static u64 armv8pmu_bias_long_counter(struct perf_event *event, u64 value) +{ + if (armv8pmu_event_needs_bias(event)) + value |= GENMASK(63, 32); + + return value; +} + +static u64 armv8pmu_unbias_long_counter(struct perf_event *event, u64 value) +{ + if (armv8pmu_event_needs_bias(event)) + value &= ~GENMASK(63, 32); + + return value; +} + static u64 armv8pmu_read_counter(struct perf_event *event) { struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); @@ -377,10 +428,10 @@ static u64 armv8pmu_read_counter(struct perf_event *event) else value = armv8pmu_read_hw_counter(event); - return value; + return armv8pmu_unbias_long_counter(event, value); } -static inline void armv8pmu_write_evcntr(int idx, u32 value) +static inline void armv8pmu_write_evcntr(int idx, u64 value) { armv8pmu_select_counter(idx); write_sysreg(value, pmxevcntr_el0); @@ -405,20 +456,14 @@ static void armv8pmu_write_counter(struct perf_event *event, u64 value) struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; + value = armv8pmu_bias_long_counter(event, value); + if (!armv8pmu_counter_valid(cpu_pmu, idx)) pr_err("CPU%u writing wrong counter %d\n", smp_processor_id(), idx); - else if (idx == ARMV8_IDX_CYCLE_COUNTER) { - /* - * The cycles counter is really a 64-bit counter. - * When treating it as a 32-bit counter, we only count - * the lower 32 bits, and set the upper 32-bits so that - * we get an interrupt upon 32-bit overflow. - */ - if (!armv8pmu_event_is_64bit(event)) - value |= 0xffffffff00000000ULL; + else if (idx == ARMV8_IDX_CYCLE_COUNTER) write_sysreg(value, pmccntr_el0); - } else + else armv8pmu_write_hw_counter(event, value); } @@ -450,86 +495,74 @@ static inline void armv8pmu_write_event_type(struct perf_event *event) } } -static inline int armv8pmu_enable_counter(int idx) +static u32 armv8pmu_event_cnten_mask(struct perf_event *event) { - u32 counter = ARMV8_IDX_TO_COUNTER(idx); - write_sysreg(BIT(counter), pmcntenset_el0); - return idx; + int counter = ARMV8_IDX_TO_COUNTER(event->hw.idx); + u32 mask = BIT(counter); + + if (armv8pmu_event_is_chained(event)) + mask |= BIT(counter - 1); + return mask; +} + +static inline void armv8pmu_enable_counter(u32 mask) +{ + write_sysreg(mask, pmcntenset_el0); } static inline void armv8pmu_enable_event_counter(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; - int idx = event->hw.idx; - u32 counter_bits = BIT(ARMV8_IDX_TO_COUNTER(idx)); + u32 mask = armv8pmu_event_cnten_mask(event); - if (armv8pmu_event_is_chained(event)) - counter_bits |= BIT(ARMV8_IDX_TO_COUNTER(idx - 1)); - - kvm_set_pmu_events(counter_bits, attr); + kvm_set_pmu_events(mask, attr); /* We rely on the hypervisor switch code to enable guest counters */ - if (!kvm_pmu_counter_deferred(attr)) { - armv8pmu_enable_counter(idx); - if (armv8pmu_event_is_chained(event)) - armv8pmu_enable_counter(idx - 1); - } + if (!kvm_pmu_counter_deferred(attr)) + armv8pmu_enable_counter(mask); } -static inline int armv8pmu_disable_counter(int idx) +static inline void armv8pmu_disable_counter(u32 mask) { - u32 counter = ARMV8_IDX_TO_COUNTER(idx); - write_sysreg(BIT(counter), pmcntenclr_el0); - return idx; + write_sysreg(mask, pmcntenclr_el0); } static inline void armv8pmu_disable_event_counter(struct perf_event *event) { - struct hw_perf_event *hwc = &event->hw; struct perf_event_attr *attr = &event->attr; - int idx = hwc->idx; - u32 counter_bits = BIT(ARMV8_IDX_TO_COUNTER(idx)); - - if (armv8pmu_event_is_chained(event)) - counter_bits |= BIT(ARMV8_IDX_TO_COUNTER(idx - 1)); + u32 mask = armv8pmu_event_cnten_mask(event); - kvm_clr_pmu_events(counter_bits); + kvm_clr_pmu_events(mask); /* We rely on the hypervisor switch code to disable guest counters */ - if (!kvm_pmu_counter_deferred(attr)) { - if (armv8pmu_event_is_chained(event)) - armv8pmu_disable_counter(idx - 1); - armv8pmu_disable_counter(idx); - } + if (!kvm_pmu_counter_deferred(attr)) + armv8pmu_disable_counter(mask); } -static inline int armv8pmu_enable_intens(int idx) +static inline void armv8pmu_enable_intens(u32 mask) { - u32 counter = ARMV8_IDX_TO_COUNTER(idx); - write_sysreg(BIT(counter), pmintenset_el1); - return idx; + write_sysreg(mask, pmintenset_el1); } -static inline int armv8pmu_enable_event_irq(struct perf_event *event) +static inline void armv8pmu_enable_event_irq(struct perf_event *event) { - return armv8pmu_enable_intens(event->hw.idx); + u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx); + armv8pmu_enable_intens(BIT(counter)); } -static inline int armv8pmu_disable_intens(int idx) +static inline void armv8pmu_disable_intens(u32 mask) { - u32 counter = ARMV8_IDX_TO_COUNTER(idx); - write_sysreg(BIT(counter), pmintenclr_el1); + write_sysreg(mask, pmintenclr_el1); isb(); /* Clear the overflow flag in case an interrupt is pending. */ - write_sysreg(BIT(counter), pmovsclr_el0); + write_sysreg(mask, pmovsclr_el0); isb(); - - return idx; } -static inline int armv8pmu_disable_event_irq(struct perf_event *event) +static inline void armv8pmu_disable_event_irq(struct perf_event *event) { - return armv8pmu_disable_intens(event->hw.idx); + u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx); + armv8pmu_disable_intens(BIT(counter)); } static inline u32 armv8pmu_getreset_flags(void) @@ -743,7 +776,8 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, /* * Otherwise use events counters */ - if (armv8pmu_event_is_64bit(event)) + if (armv8pmu_event_is_64bit(event) && + !armv8pmu_has_long_event(cpu_pmu)) return armv8pmu_get_chain_idx(cpuc, cpu_pmu); else return armv8pmu_get_single_idx(cpuc, cpu_pmu); @@ -815,13 +849,11 @@ static int armv8pmu_filter_match(struct perf_event *event) static void armv8pmu_reset(void *info) { struct arm_pmu *cpu_pmu = (struct arm_pmu *)info; - u32 idx, nb_cnt = cpu_pmu->num_events; + u32 pmcr; /* The counter and interrupt enable registers are unknown at reset. */ - for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { - armv8pmu_disable_counter(idx); - armv8pmu_disable_intens(idx); - } + armv8pmu_disable_counter(U32_MAX); + armv8pmu_disable_intens(U32_MAX); /* Clear the counters we flip at guest entry/exit */ kvm_clr_pmu_events(U32_MAX); @@ -830,8 +862,13 @@ static void armv8pmu_reset(void *info) * Initialize & Reset PMNC. Request overflow interrupt for * 64 bit cycle counter but cheat in armv8pmu_write_counter(). */ - armv8pmu_pmcr_write(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C | - ARMV8_PMU_PMCR_LC); + pmcr = ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C | ARMV8_PMU_PMCR_LC; + + /* Enable long event counter support where available */ + if (armv8pmu_has_long_event(cpu_pmu)) + pmcr |= ARMV8_PMU_PMCR_LP; + + armv8pmu_pmcr_write(pmcr); } static int __armv8_pmuv3_map_event(struct perf_event *event, @@ -914,6 +951,7 @@ static void __armv8pmu_probe_pmu(void *info) if (pmuver == 0xf || pmuver == 0) return; + cpu_pmu->pmuver = pmuver; probe->present = true; /* Read the nb of CNTx counters supported from PMNC */ @@ -953,7 +991,10 @@ static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu) return probe.present ? 0 : -ENODEV; } -static int armv8_pmu_init(struct arm_pmu *cpu_pmu) +static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name, + int (*map_event)(struct perf_event *event), + const struct attribute_group *events, + const struct attribute_group *format) { int ret = armv8pmu_probe_pmu(cpu_pmu); if (ret) @@ -972,144 +1013,127 @@ static int armv8_pmu_init(struct arm_pmu *cpu_pmu) cpu_pmu->set_event_filter = armv8pmu_set_event_filter; cpu_pmu->filter_match = armv8pmu_filter_match; + cpu_pmu->name = name; + cpu_pmu->map_event = map_event; + cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ? + events : &armv8_pmuv3_events_attr_group; + cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ? + format : &armv8_pmuv3_format_attr_group; + return 0; } static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; - - cpu_pmu->name = "armv8_pmuv3"; - cpu_pmu->map_event = armv8_pmuv3_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; + return armv8_pmu_init(cpu_pmu, "armv8_pmuv3", + armv8_pmuv3_map_event, NULL, NULL); +} - return 0; +static int armv8_a34_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a34", + armv8_pmuv3_map_event, NULL, NULL); } static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; - - cpu_pmu->name = "armv8_cortex_a35"; - cpu_pmu->map_event = armv8_a53_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; - - return 0; + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a35", + armv8_a53_map_event, NULL, NULL); } static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; - - cpu_pmu->name = "armv8_cortex_a53"; - cpu_pmu->map_event = armv8_a53_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a53", + armv8_a53_map_event, NULL, NULL); +} - return 0; +static int armv8_a55_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a55", + armv8_pmuv3_map_event, NULL, NULL); } static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; - - cpu_pmu->name = "armv8_cortex_a57"; - cpu_pmu->map_event = armv8_a57_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a57", + armv8_a57_map_event, NULL, NULL); +} - return 0; +static int armv8_a65_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a65", + armv8_pmuv3_map_event, NULL, NULL); } static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; - - cpu_pmu->name = "armv8_cortex_a72"; - cpu_pmu->map_event = armv8_a57_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; - - return 0; + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a72", + armv8_a57_map_event, NULL, NULL); } static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; - - cpu_pmu->name = "armv8_cortex_a73"; - cpu_pmu->map_event = armv8_a73_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a73", + armv8_a73_map_event, NULL, NULL); +} - return 0; +static int armv8_a75_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a75", + armv8_pmuv3_map_event, NULL, NULL); } -static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) +static int armv8_a76_pmu_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a76", + armv8_pmuv3_map_event, NULL, NULL); +} - cpu_pmu->name = "armv8_cavium_thunder"; - cpu_pmu->map_event = armv8_thunder_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; +static int armv8_a77_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_cortex_a77", + armv8_pmuv3_map_event, NULL, NULL); +} - return 0; +static int armv8_e1_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_neoverse_e1", + armv8_pmuv3_map_event, NULL, NULL); } -static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) +static int armv8_n1_pmu_init(struct arm_pmu *cpu_pmu) { - int ret = armv8_pmu_init(cpu_pmu); - if (ret) - return ret; + return armv8_pmu_init(cpu_pmu, "armv8_neoverse_n1", + armv8_pmuv3_map_event, NULL, NULL); +} - cpu_pmu->name = "armv8_brcm_vulcan"; - cpu_pmu->map_event = armv8_vulcan_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv8_pmuv3_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv8_pmuv3_format_attr_group; +static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_cavium_thunder", + armv8_thunder_map_event, NULL, NULL); +} - return 0; +static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) +{ + return armv8_pmu_init(cpu_pmu, "armv8_brcm_vulcan", + armv8_vulcan_map_event, NULL, NULL); } static const struct of_device_id armv8_pmu_of_device_ids[] = { {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init}, + {.compatible = "arm,cortex-a34-pmu", .data = armv8_a34_pmu_init}, {.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init}, {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init}, + {.compatible = "arm,cortex-a55-pmu", .data = armv8_a55_pmu_init}, {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init}, + {.compatible = "arm,cortex-a65-pmu", .data = armv8_a65_pmu_init}, {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init}, {.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init}, + {.compatible = "arm,cortex-a75-pmu", .data = armv8_a75_pmu_init}, + {.compatible = "arm,cortex-a76-pmu", .data = armv8_a76_pmu_init}, + {.compatible = "arm,cortex-a77-pmu", .data = armv8_a77_pmu_init}, + {.compatible = "arm,neoverse-e1-pmu", .data = armv8_e1_pmu_init}, + {.compatible = "arm,neoverse-n1-pmu", .data = armv8_n1_pmu_init}, {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init}, {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init}, {}, diff --git a/arch/arm64/kernel/pointer_auth.c b/arch/arm64/kernel/pointer_auth.c index c507b584259d..1e77736a4f66 100644 --- a/arch/arm64/kernel/pointer_auth.c +++ b/arch/arm64/kernel/pointer_auth.c @@ -9,7 +9,7 @@ int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg) { - struct ptrauth_keys *keys = &tsk->thread.keys_user; + struct ptrauth_keys_user *keys = &tsk->thread.keys_user; unsigned long addr_key_mask = PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY; unsigned long key_mask = addr_key_mask | PR_PAC_APGAKEY; @@ -18,8 +18,7 @@ int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg) return -EINVAL; if (!arg) { - ptrauth_keys_init(keys); - ptrauth_keys_switch(keys); + ptrauth_keys_init_user(keys); return 0; } @@ -41,7 +40,5 @@ int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg) if (arg & PR_PAC_APGAKEY) get_random_bytes(&keys->apga, sizeof(keys->apga)); - ptrauth_keys_switch(keys); - return 0; } diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 00626057a384..56be4cbf771f 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -141,11 +141,11 @@ void arch_cpu_idle_dead(void) * to execute e.g. a RAM-based pin loop is not sufficient. This allows the * kexec'd kernel to use any and all RAM as it sees fit, without having to * avoid any code or data used by any SW CPU pin loop. The CPU hotplug - * functionality embodied in disable_nonboot_cpus() to achieve this. + * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this. */ void machine_shutdown(void) { - disable_nonboot_cpus(); + smp_shutdown_nonboot_cpus(reboot_cpu); } /* @@ -262,7 +262,7 @@ void __show_regs(struct pt_regs *regs) if (!user_mode(regs)) { printk("pc : %pS\n", (void *)regs->pc); - printk("lr : %pS\n", (void *)lr); + printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr)); } else { printk("pc : %016llx\n", regs->pc); printk("lr : %016llx\n", lr); @@ -376,6 +376,8 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long stack_start, */ fpsimd_flush_task_state(p); + ptrauth_thread_init_kernel(p); + if (likely(!(p->flags & PF_KTHREAD))) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; @@ -512,7 +514,6 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, contextidr_thread_switch(next); entry_task_switch(next); uao_thread_switch(next); - ptrauth_thread_switch(next); ssbs_thread_switch(next); /* diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index cd6e5fa48b9c..b3d3005d9515 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -999,7 +999,7 @@ static struct ptrauth_key pac_key_from_user(__uint128_t ukey) } static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys, - const struct ptrauth_keys *keys) + const struct ptrauth_keys_user *keys) { ukeys->apiakey = pac_key_to_user(&keys->apia); ukeys->apibkey = pac_key_to_user(&keys->apib); @@ -1007,7 +1007,7 @@ static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys, ukeys->apdbkey = pac_key_to_user(&keys->apdb); } -static void pac_address_keys_from_user(struct ptrauth_keys *keys, +static void pac_address_keys_from_user(struct ptrauth_keys_user *keys, const struct user_pac_address_keys *ukeys) { keys->apia = pac_key_from_user(ukeys->apiakey); @@ -1021,7 +1021,7 @@ static int pac_address_keys_get(struct task_struct *target, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { - struct ptrauth_keys *keys = &target->thread.keys_user; + struct ptrauth_keys_user *keys = &target->thread.keys_user; struct user_pac_address_keys user_keys; if (!system_supports_address_auth()) @@ -1038,7 +1038,7 @@ static int pac_address_keys_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct ptrauth_keys *keys = &target->thread.keys_user; + struct ptrauth_keys_user *keys = &target->thread.keys_user; struct user_pac_address_keys user_keys; int ret; @@ -1056,12 +1056,12 @@ static int pac_address_keys_set(struct task_struct *target, } static void pac_generic_keys_to_user(struct user_pac_generic_keys *ukeys, - const struct ptrauth_keys *keys) + const struct ptrauth_keys_user *keys) { ukeys->apgakey = pac_key_to_user(&keys->apga); } -static void pac_generic_keys_from_user(struct ptrauth_keys *keys, +static void pac_generic_keys_from_user(struct ptrauth_keys_user *keys, const struct user_pac_generic_keys *ukeys) { keys->apga = pac_key_from_user(ukeys->apgakey); @@ -1072,7 +1072,7 @@ static int pac_generic_keys_get(struct task_struct *target, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { - struct ptrauth_keys *keys = &target->thread.keys_user; + struct ptrauth_keys_user *keys = &target->thread.keys_user; struct user_pac_generic_keys user_keys; if (!system_supports_generic_auth()) @@ -1089,7 +1089,7 @@ static int pac_generic_keys_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct ptrauth_keys *keys = &target->thread.keys_user; + struct ptrauth_keys_user *keys = &target->thread.keys_user; struct user_pac_generic_keys user_keys; int ret; diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S index c1d7db71a726..c40ce496c78b 100644 --- a/arch/arm64/kernel/relocate_kernel.S +++ b/arch/arm64/kernel/relocate_kernel.S @@ -41,7 +41,7 @@ ENTRY(arm64_relocate_new_kernel) cmp x0, #CurrentEL_EL2 b.ne 1f mrs x0, sctlr_el2 - ldr x1, =SCTLR_ELx_FLAGS + mov_q x1, SCTLR_ELx_FLAGS bic x0, x0, x1 pre_disable_mmu_workaround msr sctlr_el2, x0 @@ -113,8 +113,6 @@ ENTRY(arm64_relocate_new_kernel) ENDPROC(arm64_relocate_new_kernel) -.ltorg - .align 3 /* To keep the 64-bit values below naturally aligned. */ .Lcopy_end: diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index a34890bf309f..3fd2c11c09fc 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -344,7 +344,7 @@ void __init setup_arch(char **cmdline_p) else psci_acpi_init(); - cpu_read_bootcpu_ops(); + init_bootcpu_ops(); smp_init_cpus(); smp_build_mpidr_hash(); @@ -371,8 +371,10 @@ void __init setup_arch(char **cmdline_p) static inline bool cpu_can_disable(unsigned int cpu) { #ifdef CONFIG_HOTPLUG_CPU - if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_can_disable) - return cpu_ops[cpu]->cpu_can_disable(cpu); + const struct cpu_operations *ops = get_cpu_ops(cpu); + + if (ops && ops->cpu_can_disable) + return ops->cpu_can_disable(cpu); #endif return false; } diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index f5b04dd8a710..7b2f2e650c44 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -3,6 +3,7 @@ #include <linux/linkage.h> #include <asm/asm-offsets.h> #include <asm/assembler.h> +#include <asm/smp.h> .text /* @@ -99,6 +100,7 @@ ENDPROC(__cpu_suspend_enter) .pushsection ".idmap.text", "awx" ENTRY(cpu_resume) bl el2_setup // if in EL2 drop to EL1 cleanly + mov x0, #ARM64_CPU_RUNTIME bl __cpu_setup /* enable the MMU early - so we can access sleep_save_stash by va */ adrp x1, swapper_pg_dir diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 5407bf5d98ac..061f60fe452f 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -93,8 +93,10 @@ static inline int op_cpu_kill(unsigned int cpu) */ static int boot_secondary(unsigned int cpu, struct task_struct *idle) { - if (cpu_ops[cpu]->cpu_boot) - return cpu_ops[cpu]->cpu_boot(cpu); + const struct cpu_operations *ops = get_cpu_ops(cpu); + + if (ops->cpu_boot) + return ops->cpu_boot(cpu); return -EOPNOTSUPP; } @@ -112,63 +114,66 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) */ secondary_data.task = idle; secondary_data.stack = task_stack_page(idle) + THREAD_SIZE; +#if defined(CONFIG_ARM64_PTR_AUTH) + secondary_data.ptrauth_key.apia.lo = idle->thread.keys_kernel.apia.lo; + secondary_data.ptrauth_key.apia.hi = idle->thread.keys_kernel.apia.hi; +#endif update_cpu_boot_status(CPU_MMU_OFF); __flush_dcache_area(&secondary_data, sizeof(secondary_data)); - /* - * Now bring the CPU into our world. - */ + /* Now bring the CPU into our world */ ret = boot_secondary(cpu, idle); - if (ret == 0) { - /* - * CPU was successfully started, wait for it to come online or - * time out. - */ - wait_for_completion_timeout(&cpu_running, - msecs_to_jiffies(5000)); - - if (!cpu_online(cpu)) { - pr_crit("CPU%u: failed to come online\n", cpu); - ret = -EIO; - } - } else { + if (ret) { pr_err("CPU%u: failed to boot: %d\n", cpu, ret); return ret; } + /* + * CPU was successfully started, wait for it to come online or + * time out. + */ + wait_for_completion_timeout(&cpu_running, + msecs_to_jiffies(5000)); + if (cpu_online(cpu)) + return 0; + + pr_crit("CPU%u: failed to come online\n", cpu); secondary_data.task = NULL; secondary_data.stack = NULL; +#if defined(CONFIG_ARM64_PTR_AUTH) + secondary_data.ptrauth_key.apia.lo = 0; + secondary_data.ptrauth_key.apia.hi = 0; +#endif __flush_dcache_area(&secondary_data, sizeof(secondary_data)); status = READ_ONCE(secondary_data.status); - if (ret && status) { - - if (status == CPU_MMU_OFF) - status = READ_ONCE(__early_cpu_boot_status); + if (status == CPU_MMU_OFF) + status = READ_ONCE(__early_cpu_boot_status); - switch (status & CPU_BOOT_STATUS_MASK) { - default: - pr_err("CPU%u: failed in unknown state : 0x%lx\n", - cpu, status); - cpus_stuck_in_kernel++; - break; - case CPU_KILL_ME: - if (!op_cpu_kill(cpu)) { - pr_crit("CPU%u: died during early boot\n", cpu); - break; - } - pr_crit("CPU%u: may not have shut down cleanly\n", cpu); - /* Fall through */ - case CPU_STUCK_IN_KERNEL: - pr_crit("CPU%u: is stuck in kernel\n", cpu); - if (status & CPU_STUCK_REASON_52_BIT_VA) - pr_crit("CPU%u: does not support 52-bit VAs\n", cpu); - if (status & CPU_STUCK_REASON_NO_GRAN) - pr_crit("CPU%u: does not support %luK granule \n", cpu, PAGE_SIZE / SZ_1K); - cpus_stuck_in_kernel++; + switch (status & CPU_BOOT_STATUS_MASK) { + default: + pr_err("CPU%u: failed in unknown state : 0x%lx\n", + cpu, status); + cpus_stuck_in_kernel++; + break; + case CPU_KILL_ME: + if (!op_cpu_kill(cpu)) { + pr_crit("CPU%u: died during early boot\n", cpu); break; - case CPU_PANIC_KERNEL: - panic("CPU%u detected unsupported configuration\n", cpu); } + pr_crit("CPU%u: may not have shut down cleanly\n", cpu); + /* Fall through */ + case CPU_STUCK_IN_KERNEL: + pr_crit("CPU%u: is stuck in kernel\n", cpu); + if (status & CPU_STUCK_REASON_52_BIT_VA) + pr_crit("CPU%u: does not support 52-bit VAs\n", cpu); + if (status & CPU_STUCK_REASON_NO_GRAN) { + pr_crit("CPU%u: does not support %luK granule\n", + cpu, PAGE_SIZE / SZ_1K); + } + cpus_stuck_in_kernel++; + break; + case CPU_PANIC_KERNEL: + panic("CPU%u detected unsupported configuration\n", cpu); } return ret; @@ -196,6 +201,7 @@ asmlinkage notrace void secondary_start_kernel(void) { u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; struct mm_struct *mm = &init_mm; + const struct cpu_operations *ops; unsigned int cpu; cpu = task_cpu(current); @@ -227,8 +233,9 @@ asmlinkage notrace void secondary_start_kernel(void) */ check_local_cpu_capabilities(); - if (cpu_ops[cpu]->cpu_postboot) - cpu_ops[cpu]->cpu_postboot(); + ops = get_cpu_ops(cpu); + if (ops->cpu_postboot) + ops->cpu_postboot(); /* * Log the CPU info before it is marked online and might get read. @@ -266,19 +273,21 @@ asmlinkage notrace void secondary_start_kernel(void) #ifdef CONFIG_HOTPLUG_CPU static int op_cpu_disable(unsigned int cpu) { + const struct cpu_operations *ops = get_cpu_ops(cpu); + /* * If we don't have a cpu_die method, abort before we reach the point * of no return. CPU0 may not have an cpu_ops, so test for it. */ - if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_die) + if (!ops || !ops->cpu_die) return -EOPNOTSUPP; /* * We may need to abort a hot unplug for some other mechanism-specific * reason. */ - if (cpu_ops[cpu]->cpu_disable) - return cpu_ops[cpu]->cpu_disable(cpu); + if (ops->cpu_disable) + return ops->cpu_disable(cpu); return 0; } @@ -314,15 +323,17 @@ int __cpu_disable(void) static int op_cpu_kill(unsigned int cpu) { + const struct cpu_operations *ops = get_cpu_ops(cpu); + /* * If we have no means of synchronising with the dying CPU, then assume * that it is really dead. We can only wait for an arbitrary length of * time and hope that it's dead, so let's skip the wait and just hope. */ - if (!cpu_ops[cpu]->cpu_kill) + if (!ops->cpu_kill) return 0; - return cpu_ops[cpu]->cpu_kill(cpu); + return ops->cpu_kill(cpu); } /* @@ -357,6 +368,7 @@ void __cpu_die(unsigned int cpu) void cpu_die(void) { unsigned int cpu = smp_processor_id(); + const struct cpu_operations *ops = get_cpu_ops(cpu); idle_task_exit(); @@ -370,12 +382,22 @@ void cpu_die(void) * mechanism must perform all required cache maintenance to ensure that * no dirty lines are lost in the process of shutting down the CPU. */ - cpu_ops[cpu]->cpu_die(cpu); + ops->cpu_die(cpu); BUG(); } #endif +static void __cpu_try_die(int cpu) +{ +#ifdef CONFIG_HOTPLUG_CPU + const struct cpu_operations *ops = get_cpu_ops(cpu); + + if (ops && ops->cpu_die) + ops->cpu_die(cpu); +#endif +} + /* * Kill the calling secondary CPU, early in bringup before it is turned * online. @@ -389,12 +411,11 @@ void cpu_die_early(void) /* Mark this CPU absent */ set_cpu_present(cpu, 0); -#ifdef CONFIG_HOTPLUG_CPU - update_cpu_boot_status(CPU_KILL_ME); - /* Check if we can park ourselves */ - if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die) - cpu_ops[cpu]->cpu_die(cpu); -#endif + if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) { + update_cpu_boot_status(CPU_KILL_ME); + __cpu_try_die(cpu); + } + update_cpu_boot_status(CPU_STUCK_IN_KERNEL); cpu_park_loop(); @@ -488,10 +509,13 @@ static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid) */ static int __init smp_cpu_setup(int cpu) { - if (cpu_read_ops(cpu)) + const struct cpu_operations *ops; + + if (init_cpu_ops(cpu)) return -ENODEV; - if (cpu_ops[cpu]->cpu_init(cpu)) + ops = get_cpu_ops(cpu); + if (ops->cpu_init(cpu)) return -ENODEV; set_cpu_possible(cpu, true); @@ -714,6 +738,7 @@ void __init smp_init_cpus(void) void __init smp_prepare_cpus(unsigned int max_cpus) { + const struct cpu_operations *ops; int err; unsigned int cpu; unsigned int this_cpu; @@ -744,10 +769,11 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (cpu == smp_processor_id()) continue; - if (!cpu_ops[cpu]) + ops = get_cpu_ops(cpu); + if (!ops) continue; - err = cpu_ops[cpu]->cpu_prepare(cpu); + err = ops->cpu_prepare(cpu); if (err) continue; @@ -863,10 +889,8 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) local_irq_disable(); sdei_mask_local_cpu(); -#ifdef CONFIG_HOTPLUG_CPU - if (cpu_ops[cpu]->cpu_die) - cpu_ops[cpu]->cpu_die(cpu); -#endif + if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) + __cpu_try_die(cpu); /* just in case */ cpu_park_loop(); @@ -1059,8 +1083,9 @@ static bool have_cpu_die(void) { #ifdef CONFIG_HOTPLUG_CPU int any_cpu = raw_smp_processor_id(); + const struct cpu_operations *ops = get_cpu_ops(any_cpu); - if (cpu_ops[any_cpu] && cpu_ops[any_cpu]->cpu_die) + if (ops && ops->cpu_die) return true; #endif return false; diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index a336cb124320..139679c745bf 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -14,6 +14,7 @@ #include <linux/stacktrace.h> #include <asm/irq.h> +#include <asm/pointer_auth.h> #include <asm/stack_pointer.h> #include <asm/stacktrace.h> @@ -86,7 +87,7 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) #ifdef CONFIG_FUNCTION_GRAPH_TRACER if (tsk->ret_stack && - (frame->pc == (unsigned long)return_to_handler)) { + (ptrauth_strip_insn_pac(frame->pc) == (unsigned long)return_to_handler)) { struct ftrace_ret_stack *ret_stack; /* * This is a case where function graph tracer has @@ -101,6 +102,8 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + frame->pc = ptrauth_strip_insn_pac(frame->pc); + /* * Frames created upon entry from EL0 have NULL FP and PC values, so * don't bother reporting these. Frames created by __noreturn functions diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index fa9528dfd0ce..0801a0f3c156 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -14,6 +14,7 @@ #include <linux/acpi.h> #include <linux/arch_topology.h> #include <linux/cacheinfo.h> +#include <linux/cpufreq.h> #include <linux/init.h> #include <linux/percpu.h> @@ -120,4 +121,183 @@ int __init parse_acpi_topology(void) } #endif +#ifdef CONFIG_ARM64_AMU_EXTN +#undef pr_fmt +#define pr_fmt(fmt) "AMU: " fmt + +static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale); +static DEFINE_PER_CPU(u64, arch_const_cycles_prev); +static DEFINE_PER_CPU(u64, arch_core_cycles_prev); +static cpumask_var_t amu_fie_cpus; + +/* Initialize counter reference per-cpu variables for the current CPU */ +void init_cpu_freq_invariance_counters(void) +{ + this_cpu_write(arch_core_cycles_prev, + read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0)); + this_cpu_write(arch_const_cycles_prev, + read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0)); +} + +static int validate_cpu_freq_invariance_counters(int cpu) +{ + u64 max_freq_hz, ratio; + + if (!cpu_has_amu_feat(cpu)) { + pr_debug("CPU%d: counters are not supported.\n", cpu); + return -EINVAL; + } + + if (unlikely(!per_cpu(arch_const_cycles_prev, cpu) || + !per_cpu(arch_core_cycles_prev, cpu))) { + pr_debug("CPU%d: cycle counters are not enabled.\n", cpu); + return -EINVAL; + } + + /* Convert maximum frequency from KHz to Hz and validate */ + max_freq_hz = cpufreq_get_hw_max_freq(cpu) * 1000; + if (unlikely(!max_freq_hz)) { + pr_debug("CPU%d: invalid maximum frequency.\n", cpu); + return -EINVAL; + } + + /* + * Pre-compute the fixed ratio between the frequency of the constant + * counter and the maximum frequency of the CPU. + * + * const_freq + * arch_max_freq_scale = ---------------- * SCHED_CAPACITY_SCALE² + * cpuinfo_max_freq + * + * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE² + * in order to ensure a good resolution for arch_max_freq_scale for + * very low arch timer frequencies (down to the KHz range which should + * be unlikely). + */ + ratio = (u64)arch_timer_get_rate() << (2 * SCHED_CAPACITY_SHIFT); + ratio = div64_u64(ratio, max_freq_hz); + if (!ratio) { + WARN_ONCE(1, "System timer frequency too low.\n"); + return -EINVAL; + } + + per_cpu(arch_max_freq_scale, cpu) = (unsigned long)ratio; + + return 0; +} + +static inline bool +enable_policy_freq_counters(int cpu, cpumask_var_t valid_cpus) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + + if (!policy) { + pr_debug("CPU%d: No cpufreq policy found.\n", cpu); + return false; + } + + if (cpumask_subset(policy->related_cpus, valid_cpus)) + cpumask_or(amu_fie_cpus, policy->related_cpus, + amu_fie_cpus); + + cpufreq_cpu_put(policy); + + return true; +} + +static DEFINE_STATIC_KEY_FALSE(amu_fie_key); +#define amu_freq_invariant() static_branch_unlikely(&amu_fie_key) + +static int __init init_amu_fie(void) +{ + cpumask_var_t valid_cpus; + bool have_policy = false; + int ret = 0; + int cpu; + + if (!zalloc_cpumask_var(&valid_cpus, GFP_KERNEL)) + return -ENOMEM; + + if (!zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto free_valid_mask; + } + + for_each_present_cpu(cpu) { + if (validate_cpu_freq_invariance_counters(cpu)) + continue; + cpumask_set_cpu(cpu, valid_cpus); + have_policy |= enable_policy_freq_counters(cpu, valid_cpus); + } + + /* + * If we are not restricted by cpufreq policies, we only enable + * the use of the AMU feature for FIE if all CPUs support AMU. + * Otherwise, enable_policy_freq_counters has already enabled + * policy cpus. + */ + if (!have_policy && cpumask_equal(valid_cpus, cpu_present_mask)) + cpumask_or(amu_fie_cpus, amu_fie_cpus, valid_cpus); + + if (!cpumask_empty(amu_fie_cpus)) { + pr_info("CPUs[%*pbl]: counters will be used for FIE.", + cpumask_pr_args(amu_fie_cpus)); + static_branch_enable(&amu_fie_key); + } + +free_valid_mask: + free_cpumask_var(valid_cpus); + + return ret; +} +late_initcall_sync(init_amu_fie); + +bool arch_freq_counters_available(struct cpumask *cpus) +{ + return amu_freq_invariant() && + cpumask_subset(cpus, amu_fie_cpus); +} + +void topology_scale_freq_tick(void) +{ + u64 prev_core_cnt, prev_const_cnt; + u64 core_cnt, const_cnt, scale; + int cpu = smp_processor_id(); + + if (!amu_freq_invariant()) + return; + + if (!cpumask_test_cpu(cpu, amu_fie_cpus)) + return; + + const_cnt = read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0); + core_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0); + prev_const_cnt = this_cpu_read(arch_const_cycles_prev); + prev_core_cnt = this_cpu_read(arch_core_cycles_prev); + + if (unlikely(core_cnt <= prev_core_cnt || + const_cnt <= prev_const_cnt)) + goto store_and_exit; + + /* + * /\core arch_max_freq_scale + * scale = ------- * -------------------- + * /\const SCHED_CAPACITY_SCALE + * + * See validate_cpu_freq_invariance_counters() for details on + * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT. + */ + scale = core_cnt - prev_core_cnt; + scale *= this_cpu_read(arch_max_freq_scale); + scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT, + const_cnt - prev_const_cnt); + + scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE); + this_cpu_write(freq_scale, (unsigned long)scale); + +store_and_exit: + this_cpu_write(arch_core_cycles_prev, core_cnt); + this_cpu_write(arch_const_cycles_prev, const_cnt); +} +#endif /* CONFIG_ARM64_AMU_EXTN */ diff --git a/arch/arm64/kernel/vdso/sigreturn.S b/arch/arm64/kernel/vdso/sigreturn.S index 0723aa398d6e..12324863d5c2 100644 --- a/arch/arm64/kernel/vdso/sigreturn.S +++ b/arch/arm64/kernel/vdso/sigreturn.S @@ -14,7 +14,7 @@ .text nop -ENTRY(__kernel_rt_sigreturn) +SYM_FUNC_START(__kernel_rt_sigreturn) .cfi_startproc .cfi_signal_frame .cfi_def_cfa x29, 0 @@ -23,4 +23,4 @@ ENTRY(__kernel_rt_sigreturn) mov x8, #__NR_rt_sigreturn svc #0 .cfi_endproc -ENDPROC(__kernel_rt_sigreturn) +SYM_FUNC_END(__kernel_rt_sigreturn) diff --git a/arch/arm64/kernel/vdso/vgettimeofday.c b/arch/arm64/kernel/vdso/vgettimeofday.c index 747635501a14..4236cf34d7d9 100644 --- a/arch/arm64/kernel/vdso/vgettimeofday.c +++ b/arch/arm64/kernel/vdso/vgettimeofday.c @@ -5,8 +5,6 @@ * Copyright (C) 2018 ARM Limited * */ -#include <linux/time.h> -#include <linux/types.h> int __kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index 04df57b43cb1..3964738ebbde 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -10,7 +10,18 @@ include $(srctree)/lib/vdso/Makefile # Same as cc-*option, but using CC_COMPAT instead of CC ifeq ($(CONFIG_CC_IS_CLANG), y) +COMPAT_GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE_COMPAT)elfedit)) +COMPAT_GCC_TOOLCHAIN := $(realpath $(COMPAT_GCC_TOOLCHAIN_DIR)/..) + +CC_COMPAT_CLANG_FLAGS := --target=$(notdir $(CROSS_COMPILE_COMPAT:%-=%)) +CC_COMPAT_CLANG_FLAGS += --prefix=$(COMPAT_GCC_TOOLCHAIN_DIR) +CC_COMPAT_CLANG_FLAGS += -no-integrated-as -Qunused-arguments +ifneq ($(COMPAT_GCC_TOOLCHAIN),) +CC_COMPAT_CLANG_FLAGS += --gcc-toolchain=$(COMPAT_GCC_TOOLCHAIN) +endif + CC_COMPAT ?= $(CC) +CC_COMPAT += $(CC_COMPAT_CLANG_FLAGS) else CC_COMPAT ?= $(CROSS_COMPILE_COMPAT)gcc endif diff --git a/arch/arm64/kernel/vdso32/sigreturn.S b/arch/arm64/kernel/vdso32/sigreturn.S index 1a81277c2d09..620524969696 100644 --- a/arch/arm64/kernel/vdso32/sigreturn.S +++ b/arch/arm64/kernel/vdso32/sigreturn.S @@ -10,13 +10,6 @@ #include <asm/asm-offsets.h> #include <asm/unistd.h> -#define ARM_ENTRY(name) \ - ENTRY(name) - -#define ARM_ENDPROC(name) \ - .type name, %function; \ - END(name) - .text .arm @@ -24,39 +17,39 @@ .save {r0-r15} .pad #COMPAT_SIGFRAME_REGS_OFFSET nop -ARM_ENTRY(__kernel_sigreturn_arm) +SYM_FUNC_START(__kernel_sigreturn_arm) mov r7, #__NR_compat_sigreturn svc #0 .fnend -ARM_ENDPROC(__kernel_sigreturn_arm) +SYM_FUNC_END(__kernel_sigreturn_arm) .fnstart .save {r0-r15} .pad #COMPAT_RT_SIGFRAME_REGS_OFFSET nop -ARM_ENTRY(__kernel_rt_sigreturn_arm) +SYM_FUNC_START(__kernel_rt_sigreturn_arm) mov r7, #__NR_compat_rt_sigreturn svc #0 .fnend -ARM_ENDPROC(__kernel_rt_sigreturn_arm) +SYM_FUNC_END(__kernel_rt_sigreturn_arm) .thumb .fnstart .save {r0-r15} .pad #COMPAT_SIGFRAME_REGS_OFFSET nop -ARM_ENTRY(__kernel_sigreturn_thumb) +SYM_FUNC_START(__kernel_sigreturn_thumb) mov r7, #__NR_compat_sigreturn svc #0 .fnend -ARM_ENDPROC(__kernel_sigreturn_thumb) +SYM_FUNC_END(__kernel_sigreturn_thumb) .fnstart .save {r0-r15} .pad #COMPAT_RT_SIGFRAME_REGS_OFFSET nop -ARM_ENTRY(__kernel_rt_sigreturn_thumb) +SYM_FUNC_START(__kernel_rt_sigreturn_thumb) mov r7, #__NR_compat_rt_sigreturn svc #0 .fnend -ARM_ENDPROC(__kernel_rt_sigreturn_thumb) +SYM_FUNC_END(__kernel_rt_sigreturn_thumb) diff --git a/arch/arm64/kernel/vdso32/vgettimeofday.c b/arch/arm64/kernel/vdso32/vgettimeofday.c index 54fc1c2ce93f..5acff29c5991 100644 --- a/arch/arm64/kernel/vdso32/vgettimeofday.c +++ b/arch/arm64/kernel/vdso32/vgettimeofday.c @@ -5,26 +5,16 @@ * Copyright (C) 2018 ARM Limited * */ -#include <linux/time.h> -#include <linux/types.h> int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts) { - /* The checks below are required for ABI consistency with arm */ - if ((u32)ts >= TASK_SIZE_32) - return -EFAULT; - return __cvdso_clock_gettime32(clock, ts); } int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts) { - /* The checks below are required for ABI consistency with arm */ - if ((u32)ts >= TASK_SIZE_32) - return -EFAULT; - return __cvdso_clock_gettime(clock, ts); } @@ -37,10 +27,6 @@ int __vdso_gettimeofday(struct __kernel_old_timeval *tv, int __vdso_clock_getres(clockid_t clock_id, struct old_timespec32 *res) { - /* The checks below are required for ABI consistency with arm */ - if ((u32)res >= TASK_SIZE_32) - return -EFAULT; - return __cvdso_clock_getres_time32(clock_id, res); } diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S index 160be2b4696d..6e6ed5581eed 100644 --- a/arch/arm64/kvm/hyp-init.S +++ b/arch/arm64/kvm/hyp-init.S @@ -18,7 +18,7 @@ .align 11 -ENTRY(__kvm_hyp_init) +SYM_CODE_START(__kvm_hyp_init) ventry __invalid // Synchronous EL2t ventry __invalid // IRQ EL2t ventry __invalid // FIQ EL2t @@ -60,7 +60,7 @@ alternative_else_nop_endif msr ttbr0_el2, x4 mrs x4, tcr_el1 - ldr x5, =TCR_EL2_MASK + mov_q x5, TCR_EL2_MASK and x4, x4, x5 mov x5, #TCR_EL2_RES1 orr x4, x4, x5 @@ -102,7 +102,7 @@ alternative_else_nop_endif * as well as the EE bit on BE. Drop the A flag since the compiler * is allowed to generate unaligned accesses. */ - ldr x4, =(SCTLR_EL2_RES1 | (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A)) + mov_q x4, (SCTLR_EL2_RES1 | (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A)) CPU_BE( orr x4, x4, #SCTLR_ELx_EE) msr sctlr_el2, x4 isb @@ -117,9 +117,9 @@ CPU_BE( orr x4, x4, #SCTLR_ELx_EE) /* Hello, World! */ eret -ENDPROC(__kvm_hyp_init) +SYM_CODE_END(__kvm_hyp_init) -ENTRY(__kvm_handle_stub_hvc) +SYM_CODE_START(__kvm_handle_stub_hvc) cmp x0, #HVC_SOFT_RESTART b.ne 1f @@ -142,7 +142,7 @@ reset: * case we coming via HVC_SOFT_RESTART. */ mrs x5, sctlr_el2 - ldr x6, =SCTLR_ELx_FLAGS + mov_q x6, SCTLR_ELx_FLAGS bic x5, x5, x6 // Clear SCTL_M and etc pre_disable_mmu_workaround msr sctlr_el2, x5 @@ -155,11 +155,9 @@ reset: eret 1: /* Bad stub call */ - ldr x0, =HVC_STUB_ERR + mov_q x0, HVC_STUB_ERR eret -ENDPROC(__kvm_handle_stub_hvc) - - .ltorg +SYM_CODE_END(__kvm_handle_stub_hvc) .popsection diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S index c0094d520dff..3c79a1124af2 100644 --- a/arch/arm64/kvm/hyp.S +++ b/arch/arm64/kvm/hyp.S @@ -28,7 +28,7 @@ * and is used to implement hyp stubs in the same way as in * arch/arm64/kernel/hyp_stub.S. */ -ENTRY(__kvm_call_hyp) +SYM_FUNC_START(__kvm_call_hyp) hvc #0 ret -ENDPROC(__kvm_call_hyp) +SYM_FUNC_END(__kvm_call_hyp) diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index 78ff53225691..5b8ff517ff10 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -11,12 +11,12 @@ .text .pushsection .hyp.text, "ax" -ENTRY(__fpsimd_save_state) +SYM_FUNC_START(__fpsimd_save_state) fpsimd_save x0, 1 ret -ENDPROC(__fpsimd_save_state) +SYM_FUNC_END(__fpsimd_save_state) -ENTRY(__fpsimd_restore_state) +SYM_FUNC_START(__fpsimd_restore_state) fpsimd_restore x0, 1 ret -ENDPROC(__fpsimd_restore_state) +SYM_FUNC_END(__fpsimd_restore_state) diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index ffa68d5713f1..c2a13ab3c471 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -180,7 +180,7 @@ el2_error: eret sb -ENTRY(__hyp_do_panic) +SYM_FUNC_START(__hyp_do_panic) mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ PSR_MODE_EL1h) msr spsr_el2, lr @@ -188,18 +188,19 @@ ENTRY(__hyp_do_panic) msr elr_el2, lr eret sb -ENDPROC(__hyp_do_panic) +SYM_FUNC_END(__hyp_do_panic) -ENTRY(__hyp_panic) +SYM_CODE_START(__hyp_panic) get_host_ctxt x0, x1 b hyp_panic -ENDPROC(__hyp_panic) +SYM_CODE_END(__hyp_panic) .macro invalid_vector label, target = __hyp_panic .align 2 +SYM_CODE_START(\label) \label: b \target -ENDPROC(\label) +SYM_CODE_END(\label) .endm /* None of these should ever happen */ @@ -246,7 +247,7 @@ check_preamble_length 661b, 662b check_preamble_length 661b, 662b .endm -ENTRY(__kvm_hyp_vector) +SYM_CODE_START(__kvm_hyp_vector) invalid_vect el2t_sync_invalid // Synchronous EL2t invalid_vect el2t_irq_invalid // IRQ EL2t invalid_vect el2t_fiq_invalid // FIQ EL2t @@ -266,7 +267,7 @@ ENTRY(__kvm_hyp_vector) valid_vect el1_irq // IRQ 32-bit EL1 invalid_vect el1_fiq_invalid // FIQ 32-bit EL1 valid_vect el1_error // Error 32-bit EL1 -ENDPROC(__kvm_hyp_vector) +SYM_CODE_END(__kvm_hyp_vector) #ifdef CONFIG_KVM_INDIRECT_VECTORS .macro hyp_ventry @@ -311,15 +312,17 @@ alternative_cb_end .endm .align 11 -ENTRY(__bp_harden_hyp_vecs_start) +SYM_CODE_START(__bp_harden_hyp_vecs) .rept BP_HARDEN_EL2_SLOTS generate_vectors .endr -ENTRY(__bp_harden_hyp_vecs_end) +1: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ + .org 1b +SYM_CODE_END(__bp_harden_hyp_vecs) .popsection -ENTRY(__smccc_workaround_1_smc_start) +SYM_CODE_START(__smccc_workaround_1_smc) esb sub sp, sp, #(8 * 4) stp x2, x3, [sp, #(8 * 0)] @@ -329,5 +332,7 @@ ENTRY(__smccc_workaround_1_smc_start) ldp x2, x3, [sp, #(8 * 0)] ldp x0, x1, [sp, #(8 * 2)] add sp, sp, #(8 * 4) -ENTRY(__smccc_workaround_1_smc_end) +1: .org __smccc_workaround_1_smc + __SMCCC_WORKAROUND_1_SMC_SZ + .org 1b +SYM_CODE_END(__smccc_workaround_1_smc) #endif diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c index 925086b46136..eaa05c3c7235 100644 --- a/arch/arm64/kvm/hyp/switch.c +++ b/arch/arm64/kvm/hyp/switch.c @@ -98,6 +98,18 @@ static void activate_traps_vhe(struct kvm_vcpu *vcpu) val = read_sysreg(cpacr_el1); val |= CPACR_EL1_TTA; val &= ~CPACR_EL1_ZEN; + + /* + * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to + * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, + * except for some missing controls, such as TAM. + * In this case, CPTR_EL2.TAM has the same position with or without + * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM + * shift value for trapping the AMU accesses. + */ + + val |= CPTR_EL2_TAM; + if (update_fp_enabled(vcpu)) { if (vcpu_has_sve(vcpu)) val |= CPACR_EL1_ZEN; @@ -119,7 +131,7 @@ static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu) __activate_traps_common(vcpu); val = CPTR_EL2_DEFAULT; - val |= CPTR_EL2_TTA | CPTR_EL2_TZ; + val |= CPTR_EL2_TTA | CPTR_EL2_TZ | CPTR_EL2_TAM; if (!update_fp_enabled(vcpu)) { val |= CPTR_EL2_TFP; __activate_traps_fpsimd32(vcpu); @@ -127,7 +139,7 @@ static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu) write_sysreg(val, cptr_el2); - if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt; isb(); @@ -146,12 +158,12 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) { u64 hcr = vcpu->arch.hcr_el2; - if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) + if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) hcr |= HCR_TVM; write_sysreg(hcr, hcr_el2); - if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) + if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2); if (has_vhe()) @@ -181,7 +193,7 @@ static void __hyp_text __deactivate_traps_nvhe(void) { u64 mdcr_el2 = read_sysreg(mdcr_el2); - if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { u64 val; /* @@ -328,7 +340,7 @@ static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) * resolve the IPA using the AT instruction. */ if (!(esr & ESR_ELx_S1PTW) && - (cpus_have_const_cap(ARM64_WORKAROUND_834220) || + (cpus_have_final_cap(ARM64_WORKAROUND_834220) || (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) { if (!__translate_far_to_hpfar(far, &hpfar)) return false; @@ -498,7 +510,7 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) if (*exit_code != ARM_EXCEPTION_TRAP) goto exit; - if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && + if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 && handle_tx2_tvm(vcpu)) return true; @@ -555,7 +567,7 @@ exit: static inline bool __hyp_text __needs_ssbd_off(struct kvm_vcpu *vcpu) { - if (!cpus_have_const_cap(ARM64_SSBD)) + if (!cpus_have_final_cap(ARM64_SSBD)) return false; return !(vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG); diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c index 7672a978926c..75b1925763f1 100644 --- a/arch/arm64/kvm/hyp/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/sysreg-sr.c @@ -71,7 +71,7 @@ static void __hyp_text __sysreg_save_el2_return_state(struct kvm_cpu_context *ct ctxt->gp_regs.regs.pc = read_sysreg_el2(SYS_ELR); ctxt->gp_regs.regs.pstate = read_sysreg_el2(SYS_SPSR); - if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) + if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) ctxt->sys_regs[DISR_EL1] = read_sysreg_s(SYS_VDISR_EL2); } @@ -118,7 +118,7 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt) write_sysreg(ctxt->sys_regs[MPIDR_EL1], vmpidr_el2); write_sysreg(ctxt->sys_regs[CSSELR_EL1], csselr_el1); - if (!cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { + if (!cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1], SYS_SCTLR); write_sysreg_el1(ctxt->sys_regs[TCR_EL1], SYS_TCR); } else if (!ctxt->__hyp_running_vcpu) { @@ -149,7 +149,7 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt) write_sysreg(ctxt->sys_regs[PAR_EL1], par_el1); write_sysreg(ctxt->sys_regs[TPIDR_EL1], tpidr_el1); - if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE) && + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE) && ctxt->__hyp_running_vcpu) { /* * Must only be done for host registers, hence the context @@ -194,7 +194,7 @@ __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt) write_sysreg_el2(ctxt->gp_regs.regs.pc, SYS_ELR); write_sysreg_el2(pstate, SYS_SPSR); - if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) + if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) write_sysreg_s(ctxt->sys_regs[DISR_EL1], SYS_VDISR_EL2); } diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c index 92f560e3e1aa..ceaddbe4279f 100644 --- a/arch/arm64/kvm/hyp/tlb.c +++ b/arch/arm64/kvm/hyp/tlb.c @@ -23,7 +23,7 @@ static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm, local_irq_save(cxt->flags); - if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) { + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) { /* * For CPUs that are affected by ARM errata 1165522 or 1530923, * we cannot trust stage-1 to be in a correct state at that @@ -63,7 +63,7 @@ static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm, static void __hyp_text __tlb_switch_to_guest_nvhe(struct kvm *kvm, struct tlb_inv_context *cxt) { - if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { u64 val; /* @@ -103,7 +103,7 @@ static void __hyp_text __tlb_switch_to_host_vhe(struct kvm *kvm, write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2); isb(); - if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) { + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) { /* Restore the registers to what they were */ write_sysreg_el1(cxt->tcr, SYS_TCR); write_sysreg_el1(cxt->sctlr, SYS_SCTLR); @@ -117,7 +117,7 @@ static void __hyp_text __tlb_switch_to_host_nvhe(struct kvm *kvm, { write_sysreg(0, vttbr_el2); - if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { + if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) { /* Ensure write of the host VMID */ isb(); /* Restore the host's TCR_EL1 */ diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 3e909b117f0c..090f46d3add1 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1003,6 +1003,20 @@ static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, { SYS_DESC(SYS_PMEVTYPERn_EL0(n)), \ access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), } +static bool access_amu(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + kvm_inject_undefined(vcpu); + + return false; +} + +/* Macro to expand the AMU counter and type registers*/ +#define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), access_amu } +#define AMU_AMEVTYPE0_EL0(n) { SYS_DESC(SYS_AMEVTYPE0_EL0(n)), access_amu } +#define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), access_amu } +#define AMU_AMEVTYPE1_EL0(n) { SYS_DESC(SYS_AMEVTYPE1_EL0(n)), access_amu } + static bool trap_ptrauth(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *rd) @@ -1078,13 +1092,25 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, (u32)r->CRn, (u32)r->CRm, (u32)r->Op2); u64 val = raz ? 0 : read_sanitised_ftr_reg(id); - if (id == SYS_ID_AA64PFR0_EL1 && !vcpu_has_sve(vcpu)) { - val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT); + if (id == SYS_ID_AA64PFR0_EL1) { + if (!vcpu_has_sve(vcpu)) + val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT); + val &= ~(0xfUL << ID_AA64PFR0_AMU_SHIFT); } else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) { val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) | (0xfUL << ID_AA64ISAR1_API_SHIFT) | (0xfUL << ID_AA64ISAR1_GPA_SHIFT) | (0xfUL << ID_AA64ISAR1_GPI_SHIFT)); + } else if (id == SYS_ID_AA64DFR0_EL1) { + /* Limit guests to PMUv3 for ARMv8.1 */ + val = cpuid_feature_cap_perfmon_field(val, + ID_AA64DFR0_PMUVER_SHIFT, + ID_AA64DFR0_PMUVER_8_1); + } else if (id == SYS_ID_DFR0_EL1) { + /* Limit guests to PMUv3 for ARMv8.1 */ + val = cpuid_feature_cap_perfmon_field(val, + ID_DFR0_PERFMON_SHIFT, + ID_DFR0_PERFMON_8_1); } return val; @@ -1565,6 +1591,79 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 }, { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 }, + { SYS_DESC(SYS_AMCR_EL0), access_amu }, + { SYS_DESC(SYS_AMCFGR_EL0), access_amu }, + { SYS_DESC(SYS_AMCGCR_EL0), access_amu }, + { SYS_DESC(SYS_AMUSERENR_EL0), access_amu }, + { SYS_DESC(SYS_AMCNTENCLR0_EL0), access_amu }, + { SYS_DESC(SYS_AMCNTENSET0_EL0), access_amu }, + { SYS_DESC(SYS_AMCNTENCLR1_EL0), access_amu }, + { SYS_DESC(SYS_AMCNTENSET1_EL0), access_amu }, + AMU_AMEVCNTR0_EL0(0), + AMU_AMEVCNTR0_EL0(1), + AMU_AMEVCNTR0_EL0(2), + AMU_AMEVCNTR0_EL0(3), + AMU_AMEVCNTR0_EL0(4), + AMU_AMEVCNTR0_EL0(5), + AMU_AMEVCNTR0_EL0(6), + AMU_AMEVCNTR0_EL0(7), + AMU_AMEVCNTR0_EL0(8), + AMU_AMEVCNTR0_EL0(9), + AMU_AMEVCNTR0_EL0(10), + AMU_AMEVCNTR0_EL0(11), + AMU_AMEVCNTR0_EL0(12), + AMU_AMEVCNTR0_EL0(13), + AMU_AMEVCNTR0_EL0(14), + AMU_AMEVCNTR0_EL0(15), + AMU_AMEVTYPE0_EL0(0), + AMU_AMEVTYPE0_EL0(1), + AMU_AMEVTYPE0_EL0(2), + AMU_AMEVTYPE0_EL0(3), + AMU_AMEVTYPE0_EL0(4), + AMU_AMEVTYPE0_EL0(5), + AMU_AMEVTYPE0_EL0(6), + AMU_AMEVTYPE0_EL0(7), + AMU_AMEVTYPE0_EL0(8), + AMU_AMEVTYPE0_EL0(9), + AMU_AMEVTYPE0_EL0(10), + AMU_AMEVTYPE0_EL0(11), + AMU_AMEVTYPE0_EL0(12), + AMU_AMEVTYPE0_EL0(13), + AMU_AMEVTYPE0_EL0(14), + AMU_AMEVTYPE0_EL0(15), + AMU_AMEVCNTR1_EL0(0), + AMU_AMEVCNTR1_EL0(1), + AMU_AMEVCNTR1_EL0(2), + AMU_AMEVCNTR1_EL0(3), + AMU_AMEVCNTR1_EL0(4), + AMU_AMEVCNTR1_EL0(5), + AMU_AMEVCNTR1_EL0(6), + AMU_AMEVCNTR1_EL0(7), + AMU_AMEVCNTR1_EL0(8), + AMU_AMEVCNTR1_EL0(9), + AMU_AMEVCNTR1_EL0(10), + AMU_AMEVCNTR1_EL0(11), + AMU_AMEVCNTR1_EL0(12), + AMU_AMEVCNTR1_EL0(13), + AMU_AMEVCNTR1_EL0(14), + AMU_AMEVCNTR1_EL0(15), + AMU_AMEVTYPE1_EL0(0), + AMU_AMEVTYPE1_EL0(1), + AMU_AMEVTYPE1_EL0(2), + AMU_AMEVTYPE1_EL0(3), + AMU_AMEVTYPE1_EL0(4), + AMU_AMEVTYPE1_EL0(5), + AMU_AMEVTYPE1_EL0(6), + AMU_AMEVTYPE1_EL0(7), + AMU_AMEVTYPE1_EL0(8), + AMU_AMEVTYPE1_EL0(9), + AMU_AMEVTYPE1_EL0(10), + AMU_AMEVTYPE1_EL0(11), + AMU_AMEVTYPE1_EL0(12), + AMU_AMEVTYPE1_EL0(13), + AMU_AMEVTYPE1_EL0(14), + AMU_AMEVTYPE1_EL0(15), + { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer }, { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer }, { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer }, diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c index 1f82c66b32ea..60eccae2abad 100644 --- a/arch/arm64/lib/csum.c +++ b/arch/arm64/lib/csum.c @@ -124,3 +124,30 @@ unsigned int do_csum(const unsigned char *buff, int len) return sum >> 16; } + +__sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum csum) +{ + __uint128_t src, dst; + u64 sum = (__force u64)csum; + + src = *(const __uint128_t *)saddr->s6_addr; + dst = *(const __uint128_t *)daddr->s6_addr; + + sum += (__force u32)htonl(len); +#ifdef __LITTLE_ENDIAN + sum += (u32)proto << 24; +#else + sum += proto; +#endif + src += (src >> 64) | (src << 64); + dst += (dst >> 64) | (dst << 64); + + sum = accumulate(sum, src >> 64); + sum = accumulate(sum, dst >> 64); + + sum += ((sum >> 32) | (sum << 32)); + return csum_fold((__force __wsum)(sum >> 32)); +} +EXPORT_SYMBOL(csum_ipv6_magic); diff --git a/arch/arm64/lib/strcmp.S b/arch/arm64/lib/strcmp.S index 4767540d1b94..4e79566726c8 100644 --- a/arch/arm64/lib/strcmp.S +++ b/arch/arm64/lib/strcmp.S @@ -186,7 +186,7 @@ CPU_LE( rev data2, data2 ) * as carry-propagation can corrupt the upper bits if the trailing * bytes in the string contain 0x01. * However, if there is no NUL byte in the dword, we can generate - * the result directly. We ca not just subtract the bytes as the + * the result directly. We cannot just subtract the bytes as the * MSB might be significant. */ CPU_BE( cbnz has_nul, 1f ) diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index d89bb22589f6..9b26f9a88724 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -6,6 +6,7 @@ * Copyright (C) 2012 ARM Ltd. */ +#include <linux/bitfield.h> #include <linux/bitops.h> #include <linux/sched.h> #include <linux/slab.h> @@ -254,10 +255,37 @@ switch_mm_fastpath: /* Errata workaround post TTBRx_EL1 update. */ asmlinkage void post_ttbr_update_workaround(void) { + if (!IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456)) + return; + asm(ALTERNATIVE("nop; nop; nop", "ic iallu; dsb nsh; isb", - ARM64_WORKAROUND_CAVIUM_27456, - CONFIG_CAVIUM_ERRATUM_27456)); + ARM64_WORKAROUND_CAVIUM_27456)); +} + +void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) +{ + unsigned long ttbr1 = read_sysreg(ttbr1_el1); + unsigned long asid = ASID(mm); + unsigned long ttbr0 = phys_to_ttbr(pgd_phys); + + /* Skip CNP for the reserved ASID */ + if (system_supports_cnp() && asid) + ttbr0 |= TTBR_CNP_BIT; + + /* SW PAN needs a copy of the ASID in TTBR0 for entry */ + if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN)) + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); + + /* Set ASID in TTBR1 since TCR.A1 is set */ + ttbr1 &= ~TTBR_ASID_MASK; + ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); + + write_sysreg(ttbr1, ttbr1_el1); + isb(); + write_sysreg(ttbr0, ttbr0_el1); + isb(); + post_ttbr_update_workaround(); } static int asids_update_limit(void) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 128f70852bf3..9b08f7c7e6f0 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -17,6 +17,7 @@ #include <linux/mman.h> #include <linux/nodemask.h> #include <linux/memblock.h> +#include <linux/memory.h> #include <linux/fs.h> #include <linux/io.h> #include <linux/mm.h> @@ -724,6 +725,312 @@ int kern_addr_valid(unsigned long addr) return pfn_valid(pte_pfn(pte)); } + +#ifdef CONFIG_MEMORY_HOTPLUG +static void free_hotplug_page_range(struct page *page, size_t size) +{ + WARN_ON(PageReserved(page)); + free_pages((unsigned long)page_address(page), get_order(size)); +} + +static void free_hotplug_pgtable_page(struct page *page) +{ + free_hotplug_page_range(page, PAGE_SIZE); +} + +static bool pgtable_range_aligned(unsigned long start, unsigned long end, + unsigned long floor, unsigned long ceiling, + unsigned long mask) +{ + start &= mask; + if (start < floor) + return false; + + if (ceiling) { + ceiling &= mask; + if (!ceiling) + return false; + } + + if (end - 1 > ceiling - 1) + return false; + return true; +} + +static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr, + unsigned long end, bool free_mapped) +{ + pte_t *ptep, pte; + + do { + ptep = pte_offset_kernel(pmdp, addr); + pte = READ_ONCE(*ptep); + if (pte_none(pte)) + continue; + + WARN_ON(!pte_present(pte)); + pte_clear(&init_mm, addr, ptep); + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); + if (free_mapped) + free_hotplug_page_range(pte_page(pte), PAGE_SIZE); + } while (addr += PAGE_SIZE, addr < end); +} + +static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, + unsigned long end, bool free_mapped) +{ + unsigned long next; + pmd_t *pmdp, pmd; + + do { + next = pmd_addr_end(addr, end); + pmdp = pmd_offset(pudp, addr); + pmd = READ_ONCE(*pmdp); + if (pmd_none(pmd)) + continue; + + WARN_ON(!pmd_present(pmd)); + if (pmd_sect(pmd)) { + pmd_clear(pmdp); + + /* + * One TLBI should be sufficient here as the PMD_SIZE + * range is mapped with a single block entry. + */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); + if (free_mapped) + free_hotplug_page_range(pmd_page(pmd), + PMD_SIZE); + continue; + } + WARN_ON(!pmd_table(pmd)); + unmap_hotplug_pte_range(pmdp, addr, next, free_mapped); + } while (addr = next, addr < end); +} + +static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, + unsigned long end, bool free_mapped) +{ + unsigned long next; + pud_t *pudp, pud; + + do { + next = pud_addr_end(addr, end); + pudp = pud_offset(p4dp, addr); + pud = READ_ONCE(*pudp); + if (pud_none(pud)) + continue; + + WARN_ON(!pud_present(pud)); + if (pud_sect(pud)) { + pud_clear(pudp); + + /* + * One TLBI should be sufficient here as the PUD_SIZE + * range is mapped with a single block entry. + */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); + if (free_mapped) + free_hotplug_page_range(pud_page(pud), + PUD_SIZE); + continue; + } + WARN_ON(!pud_table(pud)); + unmap_hotplug_pmd_range(pudp, addr, next, free_mapped); + } while (addr = next, addr < end); +} + +static void unmap_hotplug_p4d_range(pgd_t *pgdp, unsigned long addr, + unsigned long end, bool free_mapped) +{ + unsigned long next; + p4d_t *p4dp, p4d; + + do { + next = p4d_addr_end(addr, end); + p4dp = p4d_offset(pgdp, addr); + p4d = READ_ONCE(*p4dp); + if (p4d_none(p4d)) + continue; + + WARN_ON(!p4d_present(p4d)); + unmap_hotplug_pud_range(p4dp, addr, next, free_mapped); + } while (addr = next, addr < end); +} + +static void unmap_hotplug_range(unsigned long addr, unsigned long end, + bool free_mapped) +{ + unsigned long next; + pgd_t *pgdp, pgd; + + do { + next = pgd_addr_end(addr, end); + pgdp = pgd_offset_k(addr); + pgd = READ_ONCE(*pgdp); + if (pgd_none(pgd)) + continue; + + WARN_ON(!pgd_present(pgd)); + unmap_hotplug_p4d_range(pgdp, addr, next, free_mapped); + } while (addr = next, addr < end); +} + +static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned long floor, + unsigned long ceiling) +{ + pte_t *ptep, pte; + unsigned long i, start = addr; + + do { + ptep = pte_offset_kernel(pmdp, addr); + pte = READ_ONCE(*ptep); + + /* + * This is just a sanity check here which verifies that + * pte clearing has been done by earlier unmap loops. + */ + WARN_ON(!pte_none(pte)); + } while (addr += PAGE_SIZE, addr < end); + + if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK)) + return; + + /* + * Check whether we can free the pte page if the rest of the + * entries are empty. Overlap with other regions have been + * handled by the floor/ceiling check. + */ + ptep = pte_offset_kernel(pmdp, 0UL); + for (i = 0; i < PTRS_PER_PTE; i++) { + if (!pte_none(READ_ONCE(ptep[i]))) + return; + } + + pmd_clear(pmdp); + __flush_tlb_kernel_pgtable(start); + free_hotplug_pgtable_page(virt_to_page(ptep)); +} + +static void free_empty_pmd_table(pud_t *pudp, unsigned long addr, + unsigned long end, unsigned long floor, + unsigned long ceiling) +{ + pmd_t *pmdp, pmd; + unsigned long i, next, start = addr; + + do { + next = pmd_addr_end(addr, end); + pmdp = pmd_offset(pudp, addr); + pmd = READ_ONCE(*pmdp); + if (pmd_none(pmd)) + continue; + + WARN_ON(!pmd_present(pmd) || !pmd_table(pmd) || pmd_sect(pmd)); + free_empty_pte_table(pmdp, addr, next, floor, ceiling); + } while (addr = next, addr < end); + + if (CONFIG_PGTABLE_LEVELS <= 2) + return; + + if (!pgtable_range_aligned(start, end, floor, ceiling, PUD_MASK)) + return; + + /* + * Check whether we can free the pmd page if the rest of the + * entries are empty. Overlap with other regions have been + * handled by the floor/ceiling check. + */ + pmdp = pmd_offset(pudp, 0UL); + for (i = 0; i < PTRS_PER_PMD; i++) { + if (!pmd_none(READ_ONCE(pmdp[i]))) + return; + } + + pud_clear(pudp); + __flush_tlb_kernel_pgtable(start); + free_hotplug_pgtable_page(virt_to_page(pmdp)); +} + +static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr, + unsigned long end, unsigned long floor, + unsigned long ceiling) +{ + pud_t *pudp, pud; + unsigned long i, next, start = addr; + + do { + next = pud_addr_end(addr, end); + pudp = pud_offset(p4dp, addr); + pud = READ_ONCE(*pudp); + if (pud_none(pud)) + continue; + + WARN_ON(!pud_present(pud) || !pud_table(pud) || pud_sect(pud)); + free_empty_pmd_table(pudp, addr, next, floor, ceiling); + } while (addr = next, addr < end); + + if (CONFIG_PGTABLE_LEVELS <= 3) + return; + + if (!pgtable_range_aligned(start, end, floor, ceiling, PGDIR_MASK)) + return; + + /* + * Check whether we can free the pud page if the rest of the + * entries are empty. Overlap with other regions have been + * handled by the floor/ceiling check. + */ + pudp = pud_offset(p4dp, 0UL); + for (i = 0; i < PTRS_PER_PUD; i++) { + if (!pud_none(READ_ONCE(pudp[i]))) + return; + } + + p4d_clear(p4dp); + __flush_tlb_kernel_pgtable(start); + free_hotplug_pgtable_page(virt_to_page(pudp)); +} + +static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr, + unsigned long end, unsigned long floor, + unsigned long ceiling) +{ + unsigned long next; + p4d_t *p4dp, p4d; + + do { + next = p4d_addr_end(addr, end); + p4dp = p4d_offset(pgdp, addr); + p4d = READ_ONCE(*p4dp); + if (p4d_none(p4d)) + continue; + + WARN_ON(!p4d_present(p4d)); + free_empty_pud_table(p4dp, addr, next, floor, ceiling); + } while (addr = next, addr < end); +} + +static void free_empty_tables(unsigned long addr, unsigned long end, + unsigned long floor, unsigned long ceiling) +{ + unsigned long next; + pgd_t *pgdp, pgd; + + do { + next = pgd_addr_end(addr, end); + pgdp = pgd_offset_k(addr); + pgd = READ_ONCE(*pgdp); + if (pgd_none(pgd)) + continue; + + WARN_ON(!pgd_present(pgd)); + free_empty_p4d_table(pgdp, addr, next, floor, ceiling); + } while (addr = next, addr < end); +} +#endif + #ifdef CONFIG_SPARSEMEM_VMEMMAP #if !ARM64_SWAPPER_USES_SECTION_MAPS int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, @@ -771,6 +1078,12 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap) { +#ifdef CONFIG_MEMORY_HOTPLUG + WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); + + unmap_hotplug_range(start, end, true); + free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END); +#endif } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ @@ -1049,10 +1362,21 @@ int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) } #ifdef CONFIG_MEMORY_HOTPLUG +static void __remove_pgd_mapping(pgd_t *pgdir, unsigned long start, u64 size) +{ + unsigned long end = start + size; + + WARN_ON(pgdir != init_mm.pgd); + WARN_ON((start < PAGE_OFFSET) || (end > PAGE_END)); + + unmap_hotplug_range(start, end, false); + free_empty_tables(start, end, PAGE_OFFSET, PAGE_END); +} + int arch_add_memory(int nid, u64 start, u64 size, struct mhp_restrictions *restrictions) { - int flags = 0; + int ret, flags = 0; if (rodata_full || debug_pagealloc_enabled()) flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; @@ -1062,22 +1386,59 @@ int arch_add_memory(int nid, u64 start, u64 size, memblock_clear_nomap(start, size); - return __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT, + ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT, restrictions); + if (ret) + __remove_pgd_mapping(swapper_pg_dir, + __phys_to_virt(start), size); + return ret; } + void arch_remove_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap) { unsigned long start_pfn = start >> PAGE_SHIFT; unsigned long nr_pages = size >> PAGE_SHIFT; - /* - * FIXME: Cleanup page tables (also in arch_add_memory() in case - * adding fails). Until then, this function should only be used - * during memory hotplug (adding memory), not for memory - * unplug. ARCH_ENABLE_MEMORY_HOTREMOVE must not be - * unlocked yet. - */ __remove_pages(start_pfn, nr_pages, altmap); + __remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size); +} + +/* + * This memory hotplug notifier helps prevent boot memory from being + * inadvertently removed as it blocks pfn range offlining process in + * __offline_pages(). Hence this prevents both offlining as well as + * removal process for boot memory which is initially always online. + * In future if and when boot memory could be removed, this notifier + * should be dropped and free_hotplug_page_range() should handle any + * reserved pages allocated during boot. + */ +static int prevent_bootmem_remove_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct mem_section *ms; + struct memory_notify *arg = data; + unsigned long end_pfn = arg->start_pfn + arg->nr_pages; + unsigned long pfn = arg->start_pfn; + + if (action != MEM_GOING_OFFLINE) + return NOTIFY_OK; + + for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { + ms = __pfn_to_section(pfn); + if (early_section(ms)) + return NOTIFY_BAD; + } + return NOTIFY_OK; +} + +static struct notifier_block prevent_bootmem_remove_nb = { + .notifier_call = prevent_bootmem_remove_notifier, +}; + +static int __init prevent_bootmem_remove_init(void) +{ + return register_memory_notifier(&prevent_bootmem_remove_nb); } +device_initcall(prevent_bootmem_remove_init); #endif diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index aafed6902411..197a9ba2d5ea 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -11,11 +11,13 @@ #include <linux/linkage.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> +#include <asm/asm_pointer_auth.h> #include <asm/hwcap.h> #include <asm/pgtable.h> #include <asm/pgtable-hwdef.h> #include <asm/cpufeature.h> #include <asm/alternative.h> +#include <asm/smp.h> #ifdef CONFIG_ARM64_64K_PAGES #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K @@ -131,45 +133,19 @@ alternative_endif ubfx x11, x11, #1, #1 msr oslar_el1, x11 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 + reset_amuserenr_el0 x0 // Disable AMU access from EL0 alternative_if ARM64_HAS_RAS_EXTN msr_s SYS_DISR_EL1, xzr alternative_else_nop_endif + ptrauth_keys_install_kernel x14, 0, x1, x2, x3 isb ret SYM_FUNC_END(cpu_do_resume) .popsection #endif -/* - * cpu_do_switch_mm(pgd_phys, tsk) - * - * Set the translation table base pointer to be pgd_phys. - * - * - pgd_phys - physical address of new TTB - */ -SYM_FUNC_START(cpu_do_switch_mm) - mrs x2, ttbr1_el1 - mmid x1, x1 // get mm->context.id - phys_to_ttbr x3, x0 - -alternative_if ARM64_HAS_CNP - cbz x1, 1f // skip CNP for reserved ASID - orr x3, x3, #TTBR_CNP_BIT -1: -alternative_else_nop_endif -#ifdef CONFIG_ARM64_SW_TTBR0_PAN - bfi x3, x1, #48, #16 // set the ASID field in TTBR0 -#endif - bfi x2, x1, #48, #16 // set the ASID - msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set) - isb - msr ttbr0_el1, x3 // now update TTBR0 - isb - b post_ttbr_update_workaround // Back to C code... -SYM_FUNC_END(cpu_do_switch_mm) - .pushsection ".idmap.text", "awx" .macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 @@ -408,35 +384,37 @@ SYM_FUNC_END(idmap_kpti_install_ng_mappings) /* * __cpu_setup * - * Initialise the processor for turning the MMU on. Return in x0 the - * value of the SCTLR_EL1 register. + * Initialise the processor for turning the MMU on. + * + * Input: + * x0 with a flag ARM64_CPU_BOOT_PRIMARY/ARM64_CPU_BOOT_SECONDARY/ARM64_CPU_RUNTIME. + * Output: + * Return in x0 the value of the SCTLR_EL1 register. */ .pushsection ".idmap.text", "awx" SYM_FUNC_START(__cpu_setup) tlbi vmalle1 // Invalidate local TLB dsb nsh - mov x0, #3 << 20 - msr cpacr_el1, x0 // Enable FP/ASIMD - mov x0, #1 << 12 // Reset mdscr_el1 and disable - msr mdscr_el1, x0 // access to the DCC from EL0 + mov x1, #3 << 20 + msr cpacr_el1, x1 // Enable FP/ASIMD + mov x1, #1 << 12 // Reset mdscr_el1 and disable + msr mdscr_el1, x1 // access to the DCC from EL0 isb // Unmask debug exceptions now, enable_dbg // since this is per-cpu - reset_pmuserenr_el0 x0 // Disable PMU access from EL0 + reset_pmuserenr_el0 x1 // Disable PMU access from EL0 + reset_amuserenr_el0 x1 // Disable AMU access from EL0 + /* * Memory region attributes */ mov_q x5, MAIR_EL1_SET msr mair_el1, x5 /* - * Prepare SCTLR - */ - mov_q x0, SCTLR_EL1_SET - /* * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for * both user and kernel. */ - ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ + mov_q x10, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ TCR_TBI0 | TCR_A1 | TCR_KASAN_FLAGS tcr_clear_errata_bits x10, x9, x5 @@ -468,5 +446,51 @@ SYM_FUNC_START(__cpu_setup) 1: #endif /* CONFIG_ARM64_HW_AFDBM */ msr tcr_el1, x10 + mov x1, x0 + /* + * Prepare SCTLR + */ + mov_q x0, SCTLR_EL1_SET + +#ifdef CONFIG_ARM64_PTR_AUTH + /* No ptrauth setup for run time cpus */ + cmp x1, #ARM64_CPU_RUNTIME + b.eq 3f + + /* Check if the CPU supports ptrauth */ + mrs x2, id_aa64isar1_el1 + ubfx x2, x2, #ID_AA64ISAR1_APA_SHIFT, #8 + cbz x2, 3f + + /* + * The primary cpu keys are reset here and can be + * re-initialised with some proper values later. + */ + msr_s SYS_APIAKEYLO_EL1, xzr + msr_s SYS_APIAKEYHI_EL1, xzr + + /* Just enable ptrauth for primary cpu */ + cmp x1, #ARM64_CPU_BOOT_PRIMARY + b.eq 2f + + /* if !system_supports_address_auth() then skip enable */ +alternative_if_not ARM64_HAS_ADDRESS_AUTH + b 3f +alternative_else_nop_endif + + /* Install ptrauth key for secondary cpus */ + adr_l x2, secondary_data + ldr x3, [x2, #CPU_BOOT_TASK] // get secondary_data.task + cbz x3, 2f // check for slow booting cpus + ldp x3, x4, [x2, #CPU_BOOT_PTRAUTH_KEY] + msr_s SYS_APIAKEYLO_EL1, x3 + msr_s SYS_APIAKEYHI_EL1, x4 + +2: /* Enable ptrauth instructions */ + ldr x2, =SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \ + SCTLR_ELx_ENDA | SCTLR_ELx_ENDB + orr x0, x0, x2 +3: +#endif ret // return to head.S SYM_FUNC_END(__cpu_setup) diff --git a/arch/arm64/mm/ptdump_debugfs.c b/arch/arm64/mm/ptdump_debugfs.c index 1f2eae3e988b..d29d722ec3ec 100644 --- a/arch/arm64/mm/ptdump_debugfs.c +++ b/arch/arm64/mm/ptdump_debugfs.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/debugfs.h> +#include <linux/memory_hotplug.h> #include <linux/seq_file.h> #include <asm/ptdump.h> @@ -7,7 +8,10 @@ static int ptdump_show(struct seq_file *m, void *v) { struct ptdump_info *info = m->private; + + get_online_mems(); ptdump_walk(m, info); + put_online_mems(); return 0; } DEFINE_SHOW_ATTRIBUTE(ptdump); diff --git a/arch/c6x/platforms/timer64.c b/arch/c6x/platforms/timer64.c index d98d94303498..661f4c7c6ef6 100644 --- a/arch/c6x/platforms/timer64.c +++ b/arch/c6x/platforms/timer64.c @@ -165,13 +165,6 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction timer_iact = { - .name = "timer", - .flags = IRQF_TIMER, - .handler = timer_interrupt, - .dev_id = &t64_clockevent_device, -}; - void __init timer64_init(void) { struct clock_event_device *cd = &t64_clockevent_device; @@ -238,7 +231,9 @@ void __init timer64_init(void) cd->cpumask = cpumask_of(smp_processor_id()); clockevents_register_device(cd); - setup_irq(cd->irq, &timer_iact); + if (request_irq(cd->irq, timer_interrupt, IRQF_TIMER, "timer", + &t64_clockevent_device)) + pr_err("Failed to request irq %d (timer)\n", cd->irq); out: of_node_put(np); diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h index eaa1c3403a42..abefa125b93c 100644 --- a/arch/csky/include/asm/uaccess.h +++ b/arch/csky/include/asm/uaccess.h @@ -11,7 +11,6 @@ #include <linux/errno.h> #include <linux/types.h> #include <linux/sched.h> -#include <linux/mm.h> #include <linux/string.h> #include <linux/version.h> #include <asm/segment.h> diff --git a/arch/hexagon/include/asm/futex.h b/arch/hexagon/include/asm/futex.h index 0191f7c7193e..6b9c554aee78 100644 --- a/arch/hexagon/include/asm/futex.h +++ b/arch/hexagon/include/asm/futex.h @@ -36,7 +36,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -62,8 +63,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index 00cb38faad0c..c1019a736ff1 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -10,7 +10,6 @@ /* * User space memory access functions */ -#include <linux/mm.h> #include <asm/sections.h> /* diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c index 0bbbe652a513..619c56420aa0 100644 --- a/arch/hexagon/kernel/smp.c +++ b/arch/hexagon/kernel/smp.c @@ -114,12 +114,6 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg) local_irq_restore(flags); } -static struct irqaction ipi_intdesc = { - .handler = handle_ipi, - .flags = IRQF_TRIGGER_RISING, - .name = "ipi_handler" -}; - void __init smp_prepare_boot_cpu(void) { } @@ -132,8 +126,8 @@ void __init smp_prepare_boot_cpu(void) void start_secondary(void) { - unsigned int cpu; unsigned long thread_ptr; + unsigned int cpu, irq; /* Calculate thread_info pointer from stack pointer */ __asm__ __volatile__( @@ -155,7 +149,10 @@ void start_secondary(void) cpu = smp_processor_id(); - setup_irq(BASE_IPI_IRQ + cpu, &ipi_intdesc); + irq = BASE_IPI_IRQ + cpu; + if (request_irq(irq, handle_ipi, IRQF_TRIGGER_RISING, "ipi_handler", + NULL)) + pr_err("Failed to request irq %u (ipi_handler)\n", irq); /* Register the clock_event dummy */ setup_percpu_clockdev(); @@ -201,7 +198,7 @@ void __init smp_cpus_done(unsigned int max_cpus) void __init smp_prepare_cpus(unsigned int max_cpus) { - int i; + int i, irq = BASE_IPI_IRQ; /* * should eventually have some sort of machine @@ -213,8 +210,11 @@ void __init smp_prepare_cpus(unsigned int max_cpus) set_cpu_present(i, true); /* Also need to register the interrupts for IPI */ - if (max_cpus > 1) - setup_irq(BASE_IPI_IRQ, &ipi_intdesc); + if (max_cpus > 1) { + if (request_irq(irq, handle_ipi, IRQF_TRIGGER_RISING, + "ipi_handler", NULL)) + pr_err("Failed to request irq %d (ipi_handler)\n", irq); + } } void smp_send_reschedule(int cpu) diff --git a/arch/hexagon/kernel/time.c b/arch/hexagon/kernel/time.c index f99e9257bed4..feffe527ac92 100644 --- a/arch/hexagon/kernel/time.c +++ b/arch/hexagon/kernel/time.c @@ -143,13 +143,6 @@ static irqreturn_t timer_interrupt(int irq, void *devid) return IRQ_HANDLED; } -/* This should also be pulled from devtree */ -static struct irqaction rtos_timer_intdesc = { - .handler = timer_interrupt, - .flags = IRQF_TIMER | IRQF_TRIGGER_RISING, - .name = "rtos_timer" -}; - /* * time_init_deferred - called by start_kernel to set up timer/clock source * @@ -163,6 +156,7 @@ void __init time_init_deferred(void) { struct resource *resource = NULL; struct clock_event_device *ce_dev = &hexagon_clockevent_dev; + unsigned long flag = IRQF_TIMER | IRQF_TRIGGER_RISING; ce_dev->cpumask = cpu_all_mask; @@ -195,7 +189,8 @@ void __init time_init_deferred(void) #endif clockevents_register_device(ce_dev); - setup_irq(ce_dev->irq, &rtos_timer_intdesc); + if (request_irq(ce_dev->irq, timer_interrupt, flag, "rtos_timer", NULL)) + pr_err("Failed to register rtos_timer interrupt\n"); } void __init time_init(void) diff --git a/arch/ia64/include/asm/futex.h b/arch/ia64/include/asm/futex.h index 2e106d462196..1db26b432d8c 100644 --- a/arch/ia64/include/asm/futex.h +++ b/arch/ia64/include/asm/futex.h @@ -50,7 +50,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -74,8 +75,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/ia64/include/asm/hw_irq.h b/arch/ia64/include/asm/hw_irq.h index e6385c7bdeb0..f6ff95b4ecb1 100644 --- a/arch/ia64/include/asm/hw_irq.h +++ b/arch/ia64/include/asm/hw_irq.h @@ -113,7 +113,6 @@ extern struct irq_chip irq_type_ia64_lsapic; /* CPU-internal interrupt controlle #define ia64_register_ipi ia64_native_register_ipi #define assign_irq_vector ia64_native_assign_irq_vector #define free_irq_vector ia64_native_free_irq_vector -#define register_percpu_irq ia64_native_register_percpu_irq #define ia64_resend_irq ia64_native_resend_irq extern void ia64_native_register_ipi(void); @@ -123,7 +122,6 @@ extern void ia64_native_free_irq_vector (int vector); extern int reserve_irq_vector (int vector); extern void __setup_vector_irq(int cpu); extern void ia64_send_ipi (int cpu, int vector, int delivery_mode, int redirect); -extern void ia64_native_register_percpu_irq (ia64_vector vec, struct irqaction *action); extern void destroy_and_reserve_irq (unsigned int irq); #ifdef CONFIG_SMP diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index 89782ad3fb88..5c7e79eccaee 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -35,7 +35,6 @@ #include <linux/compiler.h> #include <linux/page-flags.h> -#include <linux/mm.h> #include <asm/intrinsics.h> #include <asm/pgtable.h> diff --git a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c index 0a34dcc435c6..f69f3fe0532e 100644 --- a/arch/ia64/kernel/efi.c +++ b/arch/ia64/kernel/efi.c @@ -45,11 +45,21 @@ #define EFI_DEBUG 0 +#define ESI_TABLE_GUID \ + EFI_GUID(0x43EA58DC, 0xCF28, 0x4b06, 0xB3, \ + 0x91, 0xB7, 0x50, 0x59, 0x34, 0x2B, 0xD4) + +static unsigned long mps_phys = EFI_INVALID_TABLE_ADDR; static __initdata unsigned long palo_phys; +unsigned long __initdata esi_phys = EFI_INVALID_TABLE_ADDR; +unsigned long hcdp_phys = EFI_INVALID_TABLE_ADDR; unsigned long sal_systab_phys = EFI_INVALID_TABLE_ADDR; -static __initdata efi_config_table_type_t arch_tables[] = { +static const efi_config_table_type_t arch_tables[] __initconst = { + {ESI_TABLE_GUID, "ESI", &esi_phys}, + {HCDP_TABLE_GUID, "HCDP", &hcdp_phys}, + {MPS_TABLE_GUID, "MPS", &mps_phys}, {PROCESSOR_ABSTRACTION_LAYER_OVERWRITE_GUID, "PALO", &palo_phys}, {SAL_SYSTEM_TABLE_GUID, "SALsystab", &sal_systab_phys}, {NULL_GUID, NULL, 0}, @@ -474,11 +484,10 @@ efi_map_pal_code (void) void __init efi_init (void) { + const efi_system_table_t *efi_systab; void *efi_map_start, *efi_map_end; - efi_char16_t *c16; u64 efi_desc_size; - char *cp, vendor[100] = "unknown"; - int i; + char *cp; set_bit(EFI_BOOT, &efi.flags); set_bit(EFI_64BIT, &efi.flags); @@ -508,42 +517,29 @@ efi_init (void) printk(KERN_INFO "Ignoring memory above %lluMB\n", max_addr >> 20); - efi.systab = __va(ia64_boot_param->efi_systab); + efi_systab = __va(ia64_boot_param->efi_systab); /* * Verify the EFI Table */ - if (efi.systab == NULL) + if (efi_systab == NULL) panic("Whoa! Can't find EFI system table.\n"); - if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) + if (efi_systab_check_header(&efi_systab->hdr, 1)) panic("Whoa! EFI system table signature incorrect\n"); - if ((efi.systab->hdr.revision >> 16) == 0) - printk(KERN_WARNING "Warning: EFI system table version " - "%d.%02d, expected 1.00 or greater\n", - efi.systab->hdr.revision >> 16, - efi.systab->hdr.revision & 0xffff); - - /* Show what we know for posterity */ - c16 = __va(efi.systab->fw_vendor); - if (c16) { - for (i = 0;i < (int) sizeof(vendor) - 1 && *c16; ++i) - vendor[i] = *c16++; - vendor[i] = '\0'; - } - printk(KERN_INFO "EFI v%u.%.02u by %s:", - efi.systab->hdr.revision >> 16, - efi.systab->hdr.revision & 0xffff, vendor); + efi_systab_report_header(&efi_systab->hdr, efi_systab->fw_vendor); palo_phys = EFI_INVALID_TABLE_ADDR; - if (efi_config_init(arch_tables) != 0) + if (efi_config_parse_tables(__va(efi_systab->tables), + efi_systab->nr_tables, + arch_tables) != 0) return; if (palo_phys != EFI_INVALID_TABLE_ADDR) handle_palo(palo_phys); - runtime = __va(efi.systab->runtime); + runtime = __va(efi_systab->runtime); efi.get_time = phys_get_time; efi.set_time = phys_set_time; efi.get_wakeup_time = phys_get_wakeup_time; @@ -1351,3 +1347,12 @@ vmcore_find_descriptor_size (unsigned long address) return ret; } #endif + +char *efi_systab_show_arch(char *str) +{ + if (mps_phys != EFI_INVALID_TABLE_ADDR) + str += sprintf(str, "MPS=0x%lx\n", mps_phys); + if (hcdp_phys != EFI_INVALID_TABLE_ADDR) + str += sprintf(str, "HCDP=0x%lx\n", hcdp_phys); + return str; +} diff --git a/arch/ia64/kernel/esi.c b/arch/ia64/kernel/esi.c index cb514126ef7f..4df57c93e0a8 100644 --- a/arch/ia64/kernel/esi.c +++ b/arch/ia64/kernel/esi.c @@ -19,10 +19,6 @@ MODULE_LICENSE("GPL"); #define MODULE_NAME "esi" -#define ESI_TABLE_GUID \ - EFI_GUID(0x43EA58DC, 0xCF28, 0x4b06, 0xB3, \ - 0x91, 0xB7, 0x50, 0x59, 0x34, 0x2B, 0xD4) - enum esi_systab_entry_type { ESI_DESC_ENTRY_POINT = 0 }; @@ -48,27 +44,18 @@ struct pdesc { static struct ia64_sal_systab *esi_systab; +extern unsigned long esi_phys; + static int __init esi_init (void) { - efi_config_table_t *config_tables; struct ia64_sal_systab *systab; - unsigned long esi = 0; char *p; int i; - config_tables = __va(efi.systab->tables); - - for (i = 0; i < (int) efi.systab->nr_tables; ++i) { - if (efi_guidcmp(config_tables[i].guid, ESI_TABLE_GUID) == 0) { - esi = config_tables[i].table; - break; - } - } - - if (!esi) + if (esi_phys == EFI_INVALID_TABLE_ADDR) return -ENODEV; - systab = __va(esi); + systab = __va(esi_phys); if (strncmp(systab->signature, "ESIT", 4) != 0) { printk(KERN_ERR "bad signature in ESI system table!"); diff --git a/arch/ia64/kernel/irq.h b/arch/ia64/kernel/irq.h new file mode 100644 index 000000000000..4d16f3cbeb1d --- /dev/null +++ b/arch/ia64/kernel/irq.h @@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +extern void register_percpu_irq(ia64_vector vec, irq_handler_t handler, + unsigned long flags, const char *name); diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c index 8e91c86e8072..e7862e4cb1e7 100644 --- a/arch/ia64/kernel/irq_ia64.c +++ b/arch/ia64/kernel/irq_ia64.c @@ -351,11 +351,6 @@ static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq_move_irqaction = { - .handler = smp_irq_move_cleanup_interrupt, - .name = "irq_move" -}; - static int __init parse_vector_domain(char *arg) { if (!arg) @@ -586,28 +581,15 @@ static irqreturn_t dummy_handler (int irq, void *dev_id) return IRQ_NONE; } -static struct irqaction ipi_irqaction = { - .handler = handle_IPI, - .name = "IPI" -}; - /* * KVM uses this interrupt to force a cpu out of guest mode */ -static struct irqaction resched_irqaction = { - .handler = dummy_handler, - .name = "resched" -}; - -static struct irqaction tlb_irqaction = { - .handler = dummy_handler, - .name = "tlb_flush" -}; #endif void -ia64_native_register_percpu_irq (ia64_vector vec, struct irqaction *action) +register_percpu_irq(ia64_vector vec, irq_handler_t handler, unsigned long flags, + const char *name) { unsigned int irq; @@ -615,8 +597,9 @@ ia64_native_register_percpu_irq (ia64_vector vec, struct irqaction *action) BUG_ON(bind_irq_vector(irq, vec, CPU_MASK_ALL)); irq_set_status_flags(irq, IRQ_PER_CPU); irq_set_chip(irq, &irq_type_ia64_lsapic); - if (action) - setup_irq(irq, action); + if (handler) + if (request_irq(irq, handler, flags, name, NULL)) + pr_err("Failed to request irq %u (%s)\n", irq, name); irq_set_handler(irq, handle_percpu_irq); } @@ -624,9 +607,10 @@ void __init ia64_native_register_ipi(void) { #ifdef CONFIG_SMP - register_percpu_irq(IA64_IPI_VECTOR, &ipi_irqaction); - register_percpu_irq(IA64_IPI_RESCHEDULE, &resched_irqaction); - register_percpu_irq(IA64_IPI_LOCAL_TLB_FLUSH, &tlb_irqaction); + register_percpu_irq(IA64_IPI_VECTOR, handle_IPI, 0, "IPI"); + register_percpu_irq(IA64_IPI_RESCHEDULE, dummy_handler, 0, "resched"); + register_percpu_irq(IA64_IPI_LOCAL_TLB_FLUSH, dummy_handler, 0, + "tlb_flush"); #endif } @@ -635,10 +619,13 @@ init_IRQ (void) { acpi_boot_init(); ia64_register_ipi(); - register_percpu_irq(IA64_SPURIOUS_INT_VECTOR, NULL); + register_percpu_irq(IA64_SPURIOUS_INT_VECTOR, NULL, 0, NULL); #ifdef CONFIG_SMP - if (vector_domain_type != VECTOR_DOMAIN_NONE) - register_percpu_irq(IA64_IRQ_MOVE_VECTOR, &irq_move_irqaction); + if (vector_domain_type != VECTOR_DOMAIN_NONE) { + register_percpu_irq(IA64_IRQ_MOVE_VECTOR, + smp_irq_move_cleanup_interrupt, 0, + "irq_move"); + } #endif #ifdef CONFIG_PERFMON pfm_init_percpu(); diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index bf2cb9294795..6fb54dfa1350 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -104,6 +104,7 @@ #include "mca_drv.h" #include "entry.h" +#include "irq.h" #if defined(IA64_MCA_DEBUG_INFO) # define IA64_MCA_DEBUG(fmt...) printk(fmt) @@ -1766,36 +1767,6 @@ ia64_mca_disable_cpe_polling(char *str) __setup("disable_cpe_poll", ia64_mca_disable_cpe_polling); -static struct irqaction cmci_irqaction = { - .handler = ia64_mca_cmc_int_handler, - .name = "cmc_hndlr" -}; - -static struct irqaction cmcp_irqaction = { - .handler = ia64_mca_cmc_int_caller, - .name = "cmc_poll" -}; - -static struct irqaction mca_rdzv_irqaction = { - .handler = ia64_mca_rendez_int_handler, - .name = "mca_rdzv" -}; - -static struct irqaction mca_wkup_irqaction = { - .handler = ia64_mca_wakeup_int_handler, - .name = "mca_wkup" -}; - -static struct irqaction mca_cpe_irqaction = { - .handler = ia64_mca_cpe_int_handler, - .name = "cpe_hndlr" -}; - -static struct irqaction mca_cpep_irqaction = { - .handler = ia64_mca_cpe_int_caller, - .name = "cpe_poll" -}; - /* Minimal format of the MCA/INIT stacks. The pseudo processes that run on * these stacks can never sleep, they cannot return from the kernel to user * space, they do not appear in a normal ps listing. So there is no need to @@ -2056,18 +2027,23 @@ void __init ia64_mca_irq_init(void) * Configure the CMCI/P vector and handler. Interrupts for CMC are * per-processor, so AP CMC interrupts are setup in smp_callin() (smpboot.c). */ - register_percpu_irq(IA64_CMC_VECTOR, &cmci_irqaction); - register_percpu_irq(IA64_CMCP_VECTOR, &cmcp_irqaction); + register_percpu_irq(IA64_CMC_VECTOR, ia64_mca_cmc_int_handler, 0, + "cmc_hndlr"); + register_percpu_irq(IA64_CMCP_VECTOR, ia64_mca_cmc_int_caller, 0, + "cmc_poll"); ia64_mca_cmc_vector_setup(); /* Setup vector on BSP */ /* Setup the MCA rendezvous interrupt vector */ - register_percpu_irq(IA64_MCA_RENDEZ_VECTOR, &mca_rdzv_irqaction); + register_percpu_irq(IA64_MCA_RENDEZ_VECTOR, ia64_mca_rendez_int_handler, + 0, "mca_rdzv"); /* Setup the MCA wakeup interrupt vector */ - register_percpu_irq(IA64_MCA_WAKEUP_VECTOR, &mca_wkup_irqaction); + register_percpu_irq(IA64_MCA_WAKEUP_VECTOR, ia64_mca_wakeup_int_handler, + 0, "mca_wkup"); /* Setup the CPEI/P handler */ - register_percpu_irq(IA64_CPEP_VECTOR, &mca_cpep_irqaction); + register_percpu_irq(IA64_CPEP_VECTOR, ia64_mca_cpe_int_caller, 0, + "cpe_poll"); } /* @@ -2108,7 +2084,9 @@ ia64_mca_late_init(void) if (irq > 0) { cpe_poll_enabled = 0; irq_set_status_flags(irq, IRQ_PER_CPU); - setup_irq(irq, &mca_cpe_irqaction); + if (request_irq(irq, ia64_mca_cpe_int_handler, + 0, "cpe_hndlr", NULL)) + pr_err("Failed to register cpe_hndlr interrupt\n"); ia64_cpe_irq = irq; ia64_mca_register_cpev(cpe_vector); IA64_MCA_DEBUG("%s: CPEI/P setup and enabled.\n", diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c index a23c3938a1c4..df257002950e 100644 --- a/arch/ia64/kernel/perfmon.c +++ b/arch/ia64/kernel/perfmon.c @@ -57,6 +57,8 @@ #include <linux/uaccess.h> #include <asm/delay.h> +#include "irq.h" + #ifdef CONFIG_PERFMON /* * perfmon context state @@ -6313,11 +6315,6 @@ pfm_flush_pmds(struct task_struct *task, pfm_context_t *ctx) } } -static struct irqaction perfmon_irqaction = { - .handler = pfm_interrupt_handler, - .name = "perfmon" -}; - static void pfm_alt_save_pmu_state(void *data) { @@ -6591,7 +6588,8 @@ pfm_init_percpu (void) pfm_unfreeze_pmu(); if (first_time) { - register_percpu_irq(IA64_PERFMON_VECTOR, &perfmon_irqaction); + register_percpu_irq(IA64_PERFMON_VECTOR, pfm_interrupt_handler, + 0, "perfmon"); first_time=0; } diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index 968b5f33e725..10cb9382ab76 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -646,14 +646,8 @@ cpu_halt (void) void machine_shutdown(void) { -#ifdef CONFIG_HOTPLUG_CPU - int cpu; + smp_shutdown_nonboot_cpus(reboot_cpu); - for_each_online_cpu(cpu) { - if (cpu != smp_processor_id()) - cpu_down(cpu); - } -#endif #ifdef CONFIG_KEXEC kexec_disable_iosapic(); #endif @@ -681,3 +675,4 @@ machine_power_off (void) machine_halt(); } +EXPORT_SYMBOL(ia64_delay_loop); diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c index 91b4024c9351..7abc5f37bfaf 100644 --- a/arch/ia64/kernel/time.c +++ b/arch/ia64/kernel/time.c @@ -32,6 +32,7 @@ #include <asm/sections.h> #include "fsyscall_gtod_data.h" +#include "irq.h" static u64 itc_get_cycles(struct clocksource *cs); @@ -380,13 +381,6 @@ static u64 itc_get_cycles(struct clocksource *cs) return now; } - -static struct irqaction timer_irqaction = { - .handler = timer_interrupt, - .flags = IRQF_IRQPOLL, - .name = "timer" -}; - void read_persistent_clock64(struct timespec64 *ts) { efi_gettimeofday(ts); @@ -395,7 +389,8 @@ void read_persistent_clock64(struct timespec64 *ts) void __init time_init (void) { - register_percpu_irq(IA64_TIMER_VECTOR, &timer_irqaction); + register_percpu_irq(IA64_TIMER_VECTOR, timer_interrupt, IRQF_IRQPOLL, + "timer"); ia64_init_itm(); } diff --git a/arch/ia64/mm/ioremap.c b/arch/ia64/mm/ioremap.c index a09cfa064536..55fd3eb753ff 100644 --- a/arch/ia64/mm/ioremap.c +++ b/arch/ia64/mm/ioremap.c @@ -8,6 +8,7 @@ #include <linux/module.h> #include <linux/efi.h> #include <linux/io.h> +#include <linux/mm.h> #include <linux/vmalloc.h> #include <asm/io.h> #include <asm/meminit.h> diff --git a/arch/m68k/Kconfig.bus b/arch/m68k/Kconfig.bus index 9d0a3a23d50e..f1be832e2b74 100644 --- a/arch/m68k/Kconfig.bus +++ b/arch/m68k/Kconfig.bus @@ -66,6 +66,6 @@ endif if !MMU config ISA_DMA_API - def_bool !M5272 + def_bool !M5272 endif diff --git a/arch/m68k/Kconfig.debug b/arch/m68k/Kconfig.debug index f43643111eaf..11b306bdd788 100644 --- a/arch/m68k/Kconfig.debug +++ b/arch/m68k/Kconfig.debug @@ -12,16 +12,16 @@ config EARLY_PRINTK bool "Early printk" depends on !(SUN3 || M68000 || COLDFIRE) help - Write kernel log output directly to a serial port. - Where implemented, output goes to the framebuffer as well. - PROM console functionality on Sun 3x is not affected by this option. + Write kernel log output directly to a serial port. + Where implemented, output goes to the framebuffer as well. + PROM console functionality on Sun 3x is not affected by this option. - Pass "earlyprintk" on the kernel command line to get a - boot console. + Pass "earlyprintk" on the kernel command line to get a + boot console. - This is useful for kernel debugging when your machine crashes very - early, i.e. before the normal console driver is loaded. - You should normally say N here, unless you want to debug such a crash. + This is useful for kernel debugging when your machine crashes very + early, i.e. before the normal console driver is loaded. + You should normally say N here, unless you want to debug such a crash. if !MMU diff --git a/arch/m68k/Kconfig.machine b/arch/m68k/Kconfig.machine index c01e103492fd..b23a66bac77f 100644 --- a/arch/m68k/Kconfig.machine +++ b/arch/m68k/Kconfig.machine @@ -269,10 +269,10 @@ config AMCORE Support for the Sysam AMCORE open-hardware generic board. config STMARK2 - bool "Sysam stmark2 board support" - depends on M5441x - help - Support for the Sysam stmark2 open-hardware generic board. + bool "Sysam stmark2 board support" + depends on M5441x + help + Support for the Sysam stmark2 open-hardware generic board. config FIREBEE bool "FireBee board support" diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index e1134c3e0b69..9818d0ed143e 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -173,7 +173,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -369,6 +368,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -613,9 +613,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 484cb1643df1..db2402c75300 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -169,7 +169,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -348,6 +347,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -569,9 +569,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index eb6a46b6d135..07a28b48eb43 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -176,7 +176,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -364,6 +363,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -591,9 +591,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index bee9263a409c..033589367cb1 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -166,7 +166,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -346,6 +345,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -562,9 +562,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index c8847a8bcbd6..ee4d02a90214 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -168,7 +168,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -347,6 +346,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -571,9 +571,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 303ffafd9cad..442eed608ebc 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -167,7 +167,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -363,6 +362,7 @@ CONFIG_INPUT_ADBHID=y CONFIG_MAC_EMUMOUSEBTN=y CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -593,9 +593,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 89a704226cd9..1de7da7c1041 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -187,7 +187,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -407,6 +406,7 @@ CONFIG_INPUT_ADBHID=y CONFIG_MAC_EMUMOUSEBTN=y CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -679,9 +679,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index f62c1f4d03a0..ced341e047db 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -165,7 +165,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -345,6 +344,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -561,9 +561,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 58dcad26a751..006188f1f063 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -166,7 +166,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -346,6 +345,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -562,9 +562,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 5d3c28d1d545..c65d985ee642 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -167,7 +167,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -353,6 +352,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -580,9 +580,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 5ef9e17dcd51..32905fe56cd7 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -163,7 +163,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -343,6 +342,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -564,9 +564,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 22e1accc60a3..32b0969668fe 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -163,7 +163,6 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -343,6 +342,7 @@ CONFIG_TCM_FILEIO=m CONFIG_TCM_PSCSI=m CONFIG_NETDEVICES=y CONFIG_DUMMY=m +CONFIG_WIREGUARD=m CONFIG_EQUALIZER=m CONFIG_NET_TEAM=m CONFIG_NET_TEAM_MODE_BROADCAST=m @@ -563,9 +563,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_LIB_BLAKE2S=m -CONFIG_CRYPTO_LIB_CURVE25519=m -CONFIG_CRYPTO_LIB_CHACHA20POLY1305=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 40712e49381b..c3a630440512 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -118,12 +118,11 @@ static int __init nfhd_init_one(int id, u32 blocks, u32 bsize) dev->bsize = bsize; dev->bshift = ffs(bsize) - 10; - dev->queue = blk_alloc_queue(GFP_KERNEL); + dev->queue = blk_alloc_queue(nfhd_make_request, NUMA_NO_NODE); if (dev->queue == NULL) goto free_dev; dev->queue->queuedata = dev; - blk_queue_make_request(dev->queue, nfhd_make_request); blk_queue_logical_block_size(dev->queue, bsize); dev->disk = alloc_disk(16); diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild index 591d53b763b7..63f12afc4874 100644 --- a/arch/m68k/include/asm/Kbuild +++ b/arch/m68k/include/asm/Kbuild @@ -8,6 +8,7 @@ generic-y += emergency-restart.h generic-y += exec.h generic-y += extable.h generic-y += futex.h +generic-y += hardirq.h generic-y += hw_irq.h generic-y += irq_regs.h generic-y += irq_work.h diff --git a/arch/m68k/include/asm/hardirq.h b/arch/m68k/include/asm/hardirq.h deleted file mode 100644 index 11793165445d..000000000000 --- a/arch/m68k/include/asm/hardirq.h +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __M68K_HARDIRQ_H -#define __M68K_HARDIRQ_H - -#include <linux/threads.h> -#include <linux/cache.h> -#include <asm/irq.h> - -#ifdef CONFIG_MMU - -static inline void ack_bad_irq(unsigned int irq) -{ - pr_crit("unexpected IRQ trap at vector %02x\n", irq); -} - -/* entry.S is sensitive to the offsets of these fields */ -typedef struct { - unsigned int __softirq_pending; -} ____cacheline_aligned irq_cpustat_t; - -#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */ - -#else - -#include <asm-generic/hardirq.h> - -#endif /* !CONFIG_MMU */ - -#endif diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h index 82ec54c2eaa4..bc1228e00518 100644 --- a/arch/m68k/include/asm/mcf_pgalloc.h +++ b/arch/m68k/include/asm/mcf_pgalloc.h @@ -28,21 +28,22 @@ extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address) return (pmd_t *) pgd; } -#define pmd_populate(mm, pmd, page) (pmd_val(*pmd) = \ - (unsigned long)(page_address(page))) +#define pmd_populate(mm, pmd, pte) (pmd_val(*pmd) = (unsigned long)(pte)) -#define pmd_populate_kernel(mm, pmd, pte) (pmd_val(*pmd) = (unsigned long)(pte)) +#define pmd_populate_kernel pmd_populate -#define pmd_pgtable(pmd) pmd_page(pmd) +#define pmd_pgtable(pmd) pfn_to_virt(pmd_val(pmd) >> PAGE_SHIFT) -static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, unsigned long address) { + struct page *page = virt_to_page(pgtable); + pgtable_pte_page_dtor(page); __free_page(page); } -static inline struct page *pte_alloc_one(struct mm_struct *mm) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm) { struct page *page = alloc_pages(GFP_DMA, 0); pte_t *pte; @@ -54,20 +55,16 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm) return NULL; } - pte = kmap(page); - if (pte) { - clear_page(pte); - __flush_page_to_ram(pte); - flush_tlb_kernel_page(pte); - nocache_page(pte); - } - kunmap(page); + pte = page_address(page); + clear_page(pte); - return page; + return pte; } -static inline void pte_free(struct mm_struct *mm, struct page *page) +static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) { + struct page *page = virt_to_page(pgtable); + pgtable_pte_page_dtor(page); __free_page(page); } @@ -90,7 +87,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) new_pgd = (pgd_t *)__get_free_page(GFP_DMA | __GFP_NOWARN); if (!new_pgd) return NULL; - memcpy(new_pgd, swapper_pg_dir, PAGE_SIZE); + memcpy(new_pgd, swapper_pg_dir, PTRS_PER_PGD * sizeof(pgd_t)); memset(new_pgd, 0, PAGE_OFFSET >> PGDIR_SHIFT); return new_pgd; } diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h index ff9cc401ffd1..c66e42917912 100644 --- a/arch/m68k/include/asm/motorola_pgalloc.h +++ b/arch/m68k/include/asm/motorola_pgalloc.h @@ -5,93 +5,71 @@ #include <asm/tlb.h> #include <asm/tlbflush.h> -extern pmd_t *get_pointer_table(void); -extern int free_pointer_table(pmd_t *); +extern void mmu_page_ctor(void *page); +extern void mmu_page_dtor(void *page); -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) -{ - pte_t *pte; +enum m68k_table_types { + TABLE_PGD = 0, + TABLE_PMD = 0, /* same size as PGD */ + TABLE_PTE = 1, +}; - pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); - if (pte) { - __flush_page_to_ram(pte); - flush_tlb_kernel_page(pte); - nocache_page(pte); - } +extern void init_pointer_table(void *table, int type); +extern void *get_pointer_table(int type); +extern int free_pointer_table(void *table, int type); - return pte; +static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) +{ + return get_pointer_table(TABLE_PTE); } static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) { - cache_page(pte); - free_page((unsigned long) pte); + free_pointer_table(pte, TABLE_PTE); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm) { - struct page *page; - pte_t *pte; - - page = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0); - if(!page) - return NULL; - if (!pgtable_pte_page_ctor(page)) { - __free_page(page); - return NULL; - } - - pte = kmap(page); - __flush_page_to_ram(pte); - flush_tlb_kernel_page(pte); - nocache_page(pte); - kunmap(page); - return page; + return get_pointer_table(TABLE_PTE); } -static inline void pte_free(struct mm_struct *mm, pgtable_t page) +static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) { - pgtable_pte_page_dtor(page); - cache_page(kmap(page)); - kunmap(page); - __free_page(page); + free_pointer_table(pgtable, TABLE_PTE); } -static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, unsigned long address) { - pgtable_pte_page_dtor(page); - cache_page(kmap(page)); - kunmap(page); - __free_page(page); + free_pointer_table(pgtable, TABLE_PTE); } static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) { - return get_pointer_table(); + return get_pointer_table(TABLE_PMD); } static inline int pmd_free(struct mm_struct *mm, pmd_t *pmd) { - return free_pointer_table(pmd); + return free_pointer_table(pmd, TABLE_PMD); } static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, unsigned long address) { - return free_pointer_table(pmd); + return free_pointer_table(pmd, TABLE_PMD); } static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) { - pmd_free(mm, (pmd_t *)pgd); + free_pointer_table(pgd, TABLE_PGD); } static inline pgd_t *pgd_alloc(struct mm_struct *mm) { - return (pgd_t *)get_pointer_table(); + return get_pointer_table(TABLE_PGD); } @@ -102,9 +80,9 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t * static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page) { - pmd_set(pmd, page_address(page)); + pmd_set(pmd, page); } -#define pmd_pgtable(pmd) pmd_page(pmd) +#define pmd_pgtable(pmd) ((pgtable_t)__pmd_page(pmd)) static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { diff --git a/arch/m68k/include/asm/motorola_pgtable.h b/arch/m68k/include/asm/motorola_pgtable.h index 62bedc61f110..4b91a470ad58 100644 --- a/arch/m68k/include/asm/motorola_pgtable.h +++ b/arch/m68k/include/asm/motorola_pgtable.h @@ -23,7 +23,18 @@ #define _DESCTYPE_MASK 0x003 #define _CACHEMASK040 (~0x060) -#define _TABLE_MASK (0xfffffe00) + +/* + * Currently set to the minimum alignment of table pointers (256 bytes). + * The hardware only uses the low 4 bits for state: + * + * 3 - Used + * 2 - Write Protected + * 0,1 - Descriptor Type + * + * and has the rest of the bits reserved. + */ +#define _TABLE_MASK (0xffffff00) #define _PAGE_TABLE (_PAGE_SHORT) #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_NOCACHE) @@ -108,13 +119,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) { - unsigned long ptbl = virt_to_phys(ptep) | _PAGE_TABLE | _PAGE_ACCESSED; - unsigned long *ptr = pmdp->pmd; - short i = 16; - while (--i >= 0) { - *ptr++ = ptbl; - ptbl += (sizeof(pte_t)*PTRS_PER_PTE/16); - } + pmd_val(*pmdp) = virt_to_phys(ptep) | _PAGE_TABLE | _PAGE_ACCESSED; } static inline void pud_set(pud_t *pudp, pmd_t *pmdp) @@ -138,13 +143,14 @@ static inline void pud_set(pud_t *pudp, pmd_t *pmdp) #define pmd_none(pmd) (!pmd_val(pmd)) #define pmd_bad(pmd) ((pmd_val(pmd) & _DESCTYPE_MASK) != _PAGE_TABLE) #define pmd_present(pmd) (pmd_val(pmd) & _PAGE_TABLE) -#define pmd_clear(pmdp) ({ \ - unsigned long *__ptr = pmdp->pmd; \ - short __i = 16; \ - while (--__i >= 0) \ - *__ptr++ = 0; \ -}) -#define pmd_page(pmd) virt_to_page(__va(pmd_val(pmd))) +#define pmd_clear(pmdp) ({ pmd_val(*pmdp) = 0; }) + +/* + * m68k does not have huge pages (020/030 actually could), but generic code + * expects pmd_page() to exists, only to then DCE it all. Provide a dummy to + * make the compiler happy. + */ +#define pmd_page(pmd) NULL #define pud_none(pud) (!pud_val(pud)) diff --git a/arch/m68k/include/asm/page.h b/arch/m68k/include/asm/page.h index 05e1e1e77a9a..da546487e177 100644 --- a/arch/m68k/include/asm/page.h +++ b/arch/m68k/include/asm/page.h @@ -22,15 +22,25 @@ * These are used to make use of C type-checking.. */ #if !defined(CONFIG_MMU) || CONFIG_PGTABLE_LEVELS == 3 -typedef struct { unsigned long pmd[16]; } pmd_t; -#define pmd_val(x) ((&x)->pmd[0]) -#define __pmd(x) ((pmd_t) { { (x) }, }) +typedef struct { unsigned long pmd; } pmd_t; +#define pmd_val(x) ((&x)->pmd) +#define __pmd(x) ((pmd_t) { (x) } ) #endif typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; + +#if defined(CONFIG_SUN3) +/* + * Sun3 still uses the asm-generic/pgalloc.h code and thus needs this + * definition. It would be possible to unify Sun3 and ColdFire pgalloc and have + * all of m68k use the same type. + */ typedef struct page *pgtable_t; +#else +typedef pte_t *pgtable_t; +#endif #define pte_val(x) ((x).pte) #define pgd_val(x) ((x).pgd) diff --git a/arch/m68k/include/asm/pgtable_mm.h b/arch/m68k/include/asm/pgtable_mm.h index 2bf5c3501e78..f0e5167de834 100644 --- a/arch/m68k/include/asm/pgtable_mm.h +++ b/arch/m68k/include/asm/pgtable_mm.h @@ -36,7 +36,7 @@ /* PMD_SHIFT determines the size of the area a second-level page table can map */ #if CONFIG_PGTABLE_LEVELS == 3 -#define PMD_SHIFT 22 +#define PMD_SHIFT 18 #endif #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) @@ -67,8 +67,8 @@ #define PTRS_PER_PMD 1 #define PTRS_PER_PGD 1024 #else -#define PTRS_PER_PTE 1024 -#define PTRS_PER_PMD 8 +#define PTRS_PER_PTE 64 +#define PTRS_PER_PMD 128 #define PTRS_PER_PGD 128 #endif #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) @@ -76,8 +76,8 @@ /* Virtual address region for use by kernel_map() */ #ifdef CONFIG_SUN3 -#define KMAP_START 0x0DC00000 -#define KMAP_END 0x0E000000 +#define KMAP_START 0x0dc00000 +#define KMAP_END 0x0e000000 #elif defined(CONFIG_COLDFIRE) #define KMAP_START 0xe0000000 #define KMAP_END 0xf0000000 diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h index 6bc80c35726d..a24cfe4a0d32 100644 --- a/arch/m68k/include/asm/uaccess_no.h +++ b/arch/m68k/include/asm/uaccess_no.h @@ -5,7 +5,6 @@ /* * User space memory access functions */ -#include <linux/mm.h> #include <linux/string.h> #include <asm/segment.h> diff --git a/arch/m68k/mm/init.c b/arch/m68k/mm/init.c index 27c453f4fffe..b88d510d4fe3 100644 --- a/arch/m68k/mm/init.c +++ b/arch/m68k/mm/init.c @@ -40,11 +40,6 @@ void *empty_zero_page; EXPORT_SYMBOL(empty_zero_page); -#if !defined(CONFIG_SUN3) && !defined(CONFIG_COLDFIRE) -extern void init_pointer_table(unsigned long ptable); -extern pmd_t *zero_pgtable; -#endif - #ifdef CONFIG_MMU pg_data_t pg_data_map[MAX_NUMNODES]; @@ -125,20 +120,31 @@ void free_initmem(void) static inline void init_pointer_tables(void) { #if defined(CONFIG_MMU) && !defined(CONFIG_SUN3) && !defined(CONFIG_COLDFIRE) - int i; + int i, j; /* insert pointer tables allocated so far into the tablelist */ - init_pointer_table((unsigned long)kernel_pg_dir); + init_pointer_table(kernel_pg_dir, TABLE_PGD); for (i = 0; i < PTRS_PER_PGD; i++) { - pud_t *pud = (pud_t *)(&kernel_pg_dir[i]); + pud_t *pud = (pud_t *)&kernel_pg_dir[i]; + pmd_t *pmd_dir; - if (pud_present(*pud)) - init_pointer_table(pgd_page_vaddr(kernel_pg_dir[i])); - } + if (!pud_present(*pud)) + continue; + + pmd_dir = (pmd_t *)pgd_page_vaddr(kernel_pg_dir[i]); + init_pointer_table(pmd_dir, TABLE_PMD); - /* insert also pointer table that we used to unmap the zero page */ - if (zero_pgtable) - init_pointer_table((unsigned long)zero_pgtable); + for (j = 0; j < PTRS_PER_PMD; j++) { + pmd_t *pmd = &pmd_dir[j]; + pte_t *pte_dir; + + if (!pmd_present(*pmd)) + continue; + + pte_dir = (pte_t *)__pmd_page(*pmd); + init_pointer_table(pte_dir, TABLE_PTE); + } + } #endif } diff --git a/arch/m68k/mm/kmap.c b/arch/m68k/mm/kmap.c index 120030ad8dc4..14d31d216cef 100644 --- a/arch/m68k/mm/kmap.c +++ b/arch/m68k/mm/kmap.c @@ -24,8 +24,6 @@ #undef DEBUG -#define PTRTREESIZE (256*1024) - /* * For 040/060 we can use the virtual memory area like other architectures, * but for 020/030 we want to use early termination page descriptors and we @@ -50,7 +48,7 @@ static inline void free_io_area(void *addr) #else -#define IO_SIZE (256*1024) +#define IO_SIZE PMD_SIZE static struct vm_struct *iolist; @@ -81,14 +79,13 @@ static void __free_io_area(void *addr, unsigned long size) #if CONFIG_PGTABLE_LEVELS == 3 if (CPU_IS_020_OR_030) { - int pmd_off = (virtaddr/PTRTREESIZE) & 15; - int pmd_type = pmd_dir->pmd[pmd_off] & _DESCTYPE_MASK; + int pmd_type = pmd_val(*pmd_dir) & _DESCTYPE_MASK; if (pmd_type == _PAGE_PRESENT) { - pmd_dir->pmd[pmd_off] = 0; - virtaddr += PTRTREESIZE; - size -= PTRTREESIZE; - continue; + pmd_clear(pmd_dir); + virtaddr += PMD_SIZE; + size -= PMD_SIZE; + } else if (pmd_type == 0) continue; } @@ -249,7 +246,7 @@ void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cachefla while ((long)size > 0) { #ifdef DEBUG - if (!(virtaddr & (PTRTREESIZE-1))) + if (!(virtaddr & (PMD_SIZE-1))) printk ("\npa=%#lx va=%#lx ", physaddr, virtaddr); #endif pgd_dir = pgd_offset_k(virtaddr); @@ -263,10 +260,10 @@ void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cachefla #if CONFIG_PGTABLE_LEVELS == 3 if (CPU_IS_020_OR_030) { - pmd_dir->pmd[(virtaddr/PTRTREESIZE) & 15] = physaddr; - physaddr += PTRTREESIZE; - virtaddr += PTRTREESIZE; - size -= PTRTREESIZE; + pmd_val(*pmd_dir) = physaddr; + physaddr += PMD_SIZE; + virtaddr += PMD_SIZE; + size -= PMD_SIZE; } else #endif { @@ -367,13 +364,12 @@ void kernel_set_cachemode(void *addr, unsigned long size, int cmode) #if CONFIG_PGTABLE_LEVELS == 3 if (CPU_IS_020_OR_030) { - int pmd_off = (virtaddr/PTRTREESIZE) & 15; + unsigned long pmd = pmd_val(*pmd_dir); - if ((pmd_dir->pmd[pmd_off] & _DESCTYPE_MASK) == _PAGE_PRESENT) { - pmd_dir->pmd[pmd_off] = (pmd_dir->pmd[pmd_off] & - _CACHEMASK040) | cmode; - virtaddr += PTRTREESIZE; - size -= PTRTREESIZE; + if ((pmd & _DESCTYPE_MASK) == _PAGE_PRESENT) { + *pmd_dir = __pmd((pmd & _CACHEMASK040) | cmode); + virtaddr += PMD_SIZE; + size -= PMD_SIZE; continue; } } diff --git a/arch/m68k/mm/memory.c b/arch/m68k/mm/memory.c index 227c04fe60d2..65e0c4071912 100644 --- a/arch/m68k/mm/memory.c +++ b/arch/m68k/mm/memory.c @@ -22,109 +22,6 @@ #include <asm/machdep.h> -/* ++andreas: {get,free}_pointer_table rewritten to use unused fields from - struct page instead of separately kmalloced struct. Stolen from - arch/sparc/mm/srmmu.c ... */ - -typedef struct list_head ptable_desc; -static LIST_HEAD(ptable_list); - -#define PD_PTABLE(page) ((ptable_desc *)&(virt_to_page(page)->lru)) -#define PD_PAGE(ptable) (list_entry(ptable, struct page, lru)) -#define PD_MARKBITS(dp) (*(unsigned char *)&PD_PAGE(dp)->index) - -#define PTABLE_SIZE (PTRS_PER_PMD * sizeof(pmd_t)) - -void __init init_pointer_table(unsigned long ptable) -{ - ptable_desc *dp; - unsigned long page = ptable & PAGE_MASK; - unsigned char mask = 1 << ((ptable - page)/PTABLE_SIZE); - - dp = PD_PTABLE(page); - if (!(PD_MARKBITS(dp) & mask)) { - PD_MARKBITS(dp) = 0xff; - list_add(dp, &ptable_list); - } - - PD_MARKBITS(dp) &= ~mask; - pr_debug("init_pointer_table: %lx, %x\n", ptable, PD_MARKBITS(dp)); - - /* unreserve the page so it's possible to free that page */ - __ClearPageReserved(PD_PAGE(dp)); - init_page_count(PD_PAGE(dp)); - - return; -} - -pmd_t *get_pointer_table (void) -{ - ptable_desc *dp = ptable_list.next; - unsigned char mask = PD_MARKBITS (dp); - unsigned char tmp; - unsigned int off; - - /* - * For a pointer table for a user process address space, a - * table is taken from a page allocated for the purpose. Each - * page can hold 8 pointer tables. The page is remapped in - * virtual address space to be noncacheable. - */ - if (mask == 0) { - void *page; - ptable_desc *new; - - if (!(page = (void *)get_zeroed_page(GFP_KERNEL))) - return NULL; - - flush_tlb_kernel_page(page); - nocache_page(page); - - new = PD_PTABLE(page); - PD_MARKBITS(new) = 0xfe; - list_add_tail(new, dp); - - return (pmd_t *)page; - } - - for (tmp = 1, off = 0; (mask & tmp) == 0; tmp <<= 1, off += PTABLE_SIZE) - ; - PD_MARKBITS(dp) = mask & ~tmp; - if (!PD_MARKBITS(dp)) { - /* move to end of list */ - list_move_tail(dp, &ptable_list); - } - return (pmd_t *) (page_address(PD_PAGE(dp)) + off); -} - -int free_pointer_table (pmd_t *ptable) -{ - ptable_desc *dp; - unsigned long page = (unsigned long)ptable & PAGE_MASK; - unsigned char mask = 1 << (((unsigned long)ptable - page)/PTABLE_SIZE); - - dp = PD_PTABLE(page); - if (PD_MARKBITS (dp) & mask) - panic ("table already free!"); - - PD_MARKBITS (dp) |= mask; - - if (PD_MARKBITS(dp) == 0xff) { - /* all tables in page are free, free page */ - list_del(dp); - cache_page((void *)page); - free_page (page); - return 1; - } else if (ptable_list.next != dp) { - /* - * move this descriptor to the front of the list, since - * it has one or more free tables. - */ - list_move(dp, &ptable_list); - } - return 0; -} - /* invalidate page in both caches */ static inline void clear040(unsigned long paddr) { diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index 4857985b8080..fc16190ec2d6 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -45,34 +45,185 @@ unsigned long mm_cachebits; EXPORT_SYMBOL(mm_cachebits); #endif + +/* + * Motorola 680x0 user's manual recommends using uncached memory for address + * translation tables. + * + * Seeing how the MMU can be external on (some of) these chips, that seems like + * a very important recommendation to follow. Provide some helpers to combat + * 'variation' amongst the users of this. + */ + +void mmu_page_ctor(void *page) +{ + __flush_page_to_ram(page); + flush_tlb_kernel_page(page); + nocache_page(page); +} + +void mmu_page_dtor(void *page) +{ + cache_page(page); +} + +/* ++andreas: {get,free}_pointer_table rewritten to use unused fields from + struct page instead of separately kmalloced struct. Stolen from + arch/sparc/mm/srmmu.c ... */ + +typedef struct list_head ptable_desc; + +static struct list_head ptable_list[2] = { + LIST_HEAD_INIT(ptable_list[0]), + LIST_HEAD_INIT(ptable_list[1]), +}; + +#define PD_PTABLE(page) ((ptable_desc *)&(virt_to_page(page)->lru)) +#define PD_PAGE(ptable) (list_entry(ptable, struct page, lru)) +#define PD_MARKBITS(dp) (*(unsigned int *)&PD_PAGE(dp)->index) + +static const int ptable_shift[2] = { + 7+2, /* PGD, PMD */ + 6+2, /* PTE */ +}; + +#define ptable_size(type) (1U << ptable_shift[type]) +#define ptable_mask(type) ((1U << (PAGE_SIZE / ptable_size(type))) - 1) + +void __init init_pointer_table(void *table, int type) +{ + ptable_desc *dp; + unsigned long ptable = (unsigned long)table; + unsigned long page = ptable & PAGE_MASK; + unsigned int mask = 1U << ((ptable - page)/ptable_size(type)); + + dp = PD_PTABLE(page); + if (!(PD_MARKBITS(dp) & mask)) { + PD_MARKBITS(dp) = ptable_mask(type); + list_add(dp, &ptable_list[type]); + } + + PD_MARKBITS(dp) &= ~mask; + pr_debug("init_pointer_table: %lx, %x\n", ptable, PD_MARKBITS(dp)); + + /* unreserve the page so it's possible to free that page */ + __ClearPageReserved(PD_PAGE(dp)); + init_page_count(PD_PAGE(dp)); + + return; +} + +void *get_pointer_table(int type) +{ + ptable_desc *dp = ptable_list[type].next; + unsigned int mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp); + unsigned int tmp, off; + + /* + * For a pointer table for a user process address space, a + * table is taken from a page allocated for the purpose. Each + * page can hold 8 pointer tables. The page is remapped in + * virtual address space to be noncacheable. + */ + if (mask == 0) { + void *page; + ptable_desc *new; + + if (!(page = (void *)get_zeroed_page(GFP_KERNEL))) + return NULL; + + if (type == TABLE_PTE) { + /* + * m68k doesn't have SPLIT_PTE_PTLOCKS for not having + * SMP. + */ + pgtable_pte_page_ctor(virt_to_page(page)); + } + + mmu_page_ctor(page); + + new = PD_PTABLE(page); + PD_MARKBITS(new) = ptable_mask(type) - 1; + list_add_tail(new, dp); + + return (pmd_t *)page; + } + + for (tmp = 1, off = 0; (mask & tmp) == 0; tmp <<= 1, off += ptable_size(type)) + ; + PD_MARKBITS(dp) = mask & ~tmp; + if (!PD_MARKBITS(dp)) { + /* move to end of list */ + list_move_tail(dp, &ptable_list[type]); + } + return page_address(PD_PAGE(dp)) + off; +} + +int free_pointer_table(void *table, int type) +{ + ptable_desc *dp; + unsigned long ptable = (unsigned long)table; + unsigned long page = ptable & PAGE_MASK; + unsigned int mask = 1U << ((ptable - page)/ptable_size(type)); + + dp = PD_PTABLE(page); + if (PD_MARKBITS (dp) & mask) + panic ("table already free!"); + + PD_MARKBITS (dp) |= mask; + + if (PD_MARKBITS(dp) == ptable_mask(type)) { + /* all tables in page are free, free page */ + list_del(dp); + mmu_page_dtor((void *)page); + if (type == TABLE_PTE) + pgtable_pte_page_dtor(virt_to_page(page)); + free_page (page); + return 1; + } else if (ptable_list[type].next != dp) { + /* + * move this descriptor to the front of the list, since + * it has one or more free tables. + */ + list_move(dp, &ptable_list[type]); + } + return 0; +} + /* size of memory already mapped in head.S */ extern __initdata unsigned long m68k_init_mapped_size; extern unsigned long availmem; +static pte_t *last_pte_table __initdata = NULL; + static pte_t * __init kernel_page_table(void) { - pte_t *ptablep; + pte_t *pte_table = last_pte_table; - ptablep = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); - if (!ptablep) - panic("%s: Failed to allocate %lu bytes align=%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE); + if (((unsigned long)last_pte_table & ~PAGE_MASK) == 0) { + pte_table = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pte_table) { + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); + } - clear_page(ptablep); - __flush_page_to_ram(ptablep); - flush_tlb_kernel_page(ptablep); - nocache_page(ptablep); + clear_page(pte_table); + mmu_page_ctor(pte_table); - return ptablep; + last_pte_table = pte_table; + } + + last_pte_table += PTRS_PER_PTE; + + return pte_table; } -static pmd_t *last_pgtable __initdata = NULL; -pmd_t *zero_pgtable __initdata = NULL; +static pmd_t *last_pmd_table __initdata = NULL; static pmd_t * __init kernel_ptr_table(void) { - if (!last_pgtable) { + if (!last_pmd_table) { unsigned long pmd, last; int i; @@ -91,33 +242,29 @@ static pmd_t * __init kernel_ptr_table(void) last = pmd; } - last_pgtable = (pmd_t *)last; + last_pmd_table = (pmd_t *)last; #ifdef DEBUG - printk("kernel_ptr_init: %p\n", last_pgtable); + printk("kernel_ptr_init: %p\n", last_pmd_table); #endif } - last_pgtable += PTRS_PER_PMD; - if (((unsigned long)last_pgtable & ~PAGE_MASK) == 0) { - last_pgtable = (pmd_t *)memblock_alloc_low(PAGE_SIZE, + last_pmd_table += PTRS_PER_PMD; + if (((unsigned long)last_pmd_table & ~PAGE_MASK) == 0) { + last_pmd_table = (pmd_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); - if (!last_pgtable) + if (!last_pmd_table) panic("%s: Failed to allocate %lu bytes align=%lx\n", __func__, PAGE_SIZE, PAGE_SIZE); - clear_page(last_pgtable); - __flush_page_to_ram(last_pgtable); - flush_tlb_kernel_page(last_pgtable); - nocache_page(last_pgtable); + clear_page(last_pmd_table); + mmu_page_ctor(last_pmd_table); } - return last_pgtable; + return last_pmd_table; } static void __init map_node(int node) { -#define PTRTREESIZE (256*1024) -#define ROOTTREESIZE (32*1024*1024) unsigned long physaddr, virtaddr, size; pgd_t *pgd_dir; p4d_t *p4d_dir; @@ -135,21 +282,21 @@ static void __init map_node(int node) while (size > 0) { #ifdef DEBUG - if (!(virtaddr & (PTRTREESIZE-1))) + if (!(virtaddr & (PMD_SIZE-1))) printk ("\npa=%#lx va=%#lx ", physaddr & PAGE_MASK, virtaddr); #endif pgd_dir = pgd_offset_k(virtaddr); if (virtaddr && CPU_IS_020_OR_030) { - if (!(virtaddr & (ROOTTREESIZE-1)) && - size >= ROOTTREESIZE) { + if (!(virtaddr & (PGDIR_SIZE-1)) && + size >= PGDIR_SIZE) { #ifdef DEBUG printk ("[very early term]"); #endif pgd_val(*pgd_dir) = physaddr; - size -= ROOTTREESIZE; - virtaddr += ROOTTREESIZE; - physaddr += ROOTTREESIZE; + size -= PGDIR_SIZE; + virtaddr += PGDIR_SIZE; + physaddr += PGDIR_SIZE; continue; } } @@ -169,24 +316,23 @@ static void __init map_node(int node) #ifdef DEBUG printk ("[early term]"); #endif - pmd_dir->pmd[(virtaddr/PTRTREESIZE) & 15] = physaddr; - physaddr += PTRTREESIZE; + pmd_val(*pmd_dir) = physaddr; + physaddr += PMD_SIZE; } else { int i; #ifdef DEBUG printk ("[zero map]"); #endif - zero_pgtable = kernel_ptr_table(); - pte_dir = (pte_t *)zero_pgtable; - pmd_dir->pmd[0] = virt_to_phys(pte_dir) | - _PAGE_TABLE | _PAGE_ACCESSED; + pte_dir = kernel_page_table(); + pmd_set(pmd_dir, pte_dir); + pte_val(*pte_dir++) = 0; physaddr += PAGE_SIZE; - for (i = 1; i < 64; physaddr += PAGE_SIZE, i++) + for (i = 1; i < PTRS_PER_PTE; physaddr += PAGE_SIZE, i++) pte_val(*pte_dir++) = physaddr; } - size -= PTRTREESIZE; - virtaddr += PTRTREESIZE; + size -= PMD_SIZE; + virtaddr += PMD_SIZE; } else { if (!pmd_present(*pmd_dir)) { #ifdef DEBUG diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 6a331bd57ea8..242f58ec086b 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -47,6 +47,8 @@ config MICROBLAZE select CPU_NO_EFFICIENT_FFS select MMU_GATHER_NO_RANGE if MMU select SPARSE_IRQ + select GENERIC_IRQ_MULTI_HANDLER + select HANDLE_DOMAIN_IRQ # Endianness selection choice diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild index e5c9170a07fc..abb33619299b 100644 --- a/arch/microblaze/include/asm/Kbuild +++ b/arch/microblaze/include/asm/Kbuild @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h -generic-y += barrier.h generic-y += bitops.h generic-y += bug.h generic-y += bugs.h @@ -13,6 +12,7 @@ generic-y += exec.h generic-y += extable.h generic-y += fb.h generic-y += hardirq.h +generic-y += hw_irq.h generic-y += irq_regs.h generic-y += irq_work.h generic-y += kdebug.h @@ -31,8 +31,10 @@ generic-y += preempt.h generic-y += serial.h generic-y += shmparam.h generic-y += syscalls.h +generic-y += tlb.h generic-y += topology.h generic-y += trace_clock.h +generic-y += user.h generic-y += vga.h generic-y += word-at-a-time.h generic-y += xor.h diff --git a/arch/microblaze/include/asm/barrier.h b/arch/microblaze/include/asm/barrier.h new file mode 100644 index 000000000000..70b0a017781b --- /dev/null +++ b/arch/microblaze/include/asm/barrier.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2015 - 2020 Xilinx, Inc. All rights reserved. + */ + +#ifndef _ASM_MICROBLAZE_BARRIER_H +#define _ASM_MICROBLAZE_BARRIER_H + +#define mb() __asm__ __volatile__ ("mbar 1" : : : "memory") + +#include <asm-generic/barrier.h> + +#endif /* _ASM_MICROBLAZE_BARRIER_H */ diff --git a/arch/microblaze/include/asm/cache.h b/arch/microblaze/include/asm/cache.h index 4efe96a036f7..a149b3e711ec 100644 --- a/arch/microblaze/include/asm/cache.h +++ b/arch/microblaze/include/asm/cache.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Cache operations * * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007-2009 PetaLogix * Copyright (C) 2003 John Williams <jwilliams@itee.uq.edu.au> - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file COPYING in the main directory of this - * archive for more details. */ #ifndef _ASM_MICROBLAZE_CACHE_H diff --git a/arch/microblaze/include/asm/cacheflush.h b/arch/microblaze/include/asm/cacheflush.h index b091de77b15b..11f56c85056b 100644 --- a/arch/microblaze/include/asm/cacheflush.h +++ b/arch/microblaze/include/asm/cacheflush.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007-2009 PetaLogix @@ -5,11 +6,6 @@ * based on v850 version which was * Copyright (C) 2001,02,03 NEC Electronics Corporation * Copyright (C) 2001,02,03 Miles Bader <miles@gnu.org> - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file COPYING in the main directory of this - * archive for more details. - * */ #ifndef _ASM_MICROBLAZE_CACHEFLUSH_H diff --git a/arch/microblaze/include/asm/checksum.h b/arch/microblaze/include/asm/checksum.h index adeecebbb0d1..2e5ebd599943 100644 --- a/arch/microblaze/include/asm/checksum.h +++ b/arch/microblaze/include/asm/checksum.h @@ -1,10 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008 Michal Simek <monstr@monstr.eu> * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_CHECKSUM_H diff --git a/arch/microblaze/include/asm/cmpxchg.h b/arch/microblaze/include/asm/cmpxchg.h index 596300c74509..3523b51aab36 100644 --- a/arch/microblaze/include/asm/cmpxchg.h +++ b/arch/microblaze/include/asm/cmpxchg.h @@ -2,42 +2,8 @@ #ifndef _ASM_MICROBLAZE_CMPXCHG_H #define _ASM_MICROBLAZE_CMPXCHG_H -#include <linux/irqflags.h> - -void __bad_xchg(volatile void *ptr, int size); - -static inline unsigned long __xchg(unsigned long x, volatile void *ptr, - int size) -{ - unsigned long ret; - unsigned long flags; - - switch (size) { - case 1: - local_irq_save(flags); - ret = *(volatile unsigned char *)ptr; - *(volatile unsigned char *)ptr = x; - local_irq_restore(flags); - break; - - case 4: - local_irq_save(flags); - ret = *(volatile unsigned long *)ptr; - *(volatile unsigned long *)ptr = x; - local_irq_restore(flags); - break; - default: - __bad_xchg(ptr, size), ret = 0; - break; - } - - return ret; -} - -#define xchg(ptr, x) \ - ((__typeof__(*(ptr))) __xchg((unsigned long)(x), (ptr), sizeof(*(ptr)))) - -#include <asm-generic/cmpxchg.h> -#include <asm-generic/cmpxchg-local.h> +#ifndef CONFIG_SMP +# include <asm-generic/cmpxchg.h> +#endif #endif /* _ASM_MICROBLAZE_CMPXCHG_H */ diff --git a/arch/microblaze/include/asm/cpuinfo.h b/arch/microblaze/include/asm/cpuinfo.h index 8f4996730552..786ffa669bf1 100644 --- a/arch/microblaze/include/asm/cpuinfo.h +++ b/arch/microblaze/include/asm/cpuinfo.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Generic support for queying CPU info * * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007-2009 PetaLogix * Copyright (C) 2007 John Williams <jwilliams@itee.uq.edu.au> - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file COPYING in the main directory of this - * archive for more details. */ #ifndef _ASM_MICROBLAZE_CPUINFO_H diff --git a/arch/microblaze/include/asm/cputable.h b/arch/microblaze/include/asm/cputable.h deleted file mode 100644 index 8b137891791f..000000000000 --- a/arch/microblaze/include/asm/cputable.h +++ /dev/null @@ -1 +0,0 @@ - diff --git a/arch/microblaze/include/asm/current.h b/arch/microblaze/include/asm/current.h index 29303ed825cc..a4bb45be30e6 100644 --- a/arch/microblaze/include/asm/current.h +++ b/arch/microblaze/include/asm/current.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_CURRENT_H diff --git a/arch/microblaze/include/asm/delay.h b/arch/microblaze/include/asm/delay.h index ea2a9cd9b159..05fe9e3e0039 100644 --- a/arch/microblaze/include/asm/delay.h +++ b/arch/microblaze/include/asm/delay.h @@ -1,10 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* - * include/asm-microblaze/delay.h - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * * Copyright (C) 2008 Michal Simek * Copyright (C) 2007 John Williams * Copyright (C) 2006 Atmark Techno, Inc. diff --git a/arch/microblaze/include/asm/dma.h b/arch/microblaze/include/asm/dma.h index 0d73d0c6de37..e6cb6d0725af 100644 --- a/arch/microblaze/include/asm/dma.h +++ b/arch/microblaze/include/asm/dma.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_DMA_H diff --git a/arch/microblaze/include/asm/elf.h b/arch/microblaze/include/asm/elf.h index 659024449064..5331a8473a46 100644 --- a/arch/microblaze/include/asm/elf.h +++ b/arch/microblaze/include/asm/elf.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_ELF_H #define _ASM_MICROBLAZE_ELF_H diff --git a/arch/microblaze/include/asm/entry.h b/arch/microblaze/include/asm/entry.h index 596e485ae707..6c42bed41166 100644 --- a/arch/microblaze/include/asm/entry.h +++ b/arch/microblaze/include/asm/entry.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Definitions used by low-level trap handlers * * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007-2009 PetaLogix * Copyright (C) 2007 John Williams <john.williams@petalogix.com> - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file COPYING in the main directory of this - * archive for more details. */ #ifndef _ASM_MICROBLAZE_ENTRY_H diff --git a/arch/microblaze/include/asm/exceptions.h b/arch/microblaze/include/asm/exceptions.h index e6a8ddea1dca..d67e65b72215 100644 --- a/arch/microblaze/include/asm/exceptions.h +++ b/arch/microblaze/include/asm/exceptions.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Preliminary support for HW exception handing for Microblaze * * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2005 John Williams <jwilliams@itee.uq.edu.au> - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file COPYING in the main directory of this - * archive for more details. */ #ifndef _ASM_MICROBLAZE_EXCEPTIONS_H diff --git a/arch/microblaze/include/asm/fixmap.h b/arch/microblaze/include/asm/fixmap.h index 06c0e2b1883f..0379ce5229e3 100644 --- a/arch/microblaze/include/asm/fixmap.h +++ b/arch/microblaze/include/asm/fixmap.h @@ -1,10 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * fixmap.h: compile-time virtual memory allocation * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * * Copyright (C) 1998 Ingo Molnar * * Copyright 2008 Freescale Semiconductor Inc. diff --git a/arch/microblaze/include/asm/flat.h b/arch/microblaze/include/asm/flat.h index 1ab86770eaee..79a749f4ad04 100644 --- a/arch/microblaze/include/asm/flat.h +++ b/arch/microblaze/include/asm/flat.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * uClinux flat-format executables * * Copyright (C) 2005 John Williams <jwilliams@itee.uq.edu.au> - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file COPYING in the main directory of this - * archive for more details. */ #ifndef _ASM_MICROBLAZE_FLAT_H diff --git a/arch/microblaze/include/asm/futex.h b/arch/microblaze/include/asm/futex.h index 8c90357e5983..86131ed84c9a 100644 --- a/arch/microblaze/include/asm/futex.h +++ b/arch/microblaze/include/asm/futex.h @@ -34,7 +34,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -56,8 +57,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/microblaze/include/asm/hw_irq.h b/arch/microblaze/include/asm/hw_irq.h deleted file mode 100644 index 8b137891791f..000000000000 --- a/arch/microblaze/include/asm/hw_irq.h +++ /dev/null @@ -1 +0,0 @@ - diff --git a/arch/microblaze/include/asm/io.h b/arch/microblaze/include/asm/io.h index d33c61737b8b..1dd6fae41897 100644 --- a/arch/microblaze/include/asm/io.h +++ b/arch/microblaze/include/asm/io.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_IO_H diff --git a/arch/microblaze/include/asm/irq.h b/arch/microblaze/include/asm/irq.h index eac2fb4b3fb9..cb6ab55d1d01 100644 --- a/arch/microblaze/include/asm/irq.h +++ b/arch/microblaze/include/asm/irq.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_IRQ_H @@ -14,7 +11,4 @@ struct pt_regs; extern void do_IRQ(struct pt_regs *regs); -/* should be defined in each interrupt controller driver */ -extern unsigned int xintc_get_irq(void); - #endif /* _ASM_MICROBLAZE_IRQ_H */ diff --git a/arch/microblaze/include/asm/irqflags.h b/arch/microblaze/include/asm/irqflags.h index c9a6262832c4..818c6c9f550d 100644 --- a/arch/microblaze/include/asm/irqflags.h +++ b/arch/microblaze/include/asm/irqflags.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_IRQFLAGS_H diff --git a/arch/microblaze/include/asm/mmu.h b/arch/microblaze/include/asm/mmu.h index 1f9edddf7f4b..97f1243101cc 100644 --- a/arch/microblaze/include/asm/mmu.h +++ b/arch/microblaze/include/asm/mmu.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_MMU_H diff --git a/arch/microblaze/include/asm/mmu_context_mm.h b/arch/microblaze/include/asm/mmu_context_mm.h index 97559fe0b953..a1c7dd48454c 100644 --- a/arch/microblaze/include/asm/mmu_context_mm.h +++ b/arch/microblaze/include/asm/mmu_context_mm.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_MMU_CONTEXT_H diff --git a/arch/microblaze/include/asm/module.h b/arch/microblaze/include/asm/module.h index 7be1347fce42..eda1c183b6c7 100644 --- a/arch/microblaze/include/asm/module.h +++ b/arch/microblaze/include/asm/module.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_MODULE_H diff --git a/arch/microblaze/include/asm/page.h b/arch/microblaze/include/asm/page.h index f4b44b24b02e..ae7215c94706 100644 --- a/arch/microblaze/include/asm/page.h +++ b/arch/microblaze/include/asm/page.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * VM ops * @@ -6,10 +7,6 @@ * Copyright (C) 2006 Atmark Techno, Inc. * Changes for MMU support: * Copyright (C) 2007 Xilinx, Inc. All rights reserved. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_PAGE_H diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h index fcf1e23f2e0a..1d7a91252d03 100644 --- a/arch/microblaze/include/asm/pgalloc.h +++ b/arch/microblaze/include/asm/pgalloc.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_PGALLOC_H diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index 2def331f9e2c..45b30878fc17 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_PGTABLE_H diff --git a/arch/microblaze/include/asm/processor.h b/arch/microblaze/include/asm/processor.h index 66b537b8d138..1ff5a82b76b6 100644 --- a/arch/microblaze/include/asm/processor.h +++ b/arch/microblaze/include/asm/processor.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_PROCESSOR_H diff --git a/arch/microblaze/include/asm/ptrace.h b/arch/microblaze/include/asm/ptrace.h index 5b18ec124e51..bfcb89df5e26 100644 --- a/arch/microblaze/include/asm/ptrace.h +++ b/arch/microblaze/include/asm/ptrace.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_PTRACE_H #define _ASM_MICROBLAZE_PTRACE_H diff --git a/arch/microblaze/include/asm/pvr.h b/arch/microblaze/include/asm/pvr.h index 4bbdb4c03b57..186ee8c3c818 100644 --- a/arch/microblaze/include/asm/pvr.h +++ b/arch/microblaze/include/asm/pvr.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for the MicroBlaze PVR (Processor Version Register) * * Copyright (C) 2009 - 2011 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007 John Williams <john.williams@petalogix.com> * Copyright (C) 2007 - 2011 PetaLogix - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file COPYING in the main directory of this - * archive for more details. */ #ifndef _ASM_MICROBLAZE_PVR_H diff --git a/arch/microblaze/include/asm/registers.h b/arch/microblaze/include/asm/registers.h index 68c3afb73877..ee81e1cba008 100644 --- a/arch/microblaze/include/asm/registers.h +++ b/arch/microblaze/include/asm/registers.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_REGISTERS_H diff --git a/arch/microblaze/include/asm/sections.h b/arch/microblaze/include/asm/sections.h index 1b281d3ea734..a9311ad84a67 100644 --- a/arch/microblaze/include/asm/sections.h +++ b/arch/microblaze/include/asm/sections.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_SECTIONS_H diff --git a/arch/microblaze/include/asm/setup.h b/arch/microblaze/include/asm/setup.h index ce9b7b786156..be10da9d87cb 100644 --- a/arch/microblaze/include/asm/setup.h +++ b/arch/microblaze/include/asm/setup.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_SETUP_H #define _ASM_MICROBLAZE_SETUP_H @@ -13,8 +10,6 @@ #include <uapi/asm/setup.h> # ifndef __ASSEMBLY__ -extern unsigned int boot_cpuid; /* move to smp.h */ - extern char cmd_line[COMMAND_LINE_SIZE]; extern char *klimit; diff --git a/arch/microblaze/include/asm/string.h b/arch/microblaze/include/asm/string.h index aec2f59298b8..34071a848b6a 100644 --- a/arch/microblaze/include/asm/string.h +++ b/arch/microblaze/include/asm/string.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_STRING_H diff --git a/arch/microblaze/include/asm/switch_to.h b/arch/microblaze/include/asm/switch_to.h index f45baa2c5e09..5afd6d9977b2 100644 --- a/arch/microblaze/include/asm/switch_to.h +++ b/arch/microblaze/include/asm/switch_to.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_SWITCH_TO_H diff --git a/arch/microblaze/include/asm/thread_info.h b/arch/microblaze/include/asm/thread_info.h index 9afe4b5bd6c8..ad8e8fcb90d3 100644 --- a/arch/microblaze/include/asm/thread_info.h +++ b/arch/microblaze/include/asm/thread_info.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_THREAD_INFO_H diff --git a/arch/microblaze/include/asm/timex.h b/arch/microblaze/include/asm/timex.h index befcf3de5532..e99cc29cbe57 100644 --- a/arch/microblaze/include/asm/timex.h +++ b/arch/microblaze/include/asm/timex.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_TIMEX_H diff --git a/arch/microblaze/include/asm/tlb.h b/arch/microblaze/include/asm/tlb.h deleted file mode 100644 index 628a78ee0a72..000000000000 --- a/arch/microblaze/include/asm/tlb.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> - * Copyright (C) 2008-2009 PetaLogix - * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ - -#ifndef _ASM_MICROBLAZE_TLB_H -#define _ASM_MICROBLAZE_TLB_H - -#include <linux/pagemap.h> -#include <asm-generic/tlb.h> - -#endif /* _ASM_MICROBLAZE_TLB_H */ diff --git a/arch/microblaze/include/asm/tlbflush.h b/arch/microblaze/include/asm/tlbflush.h index 2e1353c2d18d..6f8f5c77a050 100644 --- a/arch/microblaze/include/asm/tlbflush.h +++ b/arch/microblaze/include/asm/tlbflush.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_TLBFLUSH_H diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index a1f206b90753..070ba6139a62 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -1,18 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu> * Copyright (C) 2008-2009 PetaLogix * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_UACCESS_H #define _ASM_MICROBLAZE_UACCESS_H #include <linux/kernel.h> -#include <linux/mm.h> #include <asm/mmu.h> #include <asm/page.h> diff --git a/arch/microblaze/include/asm/unaligned.h b/arch/microblaze/include/asm/unaligned.h index b162ed880495..448299beab69 100644 --- a/arch/microblaze/include/asm/unaligned.h +++ b/arch/microblaze/include/asm/unaligned.h @@ -1,10 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2008 Michal Simek <monstr@monstr.eu> * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_UNALIGNED_H diff --git a/arch/microblaze/include/asm/unistd.h b/arch/microblaze/include/asm/unistd.h index d79d35ac6253..cfe3f888b432 100644 --- a/arch/microblaze/include/asm/unistd.h +++ b/arch/microblaze/include/asm/unistd.h @@ -1,10 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2007-2008 Michal Simek <monstr@monstr.eu> * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef _ASM_MICROBLAZE_UNISTD_H #define _ASM_MICROBLAZE_UNISTD_H diff --git a/arch/microblaze/include/asm/unwind.h b/arch/microblaze/include/asm/unwind.h index d248b7de4b13..c327d673622a 100644 --- a/arch/microblaze/include/asm/unwind.h +++ b/arch/microblaze/include/asm/unwind.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Backtrace support for Microblaze * * Copyright (C) 2010 Digital Design Corporation - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. */ #ifndef __MICROBLAZE_UNWIND_H diff --git a/arch/microblaze/include/asm/user.h b/arch/microblaze/include/asm/user.h deleted file mode 100644 index 8b137891791f..000000000000 --- a/arch/microblaze/include/asm/user.h +++ /dev/null @@ -1 +0,0 @@ - diff --git a/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c b/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c index a32daec96c12..c7ee51b0900e 100644 --- a/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c +++ b/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c @@ -22,13 +22,8 @@ #define CI(c, p) { ci->c = PVR_##p(pvr); } -#if defined(CONFIG_EARLY_PRINTK) && defined(CONFIG_SERIAL_UARTLITE_CONSOLE) #define err_printk(x) \ - early_printk("ERROR: Microblaze " x "-different for PVR and DTS\n"); -#else -#define err_printk(x) \ - pr_info("ERROR: Microblaze " x "-different for PVR and DTS\n"); -#endif + pr_err("ERROR: Microblaze " x "-different for PVR and DTS\n"); void set_cpuinfo_pvr_full(struct cpuinfo *ci, struct device_node *cpu) { diff --git a/arch/microblaze/kernel/cpu/pvr.c b/arch/microblaze/kernel/cpu/pvr.c index 8d0dc6db48cf..f139052a39bd 100644 --- a/arch/microblaze/kernel/cpu/pvr.c +++ b/arch/microblaze/kernel/cpu/pvr.c @@ -14,6 +14,7 @@ #include <linux/compiler.h> #include <asm/exceptions.h> #include <asm/pvr.h> +#include <linux/irqflags.h> /* * Until we get an assembler that knows about the pvr registers, diff --git a/arch/microblaze/kernel/entry.S b/arch/microblaze/kernel/entry.S index f6ded356394a..b179f8f6d287 100644 --- a/arch/microblaze/kernel/entry.S +++ b/arch/microblaze/kernel/entry.S @@ -958,6 +958,7 @@ ENTRY(_switch_to) nop ENTRY(_reset) + VM_OFF brai 0; /* Jump to reset vector */ /* These are compiled and loaded into high memory, then diff --git a/arch/microblaze/kernel/irq.c b/arch/microblaze/kernel/irq.c index 903dad822fad..0b37dde60a1e 100644 --- a/arch/microblaze/kernel/irq.c +++ b/arch/microblaze/kernel/irq.c @@ -20,29 +20,10 @@ #include <linux/irqchip.h> #include <linux/of_irq.h> -static u32 concurrent_irq; - void __irq_entry do_IRQ(struct pt_regs *regs) { - unsigned int irq; - struct pt_regs *old_regs = set_irq_regs(regs); trace_hardirqs_off(); - - irq_enter(); - irq = xintc_get_irq(); -next_irq: - BUG_ON(!irq); - generic_handle_irq(irq); - - irq = xintc_get_irq(); - if (irq != -1U) { - pr_debug("next irq: %d\n", irq); - ++concurrent_irq; - goto next_irq; - } - - irq_exit(); - set_irq_regs(old_regs); + handle_arch_irq(regs); trace_hardirqs_on(); } diff --git a/arch/microblaze/kernel/misc.S b/arch/microblaze/kernel/misc.S index 6759af688451..1228a09d8109 100644 --- a/arch/microblaze/kernel/misc.S +++ b/arch/microblaze/kernel/misc.S @@ -39,7 +39,7 @@ _tlbia_1: rsubi r11, r12, MICROBLAZE_TLB_SIZE - 1 bneid r11, _tlbia_1 /* loop for all entries */ addik r12, r12, 1 - /* sync */ + mbar 1 /* sync */ rtsd r15, 8 nop .size _tlbia, . - _tlbia @@ -58,6 +58,7 @@ _tlbie: blti r12, _tlbie_1 /* Check if found */ mts rtlbhi, r0 /* flush: ensure V is clear */ nop + mbar 1 /* sync */ _tlbie_1: rtsd r15, 8 nop diff --git a/arch/microblaze/kernel/setup.c b/arch/microblaze/kernel/setup.c index 511c1ab7f57f..dd121e33b8e3 100644 --- a/arch/microblaze/kernel/setup.c +++ b/arch/microblaze/kernel/setup.c @@ -41,7 +41,6 @@ DEFINE_PER_CPU(unsigned int, ENTRY_SP); /* Saved SP on kernel entry */ DEFINE_PER_CPU(unsigned int, R11_SAVE); /* Temp variable for entry */ DEFINE_PER_CPU(unsigned int, CURRENT_SAVE); /* Saved current pointer */ -unsigned int boot_cpuid; /* * Placed cmd_line to .data section because can be initialized from * ASM code. Default position is BSS section which is cleared @@ -54,7 +53,6 @@ void __init setup_arch(char **cmdline_p) *cmdline_p = boot_command_line; setup_memory(); - parse_early_param(); console_verbose(); diff --git a/arch/microblaze/kernel/timer.c b/arch/microblaze/kernel/timer.c index a6683484b3a1..f8832cf49384 100644 --- a/arch/microblaze/kernel/timer.c +++ b/arch/microblaze/kernel/timer.c @@ -161,13 +161,6 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction timer_irqaction = { - .handler = timer_interrupt, - .flags = IRQF_TIMER, - .name = "timer", - .dev_id = &clockevent_xilinx_timer, -}; - static __init int xilinx_clockevent_init(void) { clockevent_xilinx_timer.mult = @@ -309,7 +302,8 @@ static int __init xilinx_timer_init(struct device_node *timer) freq_div_hz = timer_clock_freq / HZ; - ret = setup_irq(irq, &timer_irqaction); + ret = request_irq(irq, timer_interrupt, IRQF_TIMER, "timer", + &clockevent_xilinx_timer); if (ret) { pr_err("Failed to setup IRQ"); return ret; diff --git a/arch/microblaze/kernel/vmlinux.lds.S b/arch/microblaze/kernel/vmlinux.lds.S index 2c09fa3a8a01..df07b3d06cd6 100644 --- a/arch/microblaze/kernel/vmlinux.lds.S +++ b/arch/microblaze/kernel/vmlinux.lds.S @@ -13,6 +13,7 @@ ENTRY(microblaze_start) #define RO_EXCEPTION_TABLE_ALIGN 16 +#include <asm/cache.h> #include <asm/page.h> #include <asm-generic/vmlinux.lds.h> #include <asm/thread_info.h> @@ -89,6 +90,8 @@ SECTIONS { _KERNEL_SDA_BASE_ = _ssro + (_ssro_size / 2) ; } + PERCPU_SECTION(L1_CACHE_BYTES) + . = ALIGN(PAGE_SIZE); __init_begin = .; diff --git a/arch/microblaze/mm/init.c b/arch/microblaze/mm/init.c index 1056f1674065..1ffbfa96b9b8 100644 --- a/arch/microblaze/mm/init.c +++ b/arch/microblaze/mm/init.c @@ -201,18 +201,6 @@ void __init mem_init(void) #endif mem_init_print_info(NULL); -#ifdef CONFIG_MMU - pr_info("Kernel virtual memory layout:\n"); - pr_info(" * 0x%08lx..0x%08lx : fixmap\n", FIXADDR_START, FIXADDR_TOP); -#ifdef CONFIG_HIGHMEM - pr_info(" * 0x%08lx..0x%08lx : highmem PTEs\n", - PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP)); -#endif /* CONFIG_HIGHMEM */ - pr_info(" * 0x%08lx..0x%08lx : early ioremap\n", - ioremap_bot, ioremap_base); - pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n", - (unsigned long)VMALLOC_START, VMALLOC_END); -#endif mem_init_done = 1; } @@ -347,6 +335,8 @@ asmlinkage void __init mmu_init(void) * inside 768MB limit */ memblock_set_current_limit(memory_start + lowmem_size - 1); + parse_early_param(); + /* CMA initialization */ dma_contiguous_reserve(memory_start + lowmem_size - 1); } diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 797d7f1ad5fe..f8f3dbdd7ad5 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -4,7 +4,6 @@ config MIPS default y select ARCH_32BIT_OFF_T if !64BIT select ARCH_BINFMT_ELF_STATE if MIPS_FP_SUPPORT - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_KCOV select ARCH_HAS_PTE_SPECIAL if !(32BIT && CPU_HAS_RIXI) @@ -51,6 +50,7 @@ config MIPS select HAVE_ASM_MODVERSIONS select HAVE_CBPF_JIT if !64BIT && !CPU_MICROMIPS select HAVE_CONTEXT_TRACKING + select HAVE_TIF_NOHZ select HAVE_COPY_THREAD_TLS select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_KMEMLEAK @@ -486,9 +486,11 @@ config MACH_LOONGSON64 select SYS_SUPPORTS_HIGHMEM select SYS_SUPPORTS_LITTLE_ENDIAN select SYS_SUPPORTS_ZBOOT - select LOONGSON_MC146818 select ZONE_DMA32 select NUMA + select COMMON_CLK + select USE_OF + select BUILTIN_DTB help This enables the support of Loongson-2/3 family of machines. @@ -973,6 +975,9 @@ config CAVIUM_OCTEON_SOC select SYS_HAS_EARLY_PRINTK select SYS_HAS_CPU_CAVIUM_OCTEON select HAVE_PCI + select HAVE_PLAT_DELAY + select HAVE_PLAT_FW_INIT_CMDLINE + select HAVE_PLAT_MEMCPY select ZONE_DMA32 select HOLES_IN_ZONE select GPIOLIB @@ -1229,6 +1234,15 @@ config GENERIC_ISA_DMA_SUPPORT_BROKEN bool select GENERIC_ISA_DMA +config HAVE_PLAT_DELAY + bool + +config HAVE_PLAT_FW_INIT_CMDLINE + bool + +config HAVE_PLAT_MEMCPY + bool + config ISA_DMA_API bool @@ -2676,7 +2690,7 @@ config NUMA Say Y to compile the kernel to support NUMA (Non-Uniform Memory Access). This option improves performance on systems with more than two nodes; on two node systems it is generally better to - leave it disabled; on single node systems disable this option + leave it disabled; on single node systems leave this option disabled. config SYS_SUPPORTS_NUMA @@ -2758,6 +2772,17 @@ config HW_PERF_EVENTS Enable hardware performance counter support for perf events. If disabled, perf events will use software events only. +config DMI + bool "Enable DMI scanning" + depends on MACH_LOONGSON64 + select DMI_SCAN_MACHINE_NON_EFI_FALLBACK + default y + help + Enabled scanning of DMI to identify machine quirks. Say Y + here unless you have verified that your setup is not + affected by entries in the DMI blacklist. Required by PNP + BIOS code. + config SMP bool "Multi-Processing support" depends on SYS_SUPPORTS_SMP @@ -3070,7 +3095,7 @@ endchoice choice prompt "Kernel command line type" if !CMDLINE_OVERRIDE default MIPS_CMDLINE_FROM_DTB if USE_OF && !ATH79 && !MACH_INGENIC && \ - !MIPS_MALTA && \ + !MACH_LOONGSON64 && !MIPS_MALTA && \ !CAVIUM_OCTEON_SOC default MIPS_CMDLINE_FROM_BOOTLOADER diff --git a/arch/mips/alchemy/common/time.c b/arch/mips/alchemy/common/time.c index 3a21a6a18da7..d794ffb67412 100644 --- a/arch/mips/alchemy/common/time.c +++ b/arch/mips/alchemy/common/time.c @@ -72,13 +72,6 @@ static struct clock_event_device au1x_rtcmatch2_clockdev = { .cpumask = cpu_possible_mask, }; -static struct irqaction au1x_rtcmatch2_irqaction = { - .handler = au1x_rtcmatch2_irq, - .flags = IRQF_TIMER, - .name = "timer", - .dev_id = &au1x_rtcmatch2_clockdev, -}; - static int __init alchemy_time_init(unsigned int m2int) { struct clock_event_device *cd = &au1x_rtcmatch2_clockdev; @@ -130,7 +123,9 @@ static int __init alchemy_time_init(unsigned int m2int) cd->min_delta_ns = clockevent_delta2ns(9, cd); cd->min_delta_ticks = 9; /* ~0.28ms */ clockevents_register_device(cd); - setup_irq(m2int, &au1x_rtcmatch2_irqaction); + if (request_irq(m2int, au1x_rtcmatch2_irq, IRQF_TIMER, "timer", + &au1x_rtcmatch2_clockdev)) + pr_err("Failed to register timer interrupt\n"); printk(KERN_INFO "Alchemy clocksource installed\n"); diff --git a/arch/mips/ar7/irq.c b/arch/mips/ar7/irq.c index 93a331fe0641..f0a7942d393e 100644 --- a/arch/mips/ar7/irq.c +++ b/arch/mips/ar7/irq.c @@ -83,12 +83,6 @@ static struct irq_chip ar7_sec_irq_type = { .irq_ack = ar7_ack_sec_irq, }; -static struct irqaction ar7_cascade_action = { - .handler = no_action, - .name = "AR7 cascade interrupt", - .flags = IRQF_NO_THREAD, -}; - static void __init ar7_irq_init(int base) { int i; @@ -116,8 +110,14 @@ static void __init ar7_irq_init(int base) handle_level_irq); } - setup_irq(2, &ar7_cascade_action); - setup_irq(ar7_irq_base, &ar7_cascade_action); + if (request_irq(2, no_action, IRQF_NO_THREAD, "AR7 cascade interrupt", + NULL)) + pr_err("Failed to request irq 2 (AR7 cascade interrupt)\n"); + if (request_irq(ar7_irq_base, no_action, IRQF_NO_THREAD, + "AR7 cascade interrupt", NULL)) { + pr_err("Failed to request irq %d (AR7 cascade interrupt)\n", + ar7_irq_base); + } set_c0_status(IE_IRQ0); } diff --git a/arch/mips/ath25/ar2315.c b/arch/mips/ath25/ar2315.c index 24f619199ee7..e7b53e3960c8 100644 --- a/arch/mips/ath25/ar2315.c +++ b/arch/mips/ath25/ar2315.c @@ -64,11 +64,6 @@ static irqreturn_t ar2315_ahb_err_handler(int cpl, void *dev_id) return IRQ_HANDLED; } -static struct irqaction ar2315_ahb_err_interrupt = { - .handler = ar2315_ahb_err_handler, - .name = "ar2315-ahb-error", -}; - static void ar2315_misc_irq_handler(struct irq_desc *desc) { u32 pending = ar2315_rst_reg_read(AR2315_ISR) & @@ -159,7 +154,9 @@ void __init ar2315_arch_init_irq(void) panic("Failed to add IRQ domain"); irq = irq_create_mapping(domain, AR2315_MISC_IRQ_AHB); - setup_irq(irq, &ar2315_ahb_err_interrupt); + if (request_irq(irq, ar2315_ahb_err_handler, 0, "ar2315-ahb-error", + NULL)) + pr_err("Failed to register ar2315-ahb-error interrupt\n"); irq_set_chained_handler_and_data(AR2315_IRQ_MISC, ar2315_misc_irq_handler, domain); diff --git a/arch/mips/ath25/ar5312.c b/arch/mips/ath25/ar5312.c index 47f3e98974fc..42bf2afb4765 100644 --- a/arch/mips/ath25/ar5312.c +++ b/arch/mips/ath25/ar5312.c @@ -68,11 +68,6 @@ static irqreturn_t ar5312_ahb_err_handler(int cpl, void *dev_id) return IRQ_HANDLED; } -static struct irqaction ar5312_ahb_err_interrupt = { - .handler = ar5312_ahb_err_handler, - .name = "ar5312-ahb-error", -}; - static void ar5312_misc_irq_handler(struct irq_desc *desc) { u32 pending = ar5312_rst_reg_read(AR5312_ISR) & @@ -154,7 +149,9 @@ void __init ar5312_arch_init_irq(void) panic("Failed to add IRQ domain"); irq = irq_create_mapping(domain, AR5312_MISC_IRQ_AHB_PROC); - setup_irq(irq, &ar5312_ahb_err_interrupt); + if (request_irq(irq, ar5312_ahb_err_handler, 0, "ar5312-ahb-error", + NULL)) + pr_err("Failed to register ar5312-ahb-error interrupt\n"); irq_set_chained_handler_and_data(AR5312_IRQ_MISC, ar5312_misc_irq_handler, domain); diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index 484ee28922a9..acb4fd647a30 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -15,7 +15,7 @@ #include <linux/memblock.h> #include <linux/err.h> #include <linux/clk.h> -#include <linux/clk-provider.h> +#include <linux/of_clk.h> #include <linux/of_fdt.h> #include <linux/irqchip.h> diff --git a/arch/mips/bcm63xx/irq.c b/arch/mips/bcm63xx/irq.c index ec694b9628c0..2548013442f6 100644 --- a/arch/mips/bcm63xx/irq.c +++ b/arch/mips/bcm63xx/irq.c @@ -399,26 +399,6 @@ static struct irq_chip bcm63xx_external_irq_chip = { .irq_set_type = bcm63xx_external_irq_set_type, }; -static struct irqaction cpu_ip2_cascade_action = { - .handler = no_action, - .name = "cascade_ip2", - .flags = IRQF_NO_THREAD, -}; - -#ifdef CONFIG_SMP -static struct irqaction cpu_ip3_cascade_action = { - .handler = no_action, - .name = "cascade_ip3", - .flags = IRQF_NO_THREAD, -}; -#endif - -static struct irqaction cpu_ext_cascade_action = { - .handler = no_action, - .name = "cascade_extirq", - .flags = IRQF_NO_THREAD, -}; - static void bcm63xx_init_irq(void) { int irq_bits; @@ -531,7 +511,7 @@ static void bcm63xx_init_irq(void) void __init arch_init_irq(void) { - int i; + int i, irq; bcm63xx_init_irq(); mips_cpu_irq_init(); @@ -544,14 +524,25 @@ void __init arch_init_irq(void) handle_edge_irq); if (!is_ext_irq_cascaded) { - for (i = 3; i < 3 + ext_irq_count; ++i) - setup_irq(MIPS_CPU_IRQ_BASE + i, &cpu_ext_cascade_action); + for (i = 3; i < 3 + ext_irq_count; ++i) { + irq = MIPS_CPU_IRQ_BASE + i; + if (request_irq(irq, no_action, IRQF_NO_THREAD, + "cascade_extirq", NULL)) { + pr_err("Failed to request irq %d (cascade_extirq)\n", + irq); + } + } } - setup_irq(MIPS_CPU_IRQ_BASE + 2, &cpu_ip2_cascade_action); + irq = MIPS_CPU_IRQ_BASE + 2; + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade_ip2", NULL)) + pr_err("Failed to request irq %d (cascade_ip2)\n", irq); #ifdef CONFIG_SMP if (is_ext_irq_cascaded) { - setup_irq(MIPS_CPU_IRQ_BASE + 3, &cpu_ip3_cascade_action); + irq = MIPS_CPU_IRQ_BASE + 3; + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade_ip3", + NULL)) + pr_err("Failed to request irq %d (cascade_ip3)\n", irq); bcm63xx_internal_irq_chip.irq_set_affinity = bcm63xx_internal_set_affinity; diff --git a/arch/mips/bmips/setup.c b/arch/mips/bmips/setup.c index 2f81a94c71a6..19308df5f577 100644 --- a/arch/mips/bmips/setup.c +++ b/arch/mips/bmips/setup.c @@ -10,11 +10,11 @@ #include <linux/init.h> #include <linux/bitops.h> #include <linux/memblock.h> -#include <linux/clk-provider.h> #include <linux/ioport.h> #include <linux/kernel.h> #include <linux/io.h> #include <linux/of.h> +#include <linux/of_clk.h> #include <linux/of_fdt.h> #include <linux/of_platform.h> #include <linux/libfdt.h> diff --git a/arch/mips/boot/dts/Makefile b/arch/mips/boot/dts/Makefile index 1e79cab8e269..d429a69bfe30 100644 --- a/arch/mips/boot/dts/Makefile +++ b/arch/mips/boot/dts/Makefile @@ -4,6 +4,7 @@ subdir-y += cavium-octeon subdir-y += img subdir-y += ingenic subdir-y += lantiq +subdir-y += loongson subdir-y += mscc subdir-y += mti subdir-y += netlogic diff --git a/arch/mips/boot/dts/ingenic/ci20.dts b/arch/mips/boot/dts/ingenic/ci20.dts index c340f947baa0..db0ca250bd1a 100644 --- a/arch/mips/boot/dts/ingenic/ci20.dts +++ b/arch/mips/boot/dts/ingenic/ci20.dts @@ -4,6 +4,7 @@ #include "jz4780.dtsi" #include <dt-bindings/clock/ingenic,tcu.h> #include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/input/input.h> #include <dt-bindings/interrupt-controller/irq.h> #include <dt-bindings/regulator/active-semi,8865-regulator.h> @@ -27,6 +28,17 @@ 0x30000000 0x30000000>; }; + gpio-keys { + compatible = "gpio-keys"; + + sw1 { + label = "ci20:sw1"; + linux,code = <KEY_F13>; + gpios = <&gpd 17 GPIO_ACTIVE_HIGH>; + wakeup-source; + }; + }; + leds { compatible = "gpio-leds"; @@ -58,10 +70,17 @@ eth0_power: fixedregulator@0 { compatible = "regulator-fixed"; regulator-name = "eth0_power"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; gpio = <&gpb 25 GPIO_ACTIVE_LOW>; enable-active-high; }; + ir: ir { + compatible = "gpio-ir-receiver"; + gpios = <&gpe 3 GPIO_ACTIVE_LOW>; + }; + wlan0_power: fixedregulator@1 { compatible = "regulator-fixed"; regulator-name = "wlan0_power"; diff --git a/arch/mips/boot/dts/loongson/Makefile b/arch/mips/boot/dts/loongson/Makefile new file mode 100644 index 000000000000..56d379471262 --- /dev/null +++ b/arch/mips/boot/dts/loongson/Makefile @@ -0,0 +1,4 @@ +# SPDX_License_Identifier: GPL_2.0 +dtb-$(CONFIG_MACH_LOONGSON64) += loongson3_4core_rs780e.dtb loongson3_8core_rs780e.dtb + +obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/mips/boot/dts/loongson/loongson3-package.dtsi b/arch/mips/boot/dts/loongson/loongson3-package.dtsi new file mode 100644 index 000000000000..5bb876a4de52 --- /dev/null +++ b/arch/mips/boot/dts/loongson/loongson3-package.dtsi @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <dt-bindings/interrupt-controller/irq.h> + +/ { + #address-cells = <2>; + #size-cells = <2>; + + cpuintc: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + compatible = "mti,cpu-interrupt-controller"; + }; + + package0: bus@1fe00000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <1>; + ranges = <0 0x1fe00000 0 0x1fe00000 0x100000 + 0 0x3ff00000 0 0x3ff00000 0x100000 + /* 3A HT Config Space */ + 0xefd 0xfb000000 0xefd 0xfb000000 0x10000000 + /* 3B HT Config Space */ + 0x1efd 0xfb000000 0x1efd 0xfb000000 0x10000000>; + + liointc: interrupt-controller@3ff01400 { + compatible = "loongson,liointc-1.0"; + reg = <0 0x3ff01400 0x64>; + + interrupt-controller; + #interrupt-cells = <2>; + + interrupt-parent = <&cpuintc>; + interrupts = <2>, <3>; + interrupt-names = "int0", "int1"; + + loongson,parent_int_map = <0xf0ffffff>, /* int0 */ + <0x0f000000>, /* int1 */ + <0x00000000>, /* int2 */ + <0x00000000>; /* int3 */ + + }; + + cpu_uart0: serial@1fe001e0 { + compatible = "ns16550a"; + reg = <0 0x1fe001e0 0x8>; + clock-frequency = <33000000>; + interrupt-parent = <&liointc>; + interrupts = <10 IRQ_TYPE_LEVEL_HIGH>; + no-loopback-test; + }; + + cpu_uart1: serial@1fe001e8 { + status = "disabled"; + compatible = "ns16550a"; + reg = <0 0x1fe001e8 0x8>; + clock-frequency = <33000000>; + interrupts = <10 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&liointc>; + no-loopback-test; + }; + }; +}; diff --git a/arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dts b/arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dts new file mode 100644 index 000000000000..6b5694ca0f95 --- /dev/null +++ b/arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +/dts-v1/; + +#include "loongson3-package.dtsi" +#include "rs780e-pch.dtsi" + +/ { + compatible = "loongson,loongson3-4core-rs780e"; +}; + +&package0 { + htpic: interrupt-controller@efdfb000080 { + compatible = "loongson,htpic-1.0"; + reg = <0xefd 0xfb000080 0x40>; + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-parent = <&liointc>; + interrupts = <24 IRQ_TYPE_LEVEL_HIGH>, + <25 IRQ_TYPE_LEVEL_HIGH>, + <26 IRQ_TYPE_LEVEL_HIGH>, + <27 IRQ_TYPE_LEVEL_HIGH>; + }; +}; diff --git a/arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dts b/arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dts new file mode 100644 index 000000000000..ffefa2f829b0 --- /dev/null +++ b/arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +/dts-v1/; + +#include "loongson3-package.dtsi" +#include "rs780e-pch.dtsi" + +/ { + compatible = "loongson,loongson3-8core-rs780e"; +}; + +&package0 { + htpic: interrupt-controller@1efdfb000080 { + compatible = "loongson,htpic-1.0"; + reg = <0x1efd 0xfb000080 0x40>; + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-parent = <&liointc>; + interrupts = <24 IRQ_TYPE_LEVEL_HIGH>, + <25 IRQ_TYPE_LEVEL_HIGH>, + <26 IRQ_TYPE_LEVEL_HIGH>, + <27 IRQ_TYPE_LEVEL_HIGH>; + }; +}; diff --git a/arch/mips/boot/dts/loongson/rs780e-pch.dtsi b/arch/mips/boot/dts/loongson/rs780e-pch.dtsi new file mode 100644 index 000000000000..45c54d555fa4 --- /dev/null +++ b/arch/mips/boot/dts/loongson/rs780e-pch.dtsi @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +/ { + bus@10000000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0 0x10000000 0 0x10000000 0 0x10000000 + 0 0x40000000 0 0x40000000 0 0x40000000 + 0xfd 0xfe000000 0xfd 0xfe000000 0 0x2000000 /* PCI Config Space */>; + + isa { + compatible = "isa"; + #address-cells = <2>; + #size-cells = <1>; + ranges = <1 0 0 0 0x1000>; + + rtc0: rtc@70 { + compatible = "motorola,mc146818"; + reg = <1 0x70 0x8>; + interrupts = <8>; + interrupt-parent = <&htpic>; + }; + }; + }; +}; diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-board.c b/arch/mips/cavium-octeon/executive/cvmx-helper-board.c index 2e2d45bc850d..abd11b7af22f 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-board.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-board.c @@ -207,9 +207,9 @@ int cvmx_helper_board_get_mii_address(int ipd_port) * Returns The ports link status. If the link isn't fully resolved, this must * return zero. */ -cvmx_helper_link_info_t __cvmx_helper_board_link_get(int ipd_port) +union cvmx_helper_link_info __cvmx_helper_board_link_get(int ipd_port) { - cvmx_helper_link_info_t result; + union cvmx_helper_link_info result; WARN(!octeon_is_simulation(), "Using deprecated link status - please update your DT"); diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c b/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c index e812ed9a03bb..c4b58598aa9d 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c @@ -261,7 +261,7 @@ int __cvmx_helper_rgmii_enable(int interface) * * Returns Link state */ -cvmx_helper_link_info_t __cvmx_helper_rgmii_link_get(int ipd_port) +union cvmx_helper_link_info __cvmx_helper_rgmii_link_get(int ipd_port) { int interface = cvmx_helper_get_interface_num(ipd_port); int index = cvmx_helper_get_interface_index_num(ipd_port); @@ -270,7 +270,7 @@ cvmx_helper_link_info_t __cvmx_helper_rgmii_link_get(int ipd_port) asxx_prt_loop.u64 = cvmx_read_csr(CVMX_ASXX_PRT_LOOP(interface)); if (asxx_prt_loop.s.int_loop & (1 << index)) { /* Force 1Gbps full duplex on internal loopback */ - cvmx_helper_link_info_t result; + union cvmx_helper_link_info result; result.u64 = 0; result.s.full_duplex = 1; result.s.link_up = 1; @@ -292,7 +292,7 @@ cvmx_helper_link_info_t __cvmx_helper_rgmii_link_get(int ipd_port) * Returns Zero on success, negative on failure */ int __cvmx_helper_rgmii_link_set(int ipd_port, - cvmx_helper_link_info_t link_info) + union cvmx_helper_link_info link_info) { int result = 0; int interface = cvmx_helper_get_interface_num(ipd_port); diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c b/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c index f6ebf63dc84c..e07d8f15efba 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c @@ -200,7 +200,7 @@ static int __cvmx_helper_sgmii_hardware_init_link(int interface, int index) */ static int __cvmx_helper_sgmii_hardware_init_link_speed(int interface, int index, - cvmx_helper_link_info_t + union cvmx_helper_link_info link_info) { int is_enabled; @@ -394,9 +394,9 @@ int __cvmx_helper_sgmii_enable(int interface) * * Returns Link state */ -cvmx_helper_link_info_t __cvmx_helper_sgmii_link_get(int ipd_port) +union cvmx_helper_link_info __cvmx_helper_sgmii_link_get(int ipd_port) { - cvmx_helper_link_info_t result; + union cvmx_helper_link_info result; union cvmx_pcsx_miscx_ctl_reg pcs_misc_ctl_reg; int interface = cvmx_helper_get_interface_num(ipd_port); int index = cvmx_helper_get_interface_index_num(ipd_port); @@ -505,7 +505,7 @@ cvmx_helper_link_info_t __cvmx_helper_sgmii_link_get(int ipd_port) * Returns Zero on success, negative on failure */ int __cvmx_helper_sgmii_link_set(int ipd_port, - cvmx_helper_link_info_t link_info) + union cvmx_helper_link_info link_info) { int interface = cvmx_helper_get_interface_num(ipd_port); int index = cvmx_helper_get_interface_index_num(ipd_port); diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c b/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c index 2a574d272671..525914e9b22d 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c @@ -140,9 +140,9 @@ int __cvmx_helper_spi_enable(int interface) * * Returns Link state */ -cvmx_helper_link_info_t __cvmx_helper_spi_link_get(int ipd_port) +union cvmx_helper_link_info __cvmx_helper_spi_link_get(int ipd_port) { - cvmx_helper_link_info_t result; + union cvmx_helper_link_info result; int interface = cvmx_helper_get_interface_num(ipd_port); int index = cvmx_helper_get_interface_index_num(ipd_port); result.u64 = 0; @@ -193,7 +193,7 @@ cvmx_helper_link_info_t __cvmx_helper_spi_link_get(int ipd_port) * * Returns Zero on success, negative on failure */ -int __cvmx_helper_spi_link_set(int ipd_port, cvmx_helper_link_info_t link_info) +int __cvmx_helper_spi_link_set(int ipd_port, union cvmx_helper_link_info link_info) { /* Nothing to do. If we have a SPI4000 then the setup was already performed by cvmx_spi4000_check_speed(). If not then there isn't any link diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c b/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c index 93a498d05184..842990e8404f 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c @@ -259,13 +259,13 @@ int __cvmx_helper_xaui_enable(int interface) * * Returns Link state */ -cvmx_helper_link_info_t __cvmx_helper_xaui_link_get(int ipd_port) +union cvmx_helper_link_info __cvmx_helper_xaui_link_get(int ipd_port) { int interface = cvmx_helper_get_interface_num(ipd_port); union cvmx_gmxx_tx_xaui_ctl gmxx_tx_xaui_ctl; union cvmx_gmxx_rx_xaui_ctl gmxx_rx_xaui_ctl; union cvmx_pcsxx_status1_reg pcsxx_status1_reg; - cvmx_helper_link_info_t result; + union cvmx_helper_link_info result; gmxx_tx_xaui_ctl.u64 = cvmx_read_csr(CVMX_GMXX_TX_XAUI_CTL(interface)); gmxx_rx_xaui_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RX_XAUI_CTL(interface)); @@ -299,7 +299,7 @@ cvmx_helper_link_info_t __cvmx_helper_xaui_link_get(int ipd_port) * * Returns Zero on success, negative on failure */ -int __cvmx_helper_xaui_link_set(int ipd_port, cvmx_helper_link_info_t link_info) +int __cvmx_helper_xaui_link_set(int ipd_port, union cvmx_helper_link_info link_info) { int interface = cvmx_helper_get_interface_num(ipd_port); union cvmx_gmxx_tx_xaui_ctl gmxx_tx_xaui_ctl; diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper.c b/arch/mips/cavium-octeon/executive/cvmx-helper.c index de391541d6f7..6044ff471002 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper.c @@ -782,9 +782,9 @@ static int __cvmx_helper_errata_fix_ipd_ptr_alignment(void) #define INTERFACE(port) (port >> 4) #define INDEX(port) (port & 0xf) uint64_t *p64; - cvmx_pko_command_word0_t pko_command; + union cvmx_pko_command_word0 pko_command; union cvmx_buf_ptr g_buffer, pkt_buffer; - cvmx_wqe_t *work; + struct cvmx_wqe *work; int size, num_segs = 0, wqe_pcnt, pkt_pcnt; union cvmx_gmxx_prtx_cfg gmx_cfg; int retry_cnt; @@ -1075,9 +1075,9 @@ int cvmx_helper_initialize_packet_io_local(void) * * Returns Link state */ -cvmx_helper_link_info_t cvmx_helper_link_get(int ipd_port) +union cvmx_helper_link_info cvmx_helper_link_get(int ipd_port) { - cvmx_helper_link_info_t result; + union cvmx_helper_link_info result; int interface = cvmx_helper_get_interface_num(ipd_port); int index = cvmx_helper_get_interface_index_num(ipd_port); @@ -1136,7 +1136,7 @@ EXPORT_SYMBOL_GPL(cvmx_helper_link_get); * * Returns Zero on success, negative on failure */ -int cvmx_helper_link_set(int ipd_port, cvmx_helper_link_info_t link_info) +int cvmx_helper_link_set(int ipd_port, union cvmx_helper_link_info link_info) { int result = -1; int interface = cvmx_helper_get_interface_num(ipd_port); diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index 6bd1e97effdf..6501a842c41a 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -2199,6 +2199,9 @@ static int octeon_irq_cib_map(struct irq_domain *d, } cd = kzalloc(sizeof(*cd), GFP_KERNEL); + if (!cd) + return -ENOMEM; + cd->host_data = host_data; cd->bit = hw; diff --git a/arch/mips/cobalt/irq.c b/arch/mips/cobalt/irq.c index 965c777d3561..ead5ae41378d 100644 --- a/arch/mips/cobalt/irq.c +++ b/arch/mips/cobalt/irq.c @@ -45,18 +45,20 @@ asmlinkage void plat_irq_dispatch(void) spurious_interrupt(); } -static struct irqaction cascade = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; - void __init arch_init_irq(void) { mips_cpu_irq_init(); gt641xx_irq_init(); init_i8259_irqs(); - setup_irq(GT641XX_CASCADE_IRQ, &cascade); - setup_irq(I8259_CASCADE_IRQ, &cascade); + if (request_irq(GT641XX_CASCADE_IRQ, no_action, IRQF_NO_THREAD, + "cascade", NULL)) { + pr_err("Failed to request irq %d (cascade)\n", + GT641XX_CASCADE_IRQ); + } + if (request_irq(I8259_CASCADE_IRQ, no_action, IRQF_NO_THREAD, + "cascade", NULL)) { + pr_err("Failed to request irq %d (cascade)\n", + I8259_CASCADE_IRQ); + } } diff --git a/arch/mips/configs/ar7_defconfig b/arch/mips/configs/ar7_defconfig index cef2754bd408..cf9c6329b807 100644 --- a/arch/mips/configs/ar7_defconfig +++ b/arch/mips/configs/ar7_defconfig @@ -21,7 +21,6 @@ CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y CONFIG_BSD_DISKLABEL=y -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/mips/configs/ath25_defconfig b/arch/mips/configs/ath25_defconfig index c35add2fd716..7143441f5476 100644 --- a/arch/mips/configs/ath25_defconfig +++ b/arch/mips/configs/ath25_defconfig @@ -22,7 +22,6 @@ CONFIG_HZ_100=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set -# CONFIG_IOSCHED_CFQ is not set # CONFIG_COMPACTION is not set CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/mips/configs/ath79_defconfig b/arch/mips/configs/ath79_defconfig index 4ffc59cab436..3d14d67dc746 100644 --- a/arch/mips/configs/ath79_defconfig +++ b/arch/mips/configs/ath79_defconfig @@ -23,7 +23,6 @@ CONFIG_PCI=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/mips/configs/bcm63xx_defconfig b/arch/mips/configs/bcm63xx_defconfig index 54e2f9a659fb..861f680184b9 100644 --- a/arch/mips/configs/bcm63xx_defconfig +++ b/arch/mips/configs/bcm63xx_defconfig @@ -21,8 +21,6 @@ CONFIG_PCI=y CONFIG_PCCARD=y CONFIG_PCMCIA_BCM63XX=y # CONFIG_BLK_DEV_BSG is not set -# CONFIG_IOSCHED_DEADLINE is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_UNIX=y CONFIG_INET=y diff --git a/arch/mips/configs/bmips_be_defconfig b/arch/mips/configs/bmips_be_defconfig index f669a40e085b..032bb51defe8 100644 --- a/arch/mips/configs/bmips_be_defconfig +++ b/arch/mips/configs/bmips_be_defconfig @@ -12,8 +12,6 @@ CONFIG_NR_CPUS=4 # CONFIG_SECCOMP is not set CONFIG_MIPS_O32_FP64_SUPPORT=y # CONFIG_BLK_DEV_BSG is not set -# CONFIG_IOSCHED_DEADLINE is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_PACKET_DIAG=y diff --git a/arch/mips/configs/bmips_stb_defconfig b/arch/mips/configs/bmips_stb_defconfig index a0b775893dba..625bd2d7e685 100644 --- a/arch/mips/configs/bmips_stb_defconfig +++ b/arch/mips/configs/bmips_stb_defconfig @@ -21,8 +21,6 @@ CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y CONFIG_BMIPS_CPUFREQ=y # CONFIG_BLK_DEV_BSG is not set -# CONFIG_IOSCHED_DEADLINE is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_PACKET_DIAG=y diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig index be41df2a81fb..0db0088bbc1c 100644 --- a/arch/mips/configs/ci20_defconfig +++ b/arch/mips/configs/ci20_defconfig @@ -1,4 +1,5 @@ # CONFIG_LOCALVERSION_AUTO is not set +CONFIG_MODULES=y CONFIG_KERNEL_XZ=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y @@ -88,12 +89,14 @@ CONFIG_I2C_JZ4780=y CONFIG_SPI=y CONFIG_SPI_GPIO=y CONFIG_GPIO_SYSFS=y +CONFIG_KEYBOARD_GPIO=m # CONFIG_HWMON is not set CONFIG_WATCHDOG=y CONFIG_JZ4740_WDT=y CONFIG_REGULATOR=y CONFIG_REGULATOR_DEBUG=y CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_ACT8865=y # CONFIG_VGA_CONSOLE is not set # CONFIG_HID is not set # CONFIG_USB_SUPPORT is not set @@ -166,3 +169,21 @@ CONFIG_STACKTRACE=y # CONFIG_FTRACE is not set CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="earlycon console=ttyS4,115200 clk_ignore_unused" +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_TRIGGERS=y +CONFIG_LEDS_TRIGGER_MTD=y +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=m +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_LEDS_TRIGGER_TRANSIENT=y +CONFIG_LEDS_TRIGGER_CAMERA=m +CONFIG_LIRC=y +CONFIG_MEDIA_SUPPORT=m +CONFIG_RC_DEVICES=y +CONFIG_IR_GPIO_CIR=m +CONFIG_IR_GPIO_TX=m diff --git a/arch/mips/configs/db1xxx_defconfig b/arch/mips/configs/db1xxx_defconfig index bc9b6ae046b2..e6f3e8e3da39 100644 --- a/arch/mips/configs/db1xxx_defconfig +++ b/arch/mips/configs/db1xxx_defconfig @@ -28,7 +28,6 @@ CONFIG_PCMCIA_ALCHEMY_DEVBOARD=y CONFIG_FIRMWARE_MEMMAP=y CONFIG_BLK_DEV_BSGLIB=y CONFIG_PARTITION_ADVANCED=y -CONFIG_DEFAULT_NOOP=y CONFIG_CMA=y CONFIG_CMA_DEBUG=y CONFIG_NET=y diff --git a/arch/mips/configs/generic/board-ni169445.config b/arch/mips/configs/generic/board-ni169445.config index 1ed0d3e8715e..fc3580e4e6bc 100644 --- a/arch/mips/configs/generic/board-ni169445.config +++ b/arch/mips/configs/generic/board-ni169445.config @@ -19,7 +19,6 @@ CONFIG_MTD_NAND_ECC_SW_HAMMING=y CONFIG_MTD_NAND_ECC_SW_BCH=y CONFIG_MTD_RAW_NAND=y CONFIG_MTD_NAND_GPIO=y -CONFIG_MTD_NAND_IDS=y CONFIG_MTD_UBI=y CONFIG_MTD_UBI_BLOCK=y diff --git a/arch/mips/configs/jazz_defconfig b/arch/mips/configs/jazz_defconfig index 328d4dfeb4cb..92085df3e367 100644 --- a/arch/mips/configs/jazz_defconfig +++ b/arch/mips/configs/jazz_defconfig @@ -1,15 +1,17 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT_VOLUNTARY=y CONFIG_BSD_PROCESS_ACCT=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_RELAY=y CONFIG_EXPERT=y CONFIG_SLAB=y CONFIG_MACH_JAZZ=y CONFIG_OLIVETTI_M700=y +CONFIG_MIPS_MAGNUM_4000=y +CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y @@ -18,168 +20,17 @@ CONFIG_BINFMT_MISC=m CONFIG_NET=y CONFIG_PACKET=m CONFIG_UNIX=y -CONFIG_NET_KEY=m -CONFIG_NET_KEY_MIGRATE=y CONFIG_INET=y CONFIG_IP_MULTICAST=y CONFIG_NET_IPIP=m -CONFIG_IP_MROUTE=y -CONFIG_IP_PIMSM_V1=y -CONFIG_IP_PIMSM_V2=y -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_TCP_MD5SIG=y -CONFIG_IPV6_ROUTER_PREF=y -CONFIG_IPV6_ROUTE_INFO=y CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m -CONFIG_IPV6_TUNNEL=m -CONFIG_NETWORK_SECMARK=y -CONFIG_NETFILTER=y -CONFIG_NF_CONNTRACK=m -CONFIG_NF_CONNTRACK_SECMARK=y -CONFIG_NF_CONNTRACK_EVENTS=y -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CT_NETLINK=m -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_SECMARK=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_IP_NF_IPTABLES=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_FILTER=m -CONFIG_IP_NF_TARGET_REJECT=m -CONFIG_IP_NF_MANGLE=m -CONFIG_IP_NF_TARGET_CLUSTERIP=m -CONFIG_IP_NF_TARGET_ECN=m -CONFIG_IP_NF_TARGET_TTL=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_ARPTABLES=m -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_IP6_NF_IPTABLES=m -CONFIG_IP6_NF_MATCH_AH=m -CONFIG_IP6_NF_MATCH_EUI64=m -CONFIG_IP6_NF_MATCH_FRAG=m -CONFIG_IP6_NF_MATCH_OPTS=m -CONFIG_IP6_NF_MATCH_HL=m -CONFIG_IP6_NF_MATCH_IPV6HEADER=m -CONFIG_IP6_NF_MATCH_MH=m -CONFIG_IP6_NF_MATCH_RT=m -CONFIG_IP6_NF_TARGET_HL=m -CONFIG_IP6_NF_FILTER=m -CONFIG_IP6_NF_TARGET_REJECT=m -CONFIG_IP6_NF_MANGLE=m -CONFIG_IP6_NF_RAW=m -CONFIG_DECNET_NF_GRABULATOR=m -CONFIG_BRIDGE_NF_EBTABLES=m -CONFIG_BRIDGE_EBT_BROUTE=m -CONFIG_BRIDGE_EBT_T_FILTER=m -CONFIG_BRIDGE_EBT_T_NAT=m -CONFIG_BRIDGE_EBT_802_3=m -CONFIG_BRIDGE_EBT_AMONG=m -CONFIG_BRIDGE_EBT_ARP=m -CONFIG_BRIDGE_EBT_IP=m -CONFIG_BRIDGE_EBT_LIMIT=m -CONFIG_BRIDGE_EBT_MARK=m -CONFIG_BRIDGE_EBT_PKTTYPE=m -CONFIG_BRIDGE_EBT_STP=m -CONFIG_BRIDGE_EBT_VLAN=m -CONFIG_BRIDGE_EBT_ARPREPLY=m -CONFIG_BRIDGE_EBT_DNAT=m -CONFIG_BRIDGE_EBT_MARK_T=m -CONFIG_BRIDGE_EBT_REDIRECT=m -CONFIG_BRIDGE_EBT_SNAT=m -CONFIG_BRIDGE_EBT_LOG=m -CONFIG_BRIDGE=m -CONFIG_DECNET=m -CONFIG_NET_SCHED=y -CONFIG_NET_SCH_CBQ=m -CONFIG_NET_SCH_HTB=m -CONFIG_NET_SCH_HFSC=m -CONFIG_NET_SCH_PRIO=m -CONFIG_NET_SCH_RED=m -CONFIG_NET_SCH_SFQ=m -CONFIG_NET_SCH_TEQL=m -CONFIG_NET_SCH_TBF=m -CONFIG_NET_SCH_GRED=m -CONFIG_NET_SCH_DSMARK=m -CONFIG_NET_SCH_NETEM=m -CONFIG_NET_CLS_BASIC=m -CONFIG_NET_CLS_TCINDEX=m -CONFIG_NET_CLS_ROUTE4=m -CONFIG_NET_CLS_FW=m -CONFIG_NET_CLS_U32=m -CONFIG_NET_CLS_RSVP=m -CONFIG_NET_CLS_RSVP6=m -CONFIG_HAMRADIO=y -CONFIG_AX25=m -CONFIG_NETROM=m -CONFIG_ROSE=m -CONFIG_MKISS=m -CONFIG_6PACK=m -CONFIG_BPQETHER=m -CONFIG_CONNECTOR=m CONFIG_PARPORT=m CONFIG_PARPORT_PC=m CONFIG_PARPORT_1284=y +CONFIG_DEVTMPFS=y CONFIG_BLK_DEV_FD=m -CONFIG_PARIDE=m -CONFIG_PARIDE_PD=m -CONFIG_PARIDE_PCD=m -CONFIG_PARIDE_PF=m -CONFIG_PARIDE_PT=m -CONFIG_PARIDE_PG=m -CONFIG_PARIDE_ATEN=m -CONFIG_PARIDE_BPCK=m -CONFIG_PARIDE_BPCK6=m -CONFIG_PARIDE_COMM=m -CONFIG_PARIDE_DSTR=m -CONFIG_PARIDE_FIT2=m -CONFIG_PARIDE_FIT3=m -CONFIG_PARIDE_EPAT=m -CONFIG_PARIDE_EPIA=m -CONFIG_PARIDE_FRIQ=m -CONFIG_PARIDE_FRPW=m -CONFIG_PARIDE_KBIC=m -CONFIG_PARIDE_KTTI=m -CONFIG_PARIDE_ON20=m -CONFIG_PARIDE_ON26=m CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m @@ -194,26 +45,12 @@ CONFIG_BLK_DEV_SR=m CONFIG_BLK_DEV_SR_VENDOR=y CONFIG_SCSI_CONSTANTS=y CONFIG_SCSI_SCAN_ASYNC=y -CONFIG_SCSI_FC_ATTRS=y -CONFIG_SCSI_SAS_ATTRS=m CONFIG_ISCSI_TCP=m CONFIG_SCSI_PPA=m CONFIG_SCSI_IMM=m CONFIG_JAZZ_ESP=y -CONFIG_MD=y -CONFIG_BLK_DEV_MD=m -CONFIG_MD_LINEAR=m -CONFIG_MD_RAID0=m -CONFIG_MD_RAID1=m -CONFIG_MD_RAID10=m -CONFIG_MD_RAID456=m -CONFIG_MD_MULTIPATH=m -CONFIG_MD_FAULTY=m -CONFIG_BLK_DEV_DM=m -CONFIG_DM_SNAPSHOT=m -CONFIG_DM_MIRROR=m -CONFIG_DM_ZERO=m -CONFIG_DM_MULTIPATH=m +CONFIG_ATA=y +CONFIG_PATA_LEGACY=y CONFIG_NETDEVICES=y CONFIG_BONDING=m CONFIG_DUMMY=m @@ -221,28 +58,18 @@ CONFIG_EQUALIZER=m CONFIG_TUN=m CONFIG_MIPS_JAZZ_SONIC=y CONFIG_NE2000=m -CONFIG_PHYLIB=m -CONFIG_CICADA_PHY=m -CONFIG_DAVICOM_PHY=m -CONFIG_LXT_PHY=m -CONFIG_MARVELL_PHY=m -CONFIG_QSEMI_PHY=m -CONFIG_SMSC_PHY=m -CONFIG_VITESSE_PHY=m -CONFIG_PLIP=m -CONFIG_INPUT_FF_MEMLESS=m CONFIG_SERIO_PARKBD=m CONFIG_SERIO_RAW=m CONFIG_VT_HW_CONSOLE_BINDING=y -CONFIG_SERIAL_8250=m +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_8250_EXTENDED=y CONFIG_SERIAL_8250_SHARE_IRQ=y CONFIG_SERIAL_8250_DETECT_IRQ=y CONFIG_SERIAL_8250_RSA=y -CONFIG_PRINTER=m -CONFIG_PPDEV=m -# CONFIG_HW_RANDOM is not set -CONFIG_W1=m +CONFIG_FB=y +CONFIG_FB_G364=y +CONFIG_FRAMEBUFFER_CONSOLE=y # CONFIG_HWMON is not set CONFIG_EXT2_FS=m CONFIG_EXT3_FS=y @@ -263,78 +90,8 @@ CONFIG_VFAT_FS=m CONFIG_NTFS_FS=m CONFIG_PROC_KCORE=y CONFIG_TMPFS=y -CONFIG_ADFS_FS=m -CONFIG_AFFS_FS=m -CONFIG_HFS_FS=m -CONFIG_BEFS_FS=m -CONFIG_BFS_FS=m -CONFIG_EFS_FS=m -CONFIG_CRAMFS=m -CONFIG_VXFS_FS=m -CONFIG_MINIX_FS=m -CONFIG_HPFS_FS=m -CONFIG_QNX4FS_FS=m -CONFIG_ROMFS_FS=m -CONFIG_SYSV_FS=m CONFIG_UFS_FS=m CONFIG_NFS_FS=m CONFIG_NFSD=m CONFIG_NFSD_V3=y CONFIG_CIFS=m -CONFIG_CODA_FS=m -CONFIG_AFS_FS=m -CONFIG_NLS_CODEPAGE_437=m -CONFIG_NLS_CODEPAGE_737=m -CONFIG_NLS_CODEPAGE_775=m -CONFIG_NLS_CODEPAGE_850=m -CONFIG_NLS_CODEPAGE_852=m -CONFIG_NLS_CODEPAGE_855=m -CONFIG_NLS_CODEPAGE_857=m -CONFIG_NLS_CODEPAGE_860=m -CONFIG_NLS_CODEPAGE_861=m -CONFIG_NLS_CODEPAGE_862=m -CONFIG_NLS_CODEPAGE_863=m -CONFIG_NLS_CODEPAGE_864=m -CONFIG_NLS_CODEPAGE_865=m -CONFIG_NLS_CODEPAGE_866=m -CONFIG_NLS_CODEPAGE_869=m -CONFIG_NLS_CODEPAGE_936=m -CONFIG_NLS_CODEPAGE_950=m -CONFIG_NLS_CODEPAGE_932=m -CONFIG_NLS_CODEPAGE_949=m -CONFIG_NLS_CODEPAGE_874=m -CONFIG_NLS_ISO8859_8=m -CONFIG_NLS_CODEPAGE_1250=m -CONFIG_NLS_CODEPAGE_1251=m -CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=m -CONFIG_NLS_ISO8859_2=m -CONFIG_NLS_ISO8859_3=m -CONFIG_NLS_ISO8859_4=m -CONFIG_NLS_ISO8859_5=m -CONFIG_NLS_ISO8859_6=m -CONFIG_NLS_ISO8859_7=m -CONFIG_NLS_ISO8859_9=m -CONFIG_NLS_ISO8859_13=m -CONFIG_NLS_ISO8859_14=m -CONFIG_NLS_ISO8859_15=m -CONFIG_NLS_KOI8_R=m -CONFIG_NLS_KOI8_U=m -CONFIG_NLS_UTF8=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_ANUBIS=m -CONFIG_CRYPTO_BLOWFISH=m -CONFIG_CRYPTO_CAMELLIA=m -CONFIG_CRYPTO_CAST6=m -CONFIG_CRYPTO_FCRYPT=m -CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_TEA=m -CONFIG_CRYPTO_TWOFISH=m -CONFIG_CRC_CCITT=m diff --git a/arch/mips/configs/lasat_defconfig b/arch/mips/configs/lasat_defconfig index c66ca3785655..00cf461db971 100644 --- a/arch/mips/configs/lasat_defconfig +++ b/arch/mips/configs/lasat_defconfig @@ -16,8 +16,6 @@ CONFIG_HZ_1000=y # CONFIG_SECCOMP is not set CONFIG_PCI=y # CONFIG_BLK_DEV_BSG is not set -# CONFIG_IOSCHED_DEADLINE is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/mips/configs/lemote2f_defconfig b/arch/mips/configs/lemote2f_defconfig index f9f93427c9bd..8254d7d1396f 100644 --- a/arch/mips/configs/lemote2f_defconfig +++ b/arch/mips/configs/lemote2f_defconfig @@ -26,7 +26,7 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_BLK_DEV_INTEGRITY=y -CONFIG_IOSCHED_DEADLINE=m +CONFIG_MQ_IOSCHED_DEADLINE=m CONFIG_BINFMT_MISC=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig index 360c6b2d397a..51675f5000d6 100644 --- a/arch/mips/configs/loongson3_defconfig +++ b/arch/mips/configs/loongson3_defconfig @@ -38,8 +38,9 @@ CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_PARTITION_ADVANCED=y -CONFIG_IOSCHED_DEADLINE=m -CONFIG_CFQ_GROUP_IOSCHED=y +CONFIG_MQ_IOSCHED_DEADLINE=m +CONFIG_IOSCHED_BFQ=y +CONFIG_BFQ_GROUP_IOSCHED=y CONFIG_BINFMT_MISC=m CONFIG_KSM=y CONFIG_NET=y diff --git a/arch/mips/configs/msp71xx_defconfig b/arch/mips/configs/msp71xx_defconfig index 0fdc03fda12e..6ad1a2381226 100644 --- a/arch/mips/configs/msp71xx_defconfig +++ b/arch/mips/configs/msp71xx_defconfig @@ -14,8 +14,6 @@ CONFIG_PCI=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y -# CONFIG_IOSCHED_DEADLINE is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_UNIX=y CONFIG_XFRM_USER=y diff --git a/arch/mips/configs/pnx8335_stb225_defconfig b/arch/mips/configs/pnx8335_stb225_defconfig index 738ba3b1374b..d06db6b87959 100644 --- a/arch/mips/configs/pnx8335_stb225_defconfig +++ b/arch/mips/configs/pnx8335_stb225_defconfig @@ -14,8 +14,6 @@ CONFIG_HZ_128=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set -# CONFIG_IOSCHED_DEADLINE is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/mips/configs/rb532_defconfig b/arch/mips/configs/rb532_defconfig index 5b947183852b..252d472387aa 100644 --- a/arch/mips/configs/rb532_defconfig +++ b/arch/mips/configs/rb532_defconfig @@ -23,7 +23,6 @@ CONFIG_MODULE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y CONFIG_MAC_PARTITION=y CONFIG_BSD_DISKLABEL=y -# CONFIG_IOSCHED_CFQ is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/mips/configs/rt305x_defconfig b/arch/mips/configs/rt305x_defconfig index 110948bc6b39..8c2ead53007a 100644 --- a/arch/mips/configs/rt305x_defconfig +++ b/arch/mips/configs/rt305x_defconfig @@ -21,7 +21,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y -# CONFIG_IOSCHED_CFQ is not set # CONFIG_COREDUMP is not set # CONFIG_COMPACTION is not set CONFIG_NET=y diff --git a/arch/mips/configs/xway_defconfig b/arch/mips/configs/xway_defconfig index 49b5ea6eff62..9abbc0debc2a 100644 --- a/arch/mips/configs/xway_defconfig +++ b/arch/mips/configs/xway_defconfig @@ -23,7 +23,6 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y -# CONFIG_IOSCHED_CFQ is not set # CONFIG_COREDUMP is not set # CONFIG_COMPACTION is not set CONFIG_NET=y diff --git a/arch/mips/dec/setup.c b/arch/mips/dec/setup.c index 61a0bf13e308..d4e868b828e5 100644 --- a/arch/mips/dec/setup.c +++ b/arch/mips/dec/setup.c @@ -103,28 +103,8 @@ int_ptr asic_mask_nr_tbl[DEC_MAX_ASIC_INTS][2] = { int cpu_fpu_mask = DEC_CPU_IRQ_MASK(DEC_CPU_INR_FPU); int *fpu_kstat_irq; -static struct irqaction ioirq = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; -static struct irqaction fpuirq = { - .handler = no_action, - .name = "fpu", - .flags = IRQF_NO_THREAD, -}; - -static struct irqaction busirq = { - .name = "bus error", - .flags = IRQF_NO_THREAD, -}; - -static struct irqaction haltirq = { - .handler = dec_intr_halt, - .name = "halt", - .flags = IRQF_NO_THREAD, -}; - +static irq_handler_t busirq_handler; +static unsigned int busirq_flags = IRQF_NO_THREAD; /* * Bus error (DBE/IBE exceptions and bus interrupts) handling setup. @@ -134,21 +114,21 @@ static void __init dec_be_init(void) switch (mips_machtype) { case MACH_DS23100: /* DS2100/DS3100 Pmin/Pmax */ board_be_handler = dec_kn01_be_handler; - busirq.handler = dec_kn01_be_interrupt; - busirq.flags |= IRQF_SHARED; + busirq_handler = dec_kn01_be_interrupt; + busirq_flags |= IRQF_SHARED; dec_kn01_be_init(); break; case MACH_DS5000_1XX: /* DS5000/1xx 3min */ case MACH_DS5000_XX: /* DS5000/xx Maxine */ board_be_handler = dec_kn02xa_be_handler; - busirq.handler = dec_kn02xa_be_interrupt; + busirq_handler = dec_kn02xa_be_interrupt; dec_kn02xa_be_init(); break; case MACH_DS5000_200: /* DS5000/200 3max */ case MACH_DS5000_2X0: /* DS5000/240 3max+ */ case MACH_DS5900: /* DS5900 bigmax */ board_be_handler = dec_ecc_be_handler; - busirq.handler = dec_ecc_be_interrupt; + busirq_handler = dec_ecc_be_interrupt; dec_ecc_be_init(); break; } @@ -764,20 +744,29 @@ void __init arch_init_irq(void) int irq_fpu; irq_fpu = dec_interrupt[DEC_IRQ_FPU]; - setup_irq(irq_fpu, &fpuirq); + if (request_irq(irq_fpu, no_action, IRQF_NO_THREAD, "fpu", + NULL)) + pr_err("Failed to register fpu interrupt\n"); desc_fpu = irq_to_desc(irq_fpu); fpu_kstat_irq = this_cpu_ptr(desc_fpu->kstat_irqs); } - if (dec_interrupt[DEC_IRQ_CASCADE] >= 0) - setup_irq(dec_interrupt[DEC_IRQ_CASCADE], &ioirq); - + if (dec_interrupt[DEC_IRQ_CASCADE] >= 0) { + if (request_irq(dec_interrupt[DEC_IRQ_CASCADE], no_action, + IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to register cascade interrupt\n"); + } /* Register the bus error interrupt. */ - if (dec_interrupt[DEC_IRQ_BUS] >= 0 && busirq.handler) - setup_irq(dec_interrupt[DEC_IRQ_BUS], &busirq); - + if (dec_interrupt[DEC_IRQ_BUS] >= 0 && busirq_handler) { + if (request_irq(dec_interrupt[DEC_IRQ_BUS], busirq_handler, + busirq_flags, "bus error", busirq_handler)) + pr_err("Failed to register bus error interrupt\n"); + } /* Register the HALT interrupt. */ - if (dec_interrupt[DEC_IRQ_HALT] >= 0) - setup_irq(dec_interrupt[DEC_IRQ_HALT], &haltirq); + if (dec_interrupt[DEC_IRQ_HALT] >= 0) { + if (request_irq(dec_interrupt[DEC_IRQ_HALT], dec_intr_halt, + IRQF_NO_THREAD, "halt", NULL)) + pr_err("Failed to register halt interrupt\n"); + } } asmlinkage unsigned int dec_irq_dispatch(unsigned int irq) diff --git a/arch/mips/emma/markeins/irq.c b/arch/mips/emma/markeins/irq.c index 09427a49e7ae..4aebf559be2e 100644 --- a/arch/mips/emma/markeins/irq.c +++ b/arch/mips/emma/markeins/irq.c @@ -153,14 +153,6 @@ void emma2rh_gpio_irq_init(void) handle_edge_irq, "edge"); } -static struct irqaction irq_cascade = { - .handler = no_action, - .flags = IRQF_NO_THREAD, - .name = "cascade", - .dev_id = NULL, - .next = NULL, -}; - /* * the first level int-handler will jump here if it is a emma2rh irq */ @@ -236,6 +228,7 @@ void emma2rh_irq_dispatch(void) void __init arch_init_irq(void) { u32 reg; + int irq; /* by default, interrupts are disabled. */ emma2rh_out32(EMMA2RH_BHIF_INT_EN_0, 0); @@ -272,9 +265,15 @@ void __init arch_init_irq(void) mips_cpu_irq_init(); /* setup cascade interrupts */ - setup_irq(EMMA2RH_IRQ_BASE + EMMA2RH_SW_CASCADE, &irq_cascade); - setup_irq(EMMA2RH_IRQ_BASE + EMMA2RH_GPIO_CASCADE, &irq_cascade); - setup_irq(MIPS_CPU_IRQ_BASE + 2, &irq_cascade); + irq = EMMA2RH_IRQ_BASE + EMMA2RH_SW_CASCADE; + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", irq); + irq = EMMA2RH_IRQ_BASE + EMMA2RH_GPIO_CASCADE; + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", irq); + irq = MIPS_CPU_IRQ_BASE + 2; + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", irq); } asmlinkage void plat_irq_dispatch(void) diff --git a/arch/mips/fw/lib/cmdline.c b/arch/mips/fw/lib/cmdline.c index 6ecda64ad184..f24cbb4a39b5 100644 --- a/arch/mips/fw/lib/cmdline.c +++ b/arch/mips/fw/lib/cmdline.c @@ -16,6 +16,7 @@ int fw_argc; int *_fw_argv; int *_fw_envp; +#ifndef CONFIG_HAVE_PLAT_FW_INIT_CMDLINE void __init fw_init_cmdline(void) { int i; @@ -41,6 +42,7 @@ void __init fw_init_cmdline(void) strlcat(arcs_cmdline, " ", COMMAND_LINE_SIZE); } } +#endif char * __init fw_getcmdline(void) { diff --git a/arch/mips/generic/init.c b/arch/mips/generic/init.c index 1de215b283d6..805d0135a9f4 100644 --- a/arch/mips/generic/init.c +++ b/arch/mips/generic/init.c @@ -5,10 +5,10 @@ */ #include <linux/clk.h> -#include <linux/clk-provider.h> #include <linux/clocksource.h> #include <linux/init.h> #include <linux/irqchip.h> +#include <linux/of_clk.h> #include <linux/of_fdt.h> #include <asm/bootinfo.h> diff --git a/arch/mips/include/asm/clocksource.h b/arch/mips/include/asm/clocksource.h index cab9ae9f1e14..2f1ebbea3d72 100644 --- a/arch/mips/include/asm/clocksource.h +++ b/arch/mips/include/asm/clocksource.h @@ -3,23 +3,9 @@ * Copyright (C) 2015 Imagination Technologies * Author: Alex Smith <alex.smith@imgtec.com> */ - #ifndef __ASM_CLOCKSOURCE_H #define __ASM_CLOCKSOURCE_H -#include <linux/types.h> - -/* VDSO clocksources. */ -#define VDSO_CLOCK_NONE 0 /* No suitable clocksource. */ -#define VDSO_CLOCK_R4K 1 /* Use the coprocessor 0 count. */ -#define VDSO_CLOCK_GIC 2 /* Use the GIC. */ - -/** - * struct arch_clocksource_data - Architecture-specific clocksource information. - * @vdso_clock_mode: Method the VDSO should use to access the clocksource. - */ -struct arch_clocksource_data { - u8 vdso_clock_mode; -}; +#include <asm/vdso/clocksource.h> #endif /* __ASM_CLOCKSOURCE_H */ diff --git a/arch/mips/include/asm/dmi.h b/arch/mips/include/asm/dmi.h new file mode 100644 index 000000000000..27415a288adf --- /dev/null +++ b/arch/mips/include/asm/dmi.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_DMI_H +#define _ASM_DMI_H + +#include <linux/io.h> +#include <linux/memblock.h> + +#define dmi_early_remap(x, l) ioremap_cache(x, l) +#define dmi_early_unmap(x, l) iounmap(x) +#define dmi_remap(x, l) ioremap_cache(x, l) +#define dmi_unmap(x) iounmap(x) + +/* MIPS initialize DMI scan before SLAB is ready, so we use memblock here */ +#define dmi_alloc(l) memblock_alloc_low(l, PAGE_SIZE) + +#if defined(CONFIG_MACH_LOONGSON64) +#define SMBIOS_ENTRY_POINT_SCAN_START 0xFFFE000 +#endif + +#endif /* _ASM_DMI_H */ diff --git a/arch/mips/include/asm/elf.h b/arch/mips/include/asm/elf.h index f8f44b1a6cbb..5aa29ced6970 100644 --- a/arch/mips/include/asm/elf.h +++ b/arch/mips/include/asm/elf.h @@ -445,6 +445,9 @@ extern unsigned int elf_hwcap; #define ELF_PLATFORM __elf_platform extern const char *__elf_platform; +#define ELF_BASE_PLATFORM __elf_base_platform +extern const char *__elf_base_platform; + /* * See comments in asm-alpha/elf.h, this is the same thing * on the MIPS. diff --git a/arch/mips/include/asm/futex.h b/arch/mips/include/asm/futex.h index 110220705e97..2bf8f6014579 100644 --- a/arch/mips/include/asm/futex.h +++ b/arch/mips/include/asm/futex.h @@ -89,7 +89,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -116,8 +117,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/mips/include/asm/i8259.h b/arch/mips/include/asm/i8259.h index 97a5e41ed1ab..a54b9649de22 100644 --- a/arch/mips/include/asm/i8259.h +++ b/arch/mips/include/asm/i8259.h @@ -36,6 +36,7 @@ extern raw_spinlock_t i8259A_lock; extern void make_8259A_irq(unsigned int irq); extern void init_i8259_irqs(void); +extern struct irq_domain *__init_i8259_irqs(struct device_node *node); /** * i8159_set_poll() - Override the i8259 polling function diff --git a/arch/mips/include/asm/mach-ar7/irq.h b/arch/mips/include/asm/mach-ar7/irq.h index 7ad10e379e2b..46bb730ea970 100644 --- a/arch/mips/include/asm/mach-ar7/irq.h +++ b/arch/mips/include/asm/mach-ar7/irq.h @@ -11,6 +11,6 @@ #define NR_IRQS 256 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __ASM_AR7_IRQ_H */ diff --git a/arch/mips/include/asm/mach-ath79/irq.h b/arch/mips/include/asm/mach-ath79/irq.h index 2df1abf9e5af..882534be0715 100644 --- a/arch/mips/include/asm/mach-ath79/irq.h +++ b/arch/mips/include/asm/mach-ath79/irq.h @@ -27,6 +27,6 @@ #define ATH79_IP3_IRQ_COUNT 3 #define ATH79_IP3_IRQ(_x) (ATH79_IP3_IRQ_BASE + (_x)) -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __ASM_MACH_ATH79_IRQ_H */ diff --git a/arch/mips/include/asm/mach-au1x00/au1xxx_ide.h b/arch/mips/include/asm/mach-au1x00/au1xxx_ide.h deleted file mode 100644 index bb91b8923a49..000000000000 --- a/arch/mips/include/asm/mach-au1x00/au1xxx_ide.h +++ /dev/null @@ -1,178 +0,0 @@ -/* - * include/asm-mips/mach-au1x00/au1xxx_ide.h version 01.30.00 Aug. 02 2005 - * - * BRIEF MODULE DESCRIPTION - * AMD Alchemy Au1xxx IDE interface routines over the Static Bus - * - * Copyright (c) 2003-2005 AMD, Personal Connectivity Solutions - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any later - * version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - * - * Note: for more information, please refer "AMD Alchemy Au1200/Au1550 IDE - * Interface and Linux Device Driver" Application Note. - */ - -#ifdef CONFIG_BLK_DEV_IDE_AU1XXX_MDMA2_DBDMA -#define DMA_WAIT_TIMEOUT 100 -#define NUM_DESCRIPTORS PRD_ENTRIES -#else /* CONFIG_BLK_DEV_IDE_AU1XXX_PIO_DBDMA */ -#define NUM_DESCRIPTORS 2 -#endif - -#ifndef AU1XXX_ATA_RQSIZE -#define AU1XXX_ATA_RQSIZE 128 -#endif - -/* Disable Burstable-Support for DBDMA */ -#ifndef CONFIG_BLK_DEV_IDE_AU1XXX_BURSTABLE_ON -#define CONFIG_BLK_DEV_IDE_AU1XXX_BURSTABLE_ON 0 -#endif - -typedef struct { - u32 tx_dev_id, rx_dev_id, target_dev_id; - u32 tx_chan, rx_chan; - void *tx_desc_head, *rx_desc_head; - ide_hwif_t *hwif; -#ifdef CONFIG_BLK_DEV_IDE_AU1XXX_MDMA2_DBDMA - ide_drive_t *drive; - struct dbdma_cmd *dma_table_cpu; - dma_addr_t dma_table_dma; -#endif - int irq; - u32 regbase; - int ddma_id; -} _auide_hwif; - -/******************************************************************************/ -/* PIO Mode timing calculation : */ -/* */ -/* Static Bus Spec ATA Spec */ -/* Tcsoe = t1 */ -/* Toecs = t9 */ -/* Twcs = t9 */ -/* Tcsh = t2i | t2 */ -/* Tcsoff = t2i | t2 */ -/* Twp = t2 */ -/* Tcsw = t1 */ -/* Tpm = 0 */ -/* Ta = t1+t2 */ -/******************************************************************************/ - -#define TCSOE_MASK (0x07 << 29) -#define TOECS_MASK (0x07 << 26) -#define TWCS_MASK (0x07 << 28) -#define TCSH_MASK (0x0F << 24) -#define TCSOFF_MASK (0x07 << 20) -#define TWP_MASK (0x3F << 14) -#define TCSW_MASK (0x0F << 10) -#define TPM_MASK (0x0F << 6) -#define TA_MASK (0x3F << 0) -#define TS_MASK (1 << 8) - -/* Timing parameters PIO mode 0 */ -#define SBC_IDE_PIO0_TCSOE (0x04 << 29) -#define SBC_IDE_PIO0_TOECS (0x01 << 26) -#define SBC_IDE_PIO0_TWCS (0x02 << 28) -#define SBC_IDE_PIO0_TCSH (0x08 << 24) -#define SBC_IDE_PIO0_TCSOFF (0x07 << 20) -#define SBC_IDE_PIO0_TWP (0x10 << 14) -#define SBC_IDE_PIO0_TCSW (0x04 << 10) -#define SBC_IDE_PIO0_TPM (0x00 << 6) -#define SBC_IDE_PIO0_TA (0x15 << 0) -/* Timing parameters PIO mode 1 */ -#define SBC_IDE_PIO1_TCSOE (0x03 << 29) -#define SBC_IDE_PIO1_TOECS (0x01 << 26) -#define SBC_IDE_PIO1_TWCS (0x01 << 28) -#define SBC_IDE_PIO1_TCSH (0x06 << 24) -#define SBC_IDE_PIO1_TCSOFF (0x06 << 20) -#define SBC_IDE_PIO1_TWP (0x08 << 14) -#define SBC_IDE_PIO1_TCSW (0x03 << 10) -#define SBC_IDE_PIO1_TPM (0x00 << 6) -#define SBC_IDE_PIO1_TA (0x0B << 0) -/* Timing parameters PIO mode 2 */ -#define SBC_IDE_PIO2_TCSOE (0x05 << 29) -#define SBC_IDE_PIO2_TOECS (0x01 << 26) -#define SBC_IDE_PIO2_TWCS (0x01 << 28) -#define SBC_IDE_PIO2_TCSH (0x07 << 24) -#define SBC_IDE_PIO2_TCSOFF (0x07 << 20) -#define SBC_IDE_PIO2_TWP (0x1F << 14) -#define SBC_IDE_PIO2_TCSW (0x05 << 10) -#define SBC_IDE_PIO2_TPM (0x00 << 6) -#define SBC_IDE_PIO2_TA (0x22 << 0) -/* Timing parameters PIO mode 3 */ -#define SBC_IDE_PIO3_TCSOE (0x05 << 29) -#define SBC_IDE_PIO3_TOECS (0x01 << 26) -#define SBC_IDE_PIO3_TWCS (0x01 << 28) -#define SBC_IDE_PIO3_TCSH (0x0D << 24) -#define SBC_IDE_PIO3_TCSOFF (0x0D << 20) -#define SBC_IDE_PIO3_TWP (0x15 << 14) -#define SBC_IDE_PIO3_TCSW (0x05 << 10) -#define SBC_IDE_PIO3_TPM (0x00 << 6) -#define SBC_IDE_PIO3_TA (0x1A << 0) -/* Timing parameters PIO mode 4 */ -#define SBC_IDE_PIO4_TCSOE (0x04 << 29) -#define SBC_IDE_PIO4_TOECS (0x01 << 26) -#define SBC_IDE_PIO4_TWCS (0x01 << 28) -#define SBC_IDE_PIO4_TCSH (0x04 << 24) -#define SBC_IDE_PIO4_TCSOFF (0x04 << 20) -#define SBC_IDE_PIO4_TWP (0x0D << 14) -#define SBC_IDE_PIO4_TCSW (0x03 << 10) -#define SBC_IDE_PIO4_TPM (0x00 << 6) -#define SBC_IDE_PIO4_TA (0x12 << 0) -/* Timing parameters MDMA mode 0 */ -#define SBC_IDE_MDMA0_TCSOE (0x03 << 29) -#define SBC_IDE_MDMA0_TOECS (0x01 << 26) -#define SBC_IDE_MDMA0_TWCS (0x01 << 28) -#define SBC_IDE_MDMA0_TCSH (0x07 << 24) -#define SBC_IDE_MDMA0_TCSOFF (0x07 << 20) -#define SBC_IDE_MDMA0_TWP (0x0C << 14) -#define SBC_IDE_MDMA0_TCSW (0x03 << 10) -#define SBC_IDE_MDMA0_TPM (0x00 << 6) -#define SBC_IDE_MDMA0_TA (0x0F << 0) -/* Timing parameters MDMA mode 1 */ -#define SBC_IDE_MDMA1_TCSOE (0x05 << 29) -#define SBC_IDE_MDMA1_TOECS (0x01 << 26) -#define SBC_IDE_MDMA1_TWCS (0x01 << 28) -#define SBC_IDE_MDMA1_TCSH (0x05 << 24) -#define SBC_IDE_MDMA1_TCSOFF (0x05 << 20) -#define SBC_IDE_MDMA1_TWP (0x0F << 14) -#define SBC_IDE_MDMA1_TCSW (0x05 << 10) -#define SBC_IDE_MDMA1_TPM (0x00 << 6) -#define SBC_IDE_MDMA1_TA (0x15 << 0) -/* Timing parameters MDMA mode 2 */ -#define SBC_IDE_MDMA2_TCSOE (0x04 << 29) -#define SBC_IDE_MDMA2_TOECS (0x01 << 26) -#define SBC_IDE_MDMA2_TWCS (0x01 << 28) -#define SBC_IDE_MDMA2_TCSH (0x04 << 24) -#define SBC_IDE_MDMA2_TCSOFF (0x04 << 20) -#define SBC_IDE_MDMA2_TWP (0x0D << 14) -#define SBC_IDE_MDMA2_TCSW (0x04 << 10) -#define SBC_IDE_MDMA2_TPM (0x00 << 6) -#define SBC_IDE_MDMA2_TA (0x12 << 0) - -#define SBC_IDE_TIMING(mode) \ - (SBC_IDE_##mode##_TWCS | \ - SBC_IDE_##mode##_TCSH | \ - SBC_IDE_##mode##_TCSOFF | \ - SBC_IDE_##mode##_TWP | \ - SBC_IDE_##mode##_TCSW | \ - SBC_IDE_##mode##_TPM | \ - SBC_IDE_##mode##_TA) diff --git a/arch/mips/include/asm/mach-emma2rh/irq.h b/arch/mips/include/asm/mach-emma2rh/irq.h index 2f7155dade29..d32736736bb3 100644 --- a/arch/mips/include/asm/mach-emma2rh/irq.h +++ b/arch/mips/include/asm/mach-emma2rh/irq.h @@ -10,6 +10,6 @@ #define NR_IRQS 256 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __ASM_MACH_EMMA2RH_IRQ_H */ diff --git a/arch/mips/include/asm/mach-ip27/irq.h b/arch/mips/include/asm/mach-ip27/irq.h index fd91c58aaf7d..f45d7999fd73 100644 --- a/arch/mips/include/asm/mach-ip27/irq.h +++ b/arch/mips/include/asm/mach-ip27/irq.h @@ -12,7 +12,7 @@ #define NR_IRQS 256 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #define IP27_HUB_PEND0_IRQ (MIPS_CPU_IRQ_BASE + 2) #define IP27_HUB_PEND1_IRQ (MIPS_CPU_IRQ_BASE + 3) diff --git a/arch/mips/include/asm/mach-ip30/irq.h b/arch/mips/include/asm/mach-ip30/irq.h index e5c3dd965266..27ba899c95be 100644 --- a/arch/mips/include/asm/mach-ip30/irq.h +++ b/arch/mips/include/asm/mach-ip30/irq.h @@ -76,7 +76,7 @@ extern void __init ip30_install_ipi(void); */ #define IP30_POWER_IRQ HEART_L2_INT_POWER_BTN -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #define IP30_HEART_L0_IRQ (MIPS_CPU_IRQ_BASE + 2) #define IP30_HEART_L1_IRQ (MIPS_CPU_IRQ_BASE + 3) diff --git a/arch/mips/include/asm/mach-lantiq/falcon/irq.h b/arch/mips/include/asm/mach-lantiq/falcon/irq.h index 91d2bc03c9fa..c14312fb0574 100644 --- a/arch/mips/include/asm/mach-lantiq/falcon/irq.h +++ b/arch/mips/include/asm/mach-lantiq/falcon/irq.h @@ -11,6 +11,6 @@ #define NR_IRQS 328 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif diff --git a/arch/mips/include/asm/mach-lantiq/xway/irq.h b/arch/mips/include/asm/mach-lantiq/xway/irq.h index 76ebbf6c50ef..2980e7771f83 100644 --- a/arch/mips/include/asm/mach-lantiq/xway/irq.h +++ b/arch/mips/include/asm/mach-lantiq/xway/irq.h @@ -11,6 +11,6 @@ #define NR_IRQS 256 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif diff --git a/arch/mips/include/asm/mach-lasat/irq.h b/arch/mips/include/asm/mach-lasat/irq.h index d79cbe075084..e8994921779e 100644 --- a/arch/mips/include/asm/mach-lasat/irq.h +++ b/arch/mips/include/asm/mach-lasat/irq.h @@ -9,6 +9,6 @@ #define NR_IRQS 24 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* _ASM_MACH_LASAT_IRQ_H */ diff --git a/arch/mips/include/asm/mach-loongson64/boot_param.h b/arch/mips/include/asm/mach-loongson64/boot_param.h index 8c286bedff3e..2ed483e32d8c 100644 --- a/arch/mips/include/asm/mach-loongson64/boot_param.h +++ b/arch/mips/include/asm/mach-loongson64/boot_param.h @@ -2,6 +2,8 @@ #ifndef __ASM_MACH_LOONGSON64_BOOT_PARAM_H_ #define __ASM_MACH_LOONGSON64_BOOT_PARAM_H_ +#include <linux/types.h> + #define SYSTEM_RAM_LOW 1 #define SYSTEM_RAM_HIGH 2 #define SYSTEM_RAM_RESERVED 3 diff --git a/arch/mips/include/asm/mach-loongson64/builtin_dtbs.h b/arch/mips/include/asm/mach-loongson64/builtin_dtbs.h new file mode 100644 index 000000000000..853c6d80887b --- /dev/null +++ b/arch/mips/include/asm/mach-loongson64/builtin_dtbs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019 Jiaxun Yang <jiaxun.yang@flygoat.com> + * + * Built-in Generic dtbs for MACH_LOONGSON64 + */ + +#ifndef __ASM_MACH_LOONGSON64_BUILTIN_DTBS_H_ +#define __ASM_MACH_LOONGSON64_BUILTIN_DTBS_H_ + +extern u32 __dtb_loongson3_4core_rs780e_begin[]; +extern u32 __dtb_loongson3_8core_rs780e_begin[]; +#endif diff --git a/arch/mips/include/asm/mach-loongson64/irq.h b/arch/mips/include/asm/mach-loongson64/irq.h index 73a89913dc38..d41dc4a76e6d 100644 --- a/arch/mips/include/asm/mach-loongson64/irq.h +++ b/arch/mips/include/asm/mach-loongson64/irq.h @@ -7,34 +7,6 @@ /* cpu core interrupt numbers */ #define MIPS_CPU_IRQ_BASE 56 -#define LOONGSON_UART_IRQ (MIPS_CPU_IRQ_BASE + 2) /* UART */ -#define LOONGSON_BRIDGE_IRQ (MIPS_CPU_IRQ_BASE + 3) /* CASCADE */ -#define LOONGSON_TIMER_IRQ (MIPS_CPU_IRQ_BASE + 7) /* CPU Timer */ +#include <asm/mach-generic/irq.h> -#define LOONGSON_HT1_CFG_BASE loongson_sysconf.ht_control_base -#define LOONGSON_HT1_INT_VECTOR_BASE (LOONGSON_HT1_CFG_BASE + 0x80) -#define LOONGSON_HT1_INT_EN_BASE (LOONGSON_HT1_CFG_BASE + 0xa0) -#define LOONGSON_HT1_INT_VECTOR(n) \ - LOONGSON3_REG32(LOONGSON_HT1_INT_VECTOR_BASE, 4 * (n)) -#define LOONGSON_HT1_INTN_EN(n) \ - LOONGSON3_REG32(LOONGSON_HT1_INT_EN_BASE, 4 * (n)) - -#define LOONGSON_INT_ROUTER_OFFSET 0x1400 -#define LOONGSON_INT_ROUTER_INTEN \ - LOONGSON3_REG32(LOONGSON3_REG_BASE, LOONGSON_INT_ROUTER_OFFSET + 0x24) -#define LOONGSON_INT_ROUTER_INTENSET \ - LOONGSON3_REG32(LOONGSON3_REG_BASE, LOONGSON_INT_ROUTER_OFFSET + 0x28) -#define LOONGSON_INT_ROUTER_INTENCLR \ - LOONGSON3_REG32(LOONGSON3_REG_BASE, LOONGSON_INT_ROUTER_OFFSET + 0x2c) -#define LOONGSON_INT_ROUTER_ENTRY(n) \ - LOONGSON3_REG8(LOONGSON3_REG_BASE, LOONGSON_INT_ROUTER_OFFSET + n) -#define LOONGSON_INT_ROUTER_LPC LOONGSON_INT_ROUTER_ENTRY(0x0a) -#define LOONGSON_INT_ROUTER_HT1(n) LOONGSON_INT_ROUTER_ENTRY(n + 0x18) - -#define LOONGSON_INT_COREx_INTy(x, y) (1<<(x) | 1<<(y+4)) /* route to int y of core x */ - -extern void fixup_irqs(void); -extern void loongson3_ipi_interrupt(struct pt_regs *regs); - -#include_next <irq.h> #endif /* __ASM_MACH_LOONGSON64_IRQ_H_ */ diff --git a/arch/mips/include/asm/mach-loongson64/loongson.h b/arch/mips/include/asm/mach-loongson64/loongson.h index a8fce112a9b0..fde1b75c45ea 100644 --- a/arch/mips/include/asm/mach-loongson64/loongson.h +++ b/arch/mips/include/asm/mach-loongson64/loongson.h @@ -25,6 +25,7 @@ extern const struct plat_smp_ops loongson3_smp_ops; /* loongson-specific command line, env and memory initialization */ extern void __init prom_init_memory(void); extern void __init prom_init_env(void); +extern void *loongson_fdt_blob; /* irq operation functions */ extern void mach_irq_dispatch(unsigned int pending); diff --git a/arch/mips/include/asm/mach-malta/irq.h b/arch/mips/include/asm/mach-malta/irq.h index af9eeea250ed..e1bd4298b606 100644 --- a/arch/mips/include/asm/mach-malta/irq.h +++ b/arch/mips/include/asm/mach-malta/irq.h @@ -5,6 +5,6 @@ #define NR_IRQS 256 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __ASM_MACH_MIPS_IRQ_H */ diff --git a/arch/mips/include/asm/mach-pic32/irq.h b/arch/mips/include/asm/mach-pic32/irq.h index d2396941ed2a..ddaf999bc650 100644 --- a/arch/mips/include/asm/mach-pic32/irq.h +++ b/arch/mips/include/asm/mach-pic32/irq.h @@ -9,6 +9,6 @@ #define NR_IRQS 256 #define MIPS_CPU_IRQ_BASE 0 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __ASM_MACH_PIC32_IRQ_H */ diff --git a/arch/mips/include/asm/mach-pistachio/irq.h b/arch/mips/include/asm/mach-pistachio/irq.h index 93bc380f9536..74ac016503ad 100644 --- a/arch/mips/include/asm/mach-pistachio/irq.h +++ b/arch/mips/include/asm/mach-pistachio/irq.h @@ -10,6 +10,6 @@ #define NR_IRQS 256 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __ASM_MACH_PISTACHIO_IRQ_H */ diff --git a/arch/mips/include/asm/mach-ralink/irq.h b/arch/mips/include/asm/mach-ralink/irq.h index 86473e3999aa..2262243d19c9 100644 --- a/arch/mips/include/asm/mach-ralink/irq.h +++ b/arch/mips/include/asm/mach-ralink/irq.h @@ -5,6 +5,6 @@ #define GIC_NUM_INTRS 64 #define NR_IRQS 256 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif diff --git a/arch/mips/include/asm/mach-rm/mc146818rtc.h b/arch/mips/include/asm/mach-rm/mc146818rtc.h index 145bce096fe9..a074f4f84f75 100644 --- a/arch/mips/include/asm/mach-rm/mc146818rtc.h +++ b/arch/mips/include/asm/mach-rm/mc146818rtc.h @@ -16,6 +16,6 @@ #define mc146818_decode_year(year) ((year) + 1980) #endif -#include_next <mc146818rtc.h> +#include <asm/mach-generic/mc146818rtc.h> #endif /* __ASM_MACH_RM_MC146818RTC_H */ diff --git a/arch/mips/include/asm/mach-vr41xx/irq.h b/arch/mips/include/asm/mach-vr41xx/irq.h index 3d63afae37a8..4281b2b9344d 100644 --- a/arch/mips/include/asm/mach-vr41xx/irq.h +++ b/arch/mips/include/asm/mach-vr41xx/irq.h @@ -4,6 +4,6 @@ #include <asm/vr41xx/irq.h> /* for MIPS_CPU_IRQ_BASE */ -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __ASM_MACH_VR41XX_IRQ_H */ diff --git a/arch/mips/include/asm/mach-xilfpga/irq.h b/arch/mips/include/asm/mach-xilfpga/irq.h index b8e93fa8adcf..15ad29ec1dee 100644 --- a/arch/mips/include/asm/mach-xilfpga/irq.h +++ b/arch/mips/include/asm/mach-xilfpga/irq.h @@ -9,6 +9,6 @@ #define NR_IRQS 32 -#include_next <irq.h> +#include <asm/mach-generic/irq.h> #endif /* __MIPS_ASM_MACH_XILFPGA_IRQ_H__ */ diff --git a/arch/mips/include/asm/octeon/cvmx-helper-board.h b/arch/mips/include/asm/octeon/cvmx-helper-board.h index d7fdcf0a0088..ce52aafe7a8d 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper-board.h +++ b/arch/mips/include/asm/octeon/cvmx-helper-board.h @@ -93,7 +93,7 @@ extern int cvmx_helper_board_get_mii_address(int ipd_port); * Returns The ports link status. If the link isn't fully resolved, this must * return zero. */ -extern cvmx_helper_link_info_t __cvmx_helper_board_link_get(int ipd_port); +extern union cvmx_helper_link_info __cvmx_helper_board_link_get(int ipd_port); /** * This function is called by cvmx_helper_interface_probe() after it diff --git a/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h b/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h index ac42b5066bd9..3e79a7f898b6 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h +++ b/arch/mips/include/asm/octeon/cvmx-helper-rgmii.h @@ -74,7 +74,7 @@ extern int __cvmx_helper_rgmii_enable(int interface); * * Returns Link state */ -extern cvmx_helper_link_info_t __cvmx_helper_rgmii_link_get(int ipd_port); +extern union cvmx_helper_link_info __cvmx_helper_rgmii_link_get(int ipd_port); /** * Configure an IPD/PKO port for the specified link state. This @@ -88,6 +88,6 @@ extern cvmx_helper_link_info_t __cvmx_helper_rgmii_link_get(int ipd_port); * Returns Zero on success, negative on failure */ extern int __cvmx_helper_rgmii_link_set(int ipd_port, - cvmx_helper_link_info_t link_info); + union cvmx_helper_link_info link_info); #endif diff --git a/arch/mips/include/asm/octeon/cvmx-helper-sgmii.h b/arch/mips/include/asm/octeon/cvmx-helper-sgmii.h index 3a54dea58c0a..8aac90f1871f 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper-sgmii.h +++ b/arch/mips/include/asm/octeon/cvmx-helper-sgmii.h @@ -68,7 +68,7 @@ extern int __cvmx_helper_sgmii_enable(int interface); * * Returns Link state */ -extern cvmx_helper_link_info_t __cvmx_helper_sgmii_link_get(int ipd_port); +extern union cvmx_helper_link_info __cvmx_helper_sgmii_link_get(int ipd_port); /** * Configure an IPD/PKO port for the specified link state. This @@ -82,6 +82,6 @@ extern cvmx_helper_link_info_t __cvmx_helper_sgmii_link_get(int ipd_port); * Returns Zero on success, negative on failure */ extern int __cvmx_helper_sgmii_link_set(int ipd_port, - cvmx_helper_link_info_t link_info); + union cvmx_helper_link_info link_info); #endif diff --git a/arch/mips/include/asm/octeon/cvmx-helper-spi.h b/arch/mips/include/asm/octeon/cvmx-helper-spi.h index d5adf8592773..bc8cab9367b8 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper-spi.h +++ b/arch/mips/include/asm/octeon/cvmx-helper-spi.h @@ -65,7 +65,7 @@ extern int __cvmx_helper_spi_enable(int interface); * * Returns Link state */ -extern cvmx_helper_link_info_t __cvmx_helper_spi_link_get(int ipd_port); +extern union cvmx_helper_link_info __cvmx_helper_spi_link_get(int ipd_port); /** * Configure an IPD/PKO port for the specified link state. This @@ -79,6 +79,6 @@ extern cvmx_helper_link_info_t __cvmx_helper_spi_link_get(int ipd_port); * Returns Zero on success, negative on failure */ extern int __cvmx_helper_spi_link_set(int ipd_port, - cvmx_helper_link_info_t link_info); + union cvmx_helper_link_info link_info); #endif diff --git a/arch/mips/include/asm/octeon/cvmx-helper-util.h b/arch/mips/include/asm/octeon/cvmx-helper-util.h index e9a97e7ee604..97b27a07cfb0 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper-util.h +++ b/arch/mips/include/asm/octeon/cvmx-helper-util.h @@ -123,7 +123,7 @@ static inline int cvmx_helper_get_last_ipd_port(int interface) * * @work: Work queue entry with packet to free */ -static inline void cvmx_helper_free_packet_data(cvmx_wqe_t *work) +static inline void cvmx_helper_free_packet_data(struct cvmx_wqe *work) { uint64_t number_buffers; union cvmx_buf_ptr buffer_ptr; diff --git a/arch/mips/include/asm/octeon/cvmx-helper-xaui.h b/arch/mips/include/asm/octeon/cvmx-helper-xaui.h index 51f45b495680..c18da2eba78e 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper-xaui.h +++ b/arch/mips/include/asm/octeon/cvmx-helper-xaui.h @@ -68,7 +68,7 @@ extern int __cvmx_helper_xaui_enable(int interface); * * Returns Link state */ -extern cvmx_helper_link_info_t __cvmx_helper_xaui_link_get(int ipd_port); +extern union cvmx_helper_link_info __cvmx_helper_xaui_link_get(int ipd_port); /** * Configure an IPD/PKO port for the specified link state. This @@ -82,6 +82,6 @@ extern cvmx_helper_link_info_t __cvmx_helper_xaui_link_get(int ipd_port); * Returns Zero on success, negative on failure */ extern int __cvmx_helper_xaui_link_set(int ipd_port, - cvmx_helper_link_info_t link_info); + union cvmx_helper_link_info link_info); #endif diff --git a/arch/mips/include/asm/octeon/cvmx-helper.h b/arch/mips/include/asm/octeon/cvmx-helper.h index ba0e76f578e0..c6c99e28eefb 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper.h +++ b/arch/mips/include/asm/octeon/cvmx-helper.h @@ -51,7 +51,7 @@ typedef enum { CVMX_HELPER_INTERFACE_MODE_LOOP, } cvmx_helper_interface_mode_t; -typedef union { +union cvmx_helper_link_info { uint64_t u64; struct { uint64_t reserved_20_63:44; @@ -59,7 +59,7 @@ typedef union { uint64_t full_duplex:1; /**< 1 if the link is full duplex */ uint64_t speed:18; /**< Speed of the link in Mbps */ } s; -} cvmx_helper_link_info_t; +}; #include <asm/octeon/cvmx-helper-errata.h> #include <asm/octeon/cvmx-helper-loop.h> @@ -145,7 +145,7 @@ extern cvmx_helper_interface_mode_t cvmx_helper_interface_get_mode(int * * Returns Link state */ -extern cvmx_helper_link_info_t cvmx_helper_link_get(int ipd_port); +extern union cvmx_helper_link_info cvmx_helper_link_get(int ipd_port); /** * Configure an IPD/PKO port for the specified link state. This @@ -159,7 +159,7 @@ extern cvmx_helper_link_info_t cvmx_helper_link_get(int ipd_port); * Returns Zero on success, negative on failure */ extern int cvmx_helper_link_set(int ipd_port, - cvmx_helper_link_info_t link_info); + union cvmx_helper_link_info link_info); /** * This function probes an interface to determine the actual diff --git a/arch/mips/include/asm/octeon/cvmx-pko.h b/arch/mips/include/asm/octeon/cvmx-pko.h index 20eb9c46a75a..5b0b982a36f7 100644 --- a/arch/mips/include/asm/octeon/cvmx-pko.h +++ b/arch/mips/include/asm/octeon/cvmx-pko.h @@ -169,7 +169,7 @@ typedef union { /** * Structure of the first packet output command word. */ -typedef union { +union cvmx_pko_command_word0 { uint64_t u64; struct { #ifdef __BIG_ENDIAN_BITFIELD @@ -261,7 +261,7 @@ typedef union { uint64_t size1:2; #endif } s; -} cvmx_pko_command_word0_t; +}; /* CSR typedefs have been moved to cvmx-csr-*.h */ @@ -394,7 +394,7 @@ static inline void cvmx_pko_send_packet_prepare(uint64_t port, uint64_t queue, CVMX_TAG_SW_BITS_INTERNAL << CVMX_TAG_SW_SHIFT | CVMX_TAG_SUBGROUP_PKO << CVMX_TAG_SUBGROUP_SHIFT | (CVMX_TAG_SUBGROUP_MASK & queue); - cvmx_pow_tag_sw_full((cvmx_wqe_t *) cvmx_phys_to_ptr(0x80), tag, + cvmx_pow_tag_sw_full((struct cvmx_wqe *) cvmx_phys_to_ptr(0x80), tag, CVMX_POW_TAG_TYPE_ATOMIC, 0); } } @@ -419,7 +419,7 @@ static inline void cvmx_pko_send_packet_prepare(uint64_t port, uint64_t queue, static inline cvmx_pko_status_t cvmx_pko_send_packet_finish( uint64_t port, uint64_t queue, - cvmx_pko_command_word0_t pko_command, + union cvmx_pko_command_word0 pko_command, union cvmx_buf_ptr packet, cvmx_pko_lock_t use_locking) { @@ -462,7 +462,7 @@ static inline cvmx_pko_status_t cvmx_pko_send_packet_finish( static inline cvmx_pko_status_t cvmx_pko_send_packet_finish3( uint64_t port, uint64_t queue, - cvmx_pko_command_word0_t pko_command, + union cvmx_pko_command_word0 pko_command, union cvmx_buf_ptr packet, uint64_t addr, cvmx_pko_lock_t use_locking) diff --git a/arch/mips/include/asm/octeon/cvmx-pow.h b/arch/mips/include/asm/octeon/cvmx-pow.h index 410bb70e5aac..ba366f4f3c3d 100644 --- a/arch/mips/include/asm/octeon/cvmx-pow.h +++ b/arch/mips/include/asm/octeon/cvmx-pow.h @@ -1283,7 +1283,7 @@ static inline cvmx_pow_tag_req_t cvmx_pow_get_current_tag(void) * * Returns WQE pointer */ -static inline cvmx_wqe_t *cvmx_pow_get_current_wqp(void) +static inline struct cvmx_wqe *cvmx_pow_get_current_wqp(void) { cvmx_pow_load_addr_t load_addr; cvmx_pow_tag_load_resp_t load_resp; @@ -1296,7 +1296,7 @@ static inline cvmx_wqe_t *cvmx_pow_get_current_wqp(void) load_addr.sstatus.get_cur = 1; load_addr.sstatus.get_wqp = 1; load_resp.u64 = cvmx_read_csr(load_addr.u64); - return (cvmx_wqe_t *) cvmx_phys_to_ptr(load_resp.s_sstatus4.wqp); + return (struct cvmx_wqe *) cvmx_phys_to_ptr(load_resp.s_sstatus4.wqp); } #ifndef CVMX_MF_CHORD @@ -1348,7 +1348,7 @@ static inline void cvmx_pow_tag_sw_wait(void) * Returns Returns the WQE pointer from POW. Returns NULL if no work * was available. */ -static inline cvmx_wqe_t *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t +static inline struct cvmx_wqe *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t wait) { cvmx_pow_load_addr_t ptr; @@ -1368,7 +1368,7 @@ static inline cvmx_wqe_t *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t if (result.s_work.no_work) return NULL; else - return (cvmx_wqe_t *) cvmx_phys_to_ptr(result.s_work.addr); + return (struct cvmx_wqe *) cvmx_phys_to_ptr(result.s_work.addr); } /** @@ -1382,7 +1382,7 @@ static inline cvmx_wqe_t *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t * Returns Returns the WQE pointer from POW. Returns NULL if no work * was available. */ -static inline cvmx_wqe_t *cvmx_pow_work_request_sync(cvmx_pow_wait_t wait) +static inline struct cvmx_wqe *cvmx_pow_work_request_sync(cvmx_pow_wait_t wait) { if (CVMX_ENABLE_POW_CHECKS) __cvmx_pow_warn_if_pending_switch(__func__); @@ -1485,7 +1485,7 @@ static inline void cvmx_pow_work_request_async(int scr_addr, * Returns Returns the WQE from the scratch register, or NULL if no * work was available. */ -static inline cvmx_wqe_t *cvmx_pow_work_response_async(int scr_addr) +static inline struct cvmx_wqe *cvmx_pow_work_response_async(int scr_addr) { cvmx_pow_tag_load_resp_t result; @@ -1495,7 +1495,7 @@ static inline cvmx_wqe_t *cvmx_pow_work_response_async(int scr_addr) if (result.s_work.no_work) return NULL; else - return (cvmx_wqe_t *) cvmx_phys_to_ptr(result.s_work.addr); + return (struct cvmx_wqe *) cvmx_phys_to_ptr(result.s_work.addr); } /** @@ -1508,7 +1508,7 @@ static inline cvmx_wqe_t *cvmx_pow_work_response_async(int scr_addr) * Returns 0 if pointer is valid * 1 if invalid (no work was returned) */ -static inline uint64_t cvmx_pow_work_invalid(cvmx_wqe_t *wqe_ptr) +static inline uint64_t cvmx_pow_work_invalid(struct cvmx_wqe *wqe_ptr) { return wqe_ptr == NULL; } @@ -1638,7 +1638,7 @@ static inline void cvmx_pow_tag_sw(uint32_t tag, * @tag_type: type of tag * @group: group value for the work queue entry. */ -static inline void cvmx_pow_tag_sw_full_nocheck(cvmx_wqe_t *wqp, uint32_t tag, +static inline void cvmx_pow_tag_sw_full_nocheck(struct cvmx_wqe *wqp, uint32_t tag, enum cvmx_pow_tag_type tag_type, uint64_t group) { @@ -1712,7 +1712,7 @@ static inline void cvmx_pow_tag_sw_full_nocheck(cvmx_wqe_t *wqp, uint32_t tag, * @tag_type: type of tag * @group: group value for the work queue entry. */ -static inline void cvmx_pow_tag_sw_full(cvmx_wqe_t *wqp, uint32_t tag, +static inline void cvmx_pow_tag_sw_full(struct cvmx_wqe *wqp, uint32_t tag, enum cvmx_pow_tag_type tag_type, uint64_t group) { @@ -1803,7 +1803,7 @@ static inline void cvmx_pow_tag_sw_null(void) * @qos: Input queue to add to. * @grp: group value for the work queue entry. */ -static inline void cvmx_pow_work_submit(cvmx_wqe_t *wqp, uint32_t tag, +static inline void cvmx_pow_work_submit(struct cvmx_wqe *wqp, uint32_t tag, enum cvmx_pow_tag_type tag_type, uint64_t qos, uint64_t grp) { diff --git a/arch/mips/include/asm/octeon/cvmx-wqe.h b/arch/mips/include/asm/octeon/cvmx-wqe.h index 0d697aa786d4..9cec2299b81b 100644 --- a/arch/mips/include/asm/octeon/cvmx-wqe.h +++ b/arch/mips/include/asm/octeon/cvmx-wqe.h @@ -547,7 +547,7 @@ union cvmx_wqe_word1 { * * must be 8-byte aligned */ -typedef struct { +struct cvmx_wqe { /***************************************************************** * WORD 0 @@ -593,9 +593,9 @@ typedef struct { * */ -} CVMX_CACHE_LINE_ALIGNED cvmx_wqe_t; +} CVMX_CACHE_LINE_ALIGNED; -static inline int cvmx_wqe_get_port(cvmx_wqe_t *work) +static inline int cvmx_wqe_get_port(struct cvmx_wqe *work) { int port; @@ -607,7 +607,7 @@ static inline int cvmx_wqe_get_port(cvmx_wqe_t *work) return port; } -static inline void cvmx_wqe_set_port(cvmx_wqe_t *work, int port) +static inline void cvmx_wqe_set_port(struct cvmx_wqe *work, int port) { if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) work->word2.s_cn68xx.port = port; @@ -615,7 +615,7 @@ static inline void cvmx_wqe_set_port(cvmx_wqe_t *work, int port) work->word1.cn38xx.ipprt = port; } -static inline int cvmx_wqe_get_grp(cvmx_wqe_t *work) +static inline int cvmx_wqe_get_grp(struct cvmx_wqe *work) { int grp; @@ -627,7 +627,7 @@ static inline int cvmx_wqe_get_grp(cvmx_wqe_t *work) return grp; } -static inline void cvmx_wqe_set_grp(cvmx_wqe_t *work, int grp) +static inline void cvmx_wqe_set_grp(struct cvmx_wqe *work, int grp) { if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) work->word1.cn68xx.grp = grp; @@ -635,7 +635,7 @@ static inline void cvmx_wqe_set_grp(cvmx_wqe_t *work, int grp) work->word1.cn38xx.grp = grp; } -static inline int cvmx_wqe_get_qos(cvmx_wqe_t *work) +static inline int cvmx_wqe_get_qos(struct cvmx_wqe *work) { int qos; @@ -647,7 +647,7 @@ static inline int cvmx_wqe_get_qos(cvmx_wqe_t *work) return qos; } -static inline void cvmx_wqe_set_qos(cvmx_wqe_t *work, int qos) +static inline void cvmx_wqe_set_qos(struct cvmx_wqe *work, int qos) { if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) work->word1.cn68xx.qos = qos; diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h index 7619ad319400..856e12f6063d 100644 --- a/arch/mips/include/asm/processor.h +++ b/arch/mips/include/asm/processor.h @@ -22,6 +22,7 @@ #include <asm/dsemul.h> #include <asm/mipsregs.h> #include <asm/prefetch.h> +#include <asm/vdso/processor.h> /* * System setup and hardware flags.. @@ -253,13 +254,13 @@ struct thread_struct { #ifdef CONFIG_MIPS_FP_SUPPORT /* Saved fpu/fpu emulator stuff. */ struct mips_fpu_struct fpu FPU_ALIGN; -#endif /* Assigned branch delay slot 'emulation' frame */ atomic_t bd_emu_frame; /* PC of the branch from a branch delay slot 'emulation' */ unsigned long bd_emu_branch_pc; /* PC to continue from following a branch delay slot 'emulation' */ unsigned long bd_emu_cont_pc; +#endif #ifdef CONFIG_MIPS_MT_FPAFF /* Emulated instruction count */ unsigned long emulated_fp; @@ -302,7 +303,11 @@ struct thread_struct { .fpr = {{{0,},},}, \ .fcr31 = 0, \ .msacsr = 0, \ - }, + }, \ + /* Delay slot emulation */ \ + .bd_emu_frame = ATOMIC_INIT(BD_EMUFRAME_NONE), \ + .bd_emu_branch_pc = 0, \ + .bd_emu_cont_pc = 0, #else # define FPU_INIT #endif @@ -334,10 +339,6 @@ struct thread_struct { * FPU affinity state (null if not FPAFF) \ */ \ FPAFF_INIT \ - /* Delay slot emulation */ \ - .bd_emu_frame = ATOMIC_INIT(BD_EMUFRAME_NONE), \ - .bd_emu_branch_pc = 0, \ - .bd_emu_cont_pc = 0, \ /* \ * Saved DSP stuff \ */ \ @@ -385,21 +386,6 @@ unsigned long get_wchan(struct task_struct *p); #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[29]) #define KSTK_STATUS(tsk) (task_pt_regs(tsk)->cp0_status) -#ifdef CONFIG_CPU_LOONGSON64 -/* - * Loongson-3's SFB (Store-Fill-Buffer) may buffer writes indefinitely when a - * tight read loop is executed, because reads take priority over writes & the - * hardware (incorrectly) doesn't ensure that writes will eventually occur. - * - * Since spin loops of any kind should have a cpu_relax() in them, force an SFB - * flush from cpu_relax() such that any pending writes will become visible as - * expected. - */ -#define cpu_relax() smp_mb() -#else -#define cpu_relax() barrier() -#endif - /* * Return_address is a replacement for __builtin_return_address(count) * which on certain architectures cannot reasonably be implemented in GCC diff --git a/arch/mips/include/asm/sni.h b/arch/mips/include/asm/sni.h index a107201a2e1e..7dfa297ce597 100644 --- a/arch/mips/include/asm/sni.h +++ b/arch/mips/include/asm/sni.h @@ -11,6 +11,8 @@ #ifndef __ASM_SNI_H #define __ASM_SNI_H +#include <linux/irqreturn.h> + extern unsigned int sni_brd_type; #define SNI_BRD_10 2 @@ -239,6 +241,6 @@ static inline int sni_eisa_root_init(void) /* common irq stuff */ extern void (*sni_hwint)(void); -extern struct irqaction sni_isa_irq; +extern irqreturn_t sni_isa_irq_handler(int dummy, void *p); #endif /* __ASM_SNI_H */ diff --git a/arch/mips/include/asm/vdso/clocksource.h b/arch/mips/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..510e1671d898 --- /dev/null +++ b/arch/mips/include/asm/vdso/clocksource.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_R4K, \ + VDSO_CLOCKMODE_GIC + +#endif /* __ASM_VDSOCLOCKSOURCE_H */ diff --git a/arch/mips/include/asm/vdso/gettimeofday.h b/arch/mips/include/asm/vdso/gettimeofday.h index a58687e26c5d..c63ddcaea54c 100644 --- a/arch/mips/include/asm/vdso/gettimeofday.h +++ b/arch/mips/include/asm/vdso/gettimeofday.h @@ -13,19 +13,13 @@ #ifndef __ASSEMBLY__ -#include <linux/compiler.h> -#include <linux/time.h> - #include <asm/vdso/vdso.h> #include <asm/clocksource.h> -#include <asm/io.h> #include <asm/unistd.h> #include <asm/vdso.h> #define VDSO_HAS_CLOCK_GETRES 1 -#define __VDSO_USE_SYSCALL ULLONG_MAX - static __always_inline long gettimeofday_fallback( struct __kernel_old_timeval *_tv, struct timezone *_tz) @@ -175,29 +169,28 @@ static __always_inline u64 read_gic_count(const struct vdso_data *data) static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) { -#ifdef CONFIG_CLKSRC_MIPS_GIC - const struct vdso_data *data = get_vdso_data(); -#endif - u64 cycle_now; - - switch (clock_mode) { #ifdef CONFIG_CSRC_R4K - case VDSO_CLOCK_R4K: - cycle_now = read_r4k_count(); - break; + if (clock_mode == VDSO_CLOCKMODE_R4K) + return read_r4k_count(); #endif #ifdef CONFIG_CLKSRC_MIPS_GIC - case VDSO_CLOCK_GIC: - cycle_now = read_gic_count(data); - break; + if (clock_mode == VDSO_CLOCKMODE_GIC) + return read_gic_count(get_vdso_data()); #endif - default: - cycle_now = __VDSO_USE_SYSCALL; - break; - } + /* + * Core checks mode already. So this raced against a concurrent + * update. Return something. Core will do another round see the + * change and fallback to syscall. + */ + return 0; +} - return cycle_now; +static inline bool mips_vdso_hres_capable(void) +{ + return IS_ENABLED(CONFIG_CSRC_R4K) || + IS_ENABLED(CONFIG_CLKSRC_MIPS_GIC); } +#define __arch_vdso_hres_capable mips_vdso_hres_capable static __always_inline const struct vdso_data *__arch_get_vdso_data(void) { diff --git a/arch/mips/include/asm/vdso/processor.h b/arch/mips/include/asm/vdso/processor.h new file mode 100644 index 000000000000..511c95d735e6 --- /dev/null +++ b/arch/mips/include/asm/vdso/processor.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_CPU_LOONGSON64 +/* + * Loongson-3's SFB (Store-Fill-Buffer) may buffer writes indefinitely when a + * tight read loop is executed, because reads take priority over writes & the + * hardware (incorrectly) doesn't ensure that writes will eventually occur. + * + * Since spin loops of any kind should have a cpu_relax() in them, force an SFB + * flush from cpu_relax() such that any pending writes will become visible as + * expected. + */ +#define cpu_relax() smp_mb() +#else +#define cpu_relax() barrier() +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/mips/include/asm/vdso/vsyscall.h b/arch/mips/include/asm/vdso/vsyscall.h index 00d41b94ba31..47168aaf1eff 100644 --- a/arch/mips/include/asm/vdso/vsyscall.h +++ b/arch/mips/include/asm/vdso/vsyscall.h @@ -19,15 +19,6 @@ struct vdso_data *__mips_get_k_vdso_data(void) } #define __arch_get_k_vdso_data __mips_get_k_vdso_data -static __always_inline -int __mips_get_clock_mode(struct timekeeper *tk) -{ - u32 clock_mode = tk->tkr_mono.clock->archdata.vdso_clock_mode; - - return clock_mode; -} -#define __arch_get_clock_mode __mips_get_clock_mode - /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> diff --git a/arch/mips/jazz/irq.c b/arch/mips/jazz/irq.c index 5d6828b2a750..04b9c4068493 100644 --- a/arch/mips/jazz/irq.c +++ b/arch/mips/jazz/irq.c @@ -125,24 +125,18 @@ static irqreturn_t r4030_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction r4030_timer_irqaction = { - .handler = r4030_timer_interrupt, - .flags = IRQF_TIMER, - .name = "R4030 timer", -}; - void __init plat_time_init(void) { struct clock_event_device *cd = &r4030_clockevent; - struct irqaction *action = &r4030_timer_irqaction; unsigned int cpu = smp_processor_id(); BUG_ON(HZ != 100); cd->cpumask = cpumask_of(cpu); clockevents_register_device(cd); - action->dev_id = cd; - setup_irq(JAZZ_TIMER_IRQ, action); + if (request_irq(JAZZ_TIMER_IRQ, r4030_timer_interrupt, IRQF_TIMER, + "R4030 timer", cd)) + pr_err("Failed to register R4030 timer interrupt\n"); /* * Set clock to 100Hz. diff --git a/arch/mips/jz4740/time.c b/arch/mips/jz4740/time.c index 5476899f0882..605a84a250bf 100644 --- a/arch/mips/jz4740/time.c +++ b/arch/mips/jz4740/time.c @@ -4,8 +4,8 @@ * JZ4740 platform time support */ -#include <linux/clk-provider.h> #include <linux/clocksource.h> +#include <linux/of_clk.h> #include <asm/mach-jz4740/timer.h> diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index b3e8c11a8fa5..d39a2963b451 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -91,16 +91,15 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) } static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); -static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); void sb1480_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_BCM1480_INT_TIMER_0 + cpu; - struct irqaction *action = &per_cpu(sibyte_hpt_irqaction, cpu); struct clock_event_device *cd = &per_cpu(sibyte_hpt_clockevent, cpu); unsigned char *name = per_cpu(sibyte_hpt_name, cpu); + unsigned long flags = IRQF_PERCPU | IRQF_TIMER; BUG_ON(cpu > 3); /* Only have 4 general purpose timers */ @@ -133,11 +132,7 @@ void sb1480_clockevent_init(void) bcm1480_unmask_irq(cpu, irq); - action->handler = sibyte_counter_handler; - action->flags = IRQF_PERCPU | IRQF_TIMER; - action->name = name; - action->dev_id = cd; - irq_set_affinity(irq, cpumask_of(cpu)); - setup_irq(irq, action); + if (request_irq(irq, sibyte_counter_handler, flags, name, cd)) + pr_err("Failed to request irq %d (%s)\n", irq, name); } diff --git a/arch/mips/kernel/cevt-ds1287.c b/arch/mips/kernel/cevt-ds1287.c index 1e1edab4a63f..9a47fbcd4638 100644 --- a/arch/mips/kernel/cevt-ds1287.c +++ b/arch/mips/kernel/cevt-ds1287.c @@ -100,14 +100,9 @@ static irqreturn_t ds1287_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction ds1287_irqaction = { - .handler = ds1287_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "ds1287", -}; - int __init ds1287_clockevent_init(int irq) { + unsigned long flags = IRQF_PERCPU | IRQF_TIMER; struct clock_event_device *cd; cd = &ds1287_clockevent; @@ -122,5 +117,5 @@ int __init ds1287_clockevent_init(int irq) clockevents_register_device(&ds1287_clockevent); - return setup_irq(irq, &ds1287_irqaction); + return request_irq(irq, ds1287_interrupt, flags, "ds1287", NULL); } diff --git a/arch/mips/kernel/cevt-gt641xx.c b/arch/mips/kernel/cevt-gt641xx.c index eb53548d2538..5b132e8c51da 100644 --- a/arch/mips/kernel/cevt-gt641xx.c +++ b/arch/mips/kernel/cevt-gt641xx.c @@ -120,12 +120,6 @@ static irqreturn_t gt641xx_timer0_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction gt641xx_timer0_irqaction = { - .handler = gt641xx_timer0_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "gt641xx_timer0", -}; - static int __init gt641xx_timer0_clockevent_init(void) { struct clock_event_device *cd; @@ -146,6 +140,7 @@ static int __init gt641xx_timer0_clockevent_init(void) clockevents_register_device(>641xx_timer0_clockevent); - return setup_irq(GT641XX_TIMER0_IRQ, >641xx_timer0_irqaction); + return request_irq(GT641XX_TIMER0_IRQ, gt641xx_timer0_interrupt, + IRQF_PERCPU | IRQF_TIMER, "gt641xx_timer0", NULL); } arch_initcall(gt641xx_timer0_clockevent_init); diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index dd6a18bc10ab..17a9cbb8b3df 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -252,6 +252,7 @@ unsigned int __weak get_c0_compare_int(void) int r4k_clockevent_init(void) { + unsigned long flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED; unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; unsigned int irq, min_delta; @@ -291,7 +292,9 @@ int r4k_clockevent_init(void) cp0_timer_irq_installed = 1; - setup_irq(irq, &c0_compare_irqaction); + if (request_irq(irq, c0_compare_interrupt, flags, "timer", + c0_compare_interrupt)) + pr_err("Failed to request irq %d (timer)\n", irq); return 0; } diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index e1a08606c27e..0451273fab9f 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -90,16 +90,15 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) } static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); -static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); void sb1250_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_INT_TIMER_0 + cpu; - struct irqaction *action = &per_cpu(sibyte_hpt_irqaction, cpu); struct clock_event_device *cd = &per_cpu(sibyte_hpt_clockevent, cpu); unsigned char *name = per_cpu(sibyte_hpt_name, cpu); + unsigned long flags = IRQF_PERCPU | IRQF_TIMER; /* Only have 4 general purpose timers, and we use last one as hpt */ BUG_ON(cpu > 2); @@ -133,11 +132,7 @@ void sb1250_clockevent_init(void) sb1250_unmask_irq(cpu, irq); - action->handler = sibyte_counter_handler; - action->flags = IRQF_PERCPU | IRQF_TIMER; - action->name = name; - action->dev_id = cd; - irq_set_affinity(irq, cpumask_of(cpu)); - setup_irq(irq, action); + if (request_irq(irq, sibyte_counter_handler, flags, name, cd)) + pr_err("Failed to request irq %d (%s)\n", irq, name); } diff --git a/arch/mips/kernel/cevt-txx9.c b/arch/mips/kernel/cevt-txx9.c index 7b17c8f5009d..5709469c21ff 100644 --- a/arch/mips/kernel/cevt-txx9.c +++ b/arch/mips/kernel/cevt-txx9.c @@ -174,13 +174,6 @@ static irqreturn_t txx9tmr_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction txx9tmr_irq = { - .handler = txx9tmr_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "txx9tmr", - .dev_id = &txx9_clock_event_device, -}; - void __init txx9_clockevent_init(unsigned long baseaddr, int irq, unsigned int imbusclk) { @@ -202,7 +195,9 @@ void __init txx9_clockevent_init(unsigned long baseaddr, int irq, cd->irq = irq; cd->cpumask = cpumask_of(0), clockevents_register_device(cd); - setup_irq(irq, &txx9tmr_irq); + if (request_irq(irq, txx9tmr_interrupt, IRQF_PERCPU | IRQF_TIMER, + "txx9tmr", &txx9_clock_event_device)) + pr_err("Failed to request irq %d (txx9tmr)\n", irq); printk(KERN_INFO "TXx9: clockevent device at 0x%lx, irq %d\n", baseaddr, irq); } diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 6ab6b03d35ba..f21a2304401f 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -513,6 +513,13 @@ static inline void set_elf_platform(int cpu, const char *plat) __elf_platform = plat; } +static inline void set_elf_base_platform(const char *plat) +{ + if (__elf_base_platform == NULL) { + __elf_base_platform = plat; + } +} + static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) { #ifdef __NEED_VMBITS_PROBE @@ -527,36 +534,46 @@ static void set_isa(struct cpuinfo_mips *c, unsigned int isa) switch (isa) { case MIPS_CPU_ISA_M64R2: c->isa_level |= MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2; + set_elf_base_platform("mips64r2"); /* fall through */ case MIPS_CPU_ISA_M64R1: c->isa_level |= MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1; + set_elf_base_platform("mips64"); /* fall through */ case MIPS_CPU_ISA_V: c->isa_level |= MIPS_CPU_ISA_V; + set_elf_base_platform("mips5"); /* fall through */ case MIPS_CPU_ISA_IV: c->isa_level |= MIPS_CPU_ISA_IV; + set_elf_base_platform("mips4"); /* fall through */ case MIPS_CPU_ISA_III: c->isa_level |= MIPS_CPU_ISA_II | MIPS_CPU_ISA_III; + set_elf_base_platform("mips3"); break; /* R6 incompatible with everything else */ case MIPS_CPU_ISA_M64R6: c->isa_level |= MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6; + set_elf_base_platform("mips64r6"); /* fall through */ case MIPS_CPU_ISA_M32R6: c->isa_level |= MIPS_CPU_ISA_M32R6; + set_elf_base_platform("mips32r6"); /* Break here so we don't add incompatible ISAs */ break; case MIPS_CPU_ISA_M32R2: c->isa_level |= MIPS_CPU_ISA_M32R2; + set_elf_base_platform("mips32r2"); /* fall through */ case MIPS_CPU_ISA_M32R1: c->isa_level |= MIPS_CPU_ISA_M32R1; + set_elf_base_platform("mips32"); /* fall through */ case MIPS_CPU_ISA_II: c->isa_level |= MIPS_CPU_ISA_II; + set_elf_base_platform("mips2"); break; } } @@ -2113,6 +2130,7 @@ EXPORT_SYMBOL(__ua_limit); const char *__cpu_name[NR_CPUS]; const char *__elf_platform; +const char *__elf_base_platform; void cpu_probe(void) { diff --git a/arch/mips/kernel/csrc-r4k.c b/arch/mips/kernel/csrc-r4k.c index eed099f35bf1..437dda64fd7a 100644 --- a/arch/mips/kernel/csrc-r4k.c +++ b/arch/mips/kernel/csrc-r4k.c @@ -78,7 +78,7 @@ int __init init_r4k_clocksource(void) * by the VDSO (HWREna is configured by configure_hwrena()). */ if (cpu_has_mips_r2_r6 && rdhwr_count_usable()) - clocksource_mips.archdata.vdso_clock_mode = VDSO_CLOCK_R4K; + clocksource_mips.vdso_clock_mode = VDSO_CLOCKMODE_R4K; clocksource_register_hz(&clocksource_mips, mips_hpt_frequency); diff --git a/arch/mips/kernel/i8253.c b/arch/mips/kernel/i8253.c index df7ddd246eaa..ca21210e06b5 100644 --- a/arch/mips/kernel/i8253.c +++ b/arch/mips/kernel/i8253.c @@ -18,16 +18,13 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq0 = { - .handler = timer_interrupt, - .flags = IRQF_NOBALANCING | IRQF_TIMER, - .name = "timer" -}; - void __init setup_pit_timer(void) { + unsigned long flags = IRQF_NOBALANCING | IRQF_TIMER; + clockevent_i8253_init(true); - setup_irq(0, &irq0); + if (request_irq(0, timer_interrupt, flags, "timer", NULL)) + pr_err("Failed to request irq 0 (timer)\n"); } static int __init init_pit_clocksource(void) diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 339870ed92f7..b2a797557825 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -75,7 +75,9 @@ void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp) lose_fpu(0); clear_thread_flag(TIF_MSA_CTX_LIVE); clear_used_math(); +#ifdef CONFIG_MIPS_FP_SUPPORT atomic_set(¤t->thread.bd_emu_frame, BD_EMUFRAME_NONE); +#endif init_dsp(); regs->cp0_epc = pc; regs->regs[29] = sp; @@ -176,7 +178,9 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long usp, clear_tsk_thread_flag(p, TIF_FPUBOUND); #endif /* CONFIG_MIPS_MT_FPAFF */ +#ifdef CONFIG_MIPS_FP_SUPPORT atomic_set(&p->thread.bd_emu_frame, BD_EMUFRAME_NONE); +#endif if (clone_flags & CLONE_SETTLS) ti->tp_value = tls; @@ -650,8 +654,10 @@ unsigned long mips_stack_top(void) { unsigned long top = TASK_SIZE & PAGE_MASK; - /* One page for branch delay slot "emulation" */ - top -= PAGE_SIZE; + if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { + /* One page for branch delay slot "emulation" */ + top -= PAGE_SIZE; + } /* Space for the VDSO, data page & GIC user page */ top -= PAGE_ALIGN(current->thread.abi->vdso->size); diff --git a/arch/mips/kernel/rtlx-mt.c b/arch/mips/kernel/rtlx-mt.c index cb95470e2e69..38c6925a1bea 100644 --- a/arch/mips/kernel/rtlx-mt.c +++ b/arch/mips/kernel/rtlx-mt.c @@ -51,11 +51,6 @@ static irqreturn_t rtlx_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction rtlx_irq = { - .handler = rtlx_interrupt, - .name = "RTLX", -}; - static int rtlx_irq_num = MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ; void _interrupt_sp(void) @@ -124,8 +119,7 @@ int __init rtlx_module_init(void) goto out_class; } - rtlx_irq.dev_id = rtlx; - err = setup_irq(rtlx_irq_num, &rtlx_irq); + err = request_irq(rtlx_irq_num, rtlx_interrupt, 0, "RTLX", rtlx); if (err) goto out_class; diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index a7b469d89e2c..10bef8f78e7c 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -28,6 +28,7 @@ #include <linux/decompress/generic.h> #include <linux/of_fdt.h> #include <linux/of_reserved_mem.h> +#include <linux/dmi.h> #include <asm/addrspace.h> #include <asm/bootinfo.h> @@ -799,6 +800,7 @@ void __init setup_arch(char **cmdline_p) #endif arch_mem_init(cmdline_p); + dmi_setup(); resource_init(); plat_smp_setup(); diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index f510c00bda88..48d84d5fcc36 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -207,25 +207,13 @@ static irqreturn_t ipi_call_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq_resched = { - .handler = ipi_resched_interrupt, - .flags = IRQF_PERCPU, - .name = "IPI resched" -}; - -static struct irqaction irq_call = { - .handler = ipi_call_interrupt, - .flags = IRQF_PERCPU, - .name = "IPI call" -}; - -static void smp_ipi_init_one(unsigned int virq, - struct irqaction *action) +static void smp_ipi_init_one(unsigned int virq, const char *name, + irq_handler_t handler) { int ret; irq_set_handler(virq, handle_percpu_irq); - ret = setup_irq(virq, action); + ret = request_irq(virq, handler, IRQF_PERCPU, name, NULL); BUG_ON(ret); } @@ -278,12 +266,15 @@ int mips_smp_ipi_allocate(const struct cpumask *mask) int cpu; for_each_cpu(cpu, mask) { - smp_ipi_init_one(call_virq + cpu, &irq_call); - smp_ipi_init_one(sched_virq + cpu, &irq_resched); + smp_ipi_init_one(call_virq + cpu, "IPI call", + ipi_call_interrupt); + smp_ipi_init_one(sched_virq + cpu, "IPI resched", + ipi_resched_interrupt); } } else { - smp_ipi_init_one(call_virq, &irq_call); - smp_ipi_init_one(sched_virq, &irq_resched); + smp_ipi_init_one(call_virq, "IPI call", ipi_call_interrupt); + smp_ipi_init_one(sched_virq, "IPI resched", + ipi_resched_interrupt); } return 0; @@ -311,8 +302,8 @@ int mips_smp_ipi_free(const struct cpumask *mask) int cpu; for_each_cpu(cpu, mask) { - remove_irq(call_virq + cpu, &irq_call); - remove_irq(sched_virq + cpu, &irq_resched); + free_irq(call_virq + cpu, NULL); + free_irq(sched_virq + cpu, NULL); } } irq_destroy_ipi(call_virq, mask); @@ -696,29 +687,22 @@ EXPORT_SYMBOL(flush_tlb_one); #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST -static DEFINE_PER_CPU(atomic_t, tick_broadcast_count); static DEFINE_PER_CPU(call_single_data_t, tick_broadcast_csd); void tick_broadcast(const struct cpumask *mask) { - atomic_t *count; call_single_data_t *csd; int cpu; for_each_cpu(cpu, mask) { - count = &per_cpu(tick_broadcast_count, cpu); csd = &per_cpu(tick_broadcast_csd, cpu); - - if (atomic_inc_return(count) == 1) - smp_call_function_single_async(cpu, csd); + smp_call_function_single_async(cpu, csd); } } static void tick_broadcast_callee(void *info) { - int cpu = smp_processor_id(); tick_receive_broadcast(); - atomic_set(&per_cpu(tick_broadcast_count, cpu), 0); } static int __init tick_broadcast_init(void) diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index bc35f8499111..3adb7354bc01 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -71,10 +71,12 @@ subsys_initcall(init_vdso); static unsigned long vdso_base(void) { - unsigned long base; + unsigned long base = STACK_TOP; - /* Skip the delay slot emulation page */ - base = STACK_TOP + PAGE_SIZE; + if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { + /* Skip the delay slot emulation page */ + base += PAGE_SIZE; + } if (current->flags & PF_RANDOMIZE) { base += get_random_int() & (VDSO_RANDOMIZE_SIZE - 1); @@ -95,14 +97,16 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) if (down_write_killable(&mm->mmap_sem)) return -EINTR; - /* Map delay slot emulation page */ - base = mmap_region(NULL, STACK_TOP, PAGE_SIZE, - VM_READ | VM_EXEC | - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, - 0, NULL); - if (IS_ERR_VALUE(base)) { - ret = base; - goto out; + if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { + /* Map delay slot emulation page */ + base = mmap_region(NULL, STACK_TOP, PAGE_SIZE, + VM_READ | VM_EXEC | + VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, + 0, NULL); + if (IS_ERR_VALUE(base)) { + ret = base; + goto out; + } } /* diff --git a/arch/mips/lasat/interrupt.c b/arch/mips/lasat/interrupt.c index 0f3a8975081d..7965bbd0d319 100644 --- a/arch/mips/lasat/interrupt.c +++ b/arch/mips/lasat/interrupt.c @@ -90,14 +90,9 @@ asmlinkage void plat_irq_dispatch(void) } } -static struct irqaction cascade = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; - void __init arch_init_irq(void) { + int irq = LASAT_CASCADE_IRQ; int i; if (IS_LASAT_200()) { @@ -119,5 +114,6 @@ void __init arch_init_irq(void) for (i = LASAT_IRQ_BASE; i <= LASAT_IRQ_END; i++) irq_set_chip_and_handler(i, &lasat_irq_type, handle_level_irq); - setup_irq(LASAT_CASCADE_IRQ, &cascade); + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", irq); } diff --git a/arch/mips/lib/delay.c b/arch/mips/lib/delay.c index 68c495ed71e3..2e8dfc1d59c8 100644 --- a/arch/mips/lib/delay.c +++ b/arch/mips/lib/delay.c @@ -24,6 +24,8 @@ #define GCC_DADDI_IMM_ASM() "r" #endif +#ifndef CONFIG_HAVE_PLAT_DELAY + void __delay(unsigned long loops) { __asm__ __volatile__ ( @@ -63,3 +65,5 @@ void __ndelay(unsigned long ns) __delay((ns * 0x00000005ull * HZ * lpj) >> 32); } EXPORT_SYMBOL(__ndelay); + +#endif diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S index f7994d936505..88065ee433cd 100644 --- a/arch/mips/lib/memcpy.S +++ b/arch/mips/lib/memcpy.S @@ -598,6 +598,7 @@ SEXC(1) nop .endm +#ifndef CONFIG_HAVE_PLAT_MEMCPY .align 5 LEAF(memmove) EXPORT_SYMBOL(memmove) @@ -665,6 +666,8 @@ EXPORT_SYMBOL(__copy_user) /* Legacy Mode, user <-> user */ __BUILD_COPY_USER LEGACY_MODE USEROP USEROP +#endif + #ifdef CONFIG_EVA /* diff --git a/arch/mips/loongson2ef/common/bonito-irq.c b/arch/mips/loongson2ef/common/bonito-irq.c index 82352cc25e4c..c06ad412ee20 100644 --- a/arch/mips/loongson2ef/common/bonito-irq.c +++ b/arch/mips/loongson2ef/common/bonito-irq.c @@ -30,11 +30,6 @@ static struct irq_chip bonito_irq_type = { .irq_unmask = bonito_irq_enable, }; -static struct irqaction __maybe_unused dma_timeout_irqaction = { - .handler = no_action, - .name = "dma_timeout", -}; - void bonito_irq_init(void) { u32 i; @@ -44,6 +39,8 @@ void bonito_irq_init(void) handle_level_irq); #ifdef CONFIG_CPU_LOONGSON2E - setup_irq(LOONGSON_IRQ_BASE + 10, &dma_timeout_irqaction); + i = LOONGSON_IRQ_BASE + 10; + if (request_irq(i, no_action, 0, "dma_timeout", NULL)) + pr_err("Failed to request irq %d (dma_timeout)\n", i); #endif } diff --git a/arch/mips/loongson2ef/common/cs5536/cs5536_mfgpt.c b/arch/mips/loongson2ef/common/cs5536/cs5536_mfgpt.c index 30af1b7c7529..f21a540a1dd2 100644 --- a/arch/mips/loongson2ef/common/cs5536/cs5536_mfgpt.c +++ b/arch/mips/loongson2ef/common/cs5536/cs5536_mfgpt.c @@ -100,12 +100,6 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq5 = { - .handler = timer_interrupt, - .flags = IRQF_NOBALANCING | IRQF_TIMER, - .name = "timer" -}; - /* * Initialize the conversion factor and the min/max deltas of the clock event * structure and register the clock event source with the framework. @@ -134,7 +128,9 @@ void __init setup_mfgpt0_timer(void) clockevents_register_device(cd); - setup_irq(CS5536_MFGPT_INTR, &irq5); + if (request_irq(CS5536_MFGPT_INTR, timer_interrupt, + IRQF_NOBALANCING | IRQF_TIMER, "timer", NULL)) + pr_err("Failed to register timer interrupt\n"); } /* diff --git a/arch/mips/loongson2ef/fuloong-2e/irq.c b/arch/mips/loongson2ef/fuloong-2e/irq.c index 32278e7bf85c..305aa2eb74ad 100644 --- a/arch/mips/loongson2ef/fuloong-2e/irq.c +++ b/arch/mips/loongson2ef/fuloong-2e/irq.c @@ -35,14 +35,10 @@ asmlinkage void mach_irq_dispatch(unsigned int pending) spurious_interrupt(); } -static struct irqaction cascade_irqaction = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; - void __init mach_init_irq(void) { + int irq; + /* init all controller * 0-15 ------> i8259 interrupt * 16-23 ------> mips cpu interrupt @@ -59,7 +55,11 @@ void __init mach_init_irq(void) bonito_irq_init(); /* bonito irq at IP2 */ - setup_irq(MIPS_CPU_IRQ_BASE + 2, &cascade_irqaction); + irq = MIPS_CPU_IRQ_BASE + 2; + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", irq); /* 8259 irq at IP5 */ - setup_irq(MIPS_CPU_IRQ_BASE + 5, &cascade_irqaction); + irq = MIPS_CPU_IRQ_BASE + 5; + if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", irq); } diff --git a/arch/mips/loongson2ef/lemote-2f/irq.c b/arch/mips/loongson2ef/lemote-2f/irq.c index c58a044c6c07..6f00579971a3 100644 --- a/arch/mips/loongson2ef/lemote-2f/irq.c +++ b/arch/mips/loongson2ef/lemote-2f/irq.c @@ -90,18 +90,6 @@ static irqreturn_t ip6_action(int cpl, void *dev_id) return IRQ_HANDLED; } -static struct irqaction ip6_irqaction = { - .handler = ip6_action, - .name = "cascade", - .flags = IRQF_SHARED | IRQF_NO_THREAD, -}; - -static struct irqaction cascade_irqaction = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD | IRQF_NO_SUSPEND, -}; - void __init mach_init_irq(void) { /* init all controller @@ -120,7 +108,11 @@ void __init mach_init_irq(void) bonito_irq_init(); /* setup north bridge irq (bonito) */ - setup_irq(LOONGSON_NORTH_BRIDGE_IRQ, &ip6_irqaction); + if (request_irq(LOONGSON_NORTH_BRIDGE_IRQ, ip6_action, + IRQF_SHARED | IRQF_NO_THREAD, "cascade", ip6_action)) + pr_err("Failed to register north bridge cascade interrupt\n"); /* setup source bridge irq (i8259) */ - setup_irq(LOONGSON_SOUTH_BRIDGE_IRQ, &cascade_irqaction); + if (request_irq(LOONGSON_SOUTH_BRIDGE_IRQ, no_action, + IRQF_NO_THREAD | IRQF_NO_SUSPEND, "cascade", NULL)) + pr_err("Failed to register south bridge cascade interrupt\n"); } diff --git a/arch/mips/loongson32/common/irq.c b/arch/mips/loongson32/common/irq.c index 168d221d4178..9a50070f74f7 100644 --- a/arch/mips/loongson32/common/irq.c +++ b/arch/mips/loongson32/common/irq.c @@ -149,12 +149,6 @@ asmlinkage void plat_irq_dispatch(void) } -static struct irqaction cascade_irqaction = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; - static void __init ls1x_irq_init(int base) { int n; @@ -176,12 +170,17 @@ static void __init ls1x_irq_init(int base) handle_level_irq); } - setup_irq(INT0_IRQ, &cascade_irqaction); - setup_irq(INT1_IRQ, &cascade_irqaction); - setup_irq(INT2_IRQ, &cascade_irqaction); - setup_irq(INT3_IRQ, &cascade_irqaction); + if (request_irq(INT0_IRQ, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", INT0_IRQ); + if (request_irq(INT1_IRQ, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", INT1_IRQ); + if (request_irq(INT2_IRQ, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", INT2_IRQ); + if (request_irq(INT3_IRQ, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", INT3_IRQ); #if defined(CONFIG_LOONGSON1_LS1C) - setup_irq(INT4_IRQ, &cascade_irqaction); + if (request_irq(INT4_IRQ, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to request irq %d (cascade)\n", INT4_IRQ); #endif } diff --git a/arch/mips/loongson32/common/time.c b/arch/mips/loongson32/common/time.c index 4cc73f7ac0d4..459b15c96d3b 100644 --- a/arch/mips/loongson32/common/time.c +++ b/arch/mips/loongson32/common/time.c @@ -176,13 +176,6 @@ static struct clock_event_device ls1x_clockevent = { .tick_resume = ls1x_clockevent_tick_resume, }; -static struct irqaction ls1x_pwmtimer_irqaction = { - .name = "ls1x-pwmtimer", - .handler = ls1x_clockevent_isr, - .dev_id = &ls1x_clockevent, - .flags = IRQF_PERCPU | IRQF_TIMER, -}; - static void __init ls1x_time_init(void) { struct clock_event_device *cd = &ls1x_clockevent; @@ -206,7 +199,10 @@ static void __init ls1x_time_init(void) if (ret) panic(KERN_ERR "Failed to register clocksource: %d\n", ret); - setup_irq(LS1X_TIMER_IRQ, &ls1x_pwmtimer_irqaction); + if (request_irq(LS1X_TIMER_IRQ, ls1x_clockevent_isr, + IRQF_PERCPU | IRQF_TIMER, "ls1x-pwmtimer", + &ls1x_clockevent)) + pr_err("Failed to register ls1x-pwmtimer interrupt\n"); } #endif /* CONFIG_CEVT_CSRC_LS1X */ diff --git a/arch/mips/loongson64/Makefile b/arch/mips/loongson64/Makefile index 7821891bc5d0..b7f40b179c71 100644 --- a/arch/mips/loongson64/Makefile +++ b/arch/mips/loongson64/Makefile @@ -2,7 +2,7 @@ # # Makefile for Loongson-3 family machines # -obj-$(CONFIG_MACH_LOONGSON64) += irq.o cop2-ex.o platform.o acpi_init.o dma.o \ +obj-$(CONFIG_MACH_LOONGSON64) += cop2-ex.o platform.o acpi_init.o dma.o \ setup.o init.o env.o time.o reset.o \ obj-$(CONFIG_SMP) += smp.o diff --git a/arch/mips/loongson64/env.c b/arch/mips/loongson64/env.c index 0daeb7bcf023..2554ef11170d 100644 --- a/arch/mips/loongson64/env.c +++ b/arch/mips/loongson64/env.c @@ -17,6 +17,7 @@ #include <asm/bootinfo.h> #include <loongson.h> #include <boot_param.h> +#include <builtin_dtbs.h> #include <workarounds.h> u32 cpu_clock_freq; @@ -120,6 +121,28 @@ void __init prom_init_env(void) loongson_sysconf.cores_per_node - 1) / loongson_sysconf.cores_per_node; + if ((read_c0_prid() & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64C) { + switch (read_c0_prid() & PRID_REV_MASK) { + case PRID_REV_LOONGSON3A_R1: + case PRID_REV_LOONGSON3A_R2_0: + case PRID_REV_LOONGSON3A_R2_1: + case PRID_REV_LOONGSON3A_R3_0: + case PRID_REV_LOONGSON3A_R3_1: + loongson_fdt_blob = __dtb_loongson3_4core_rs780e_begin; + break; + case PRID_REV_LOONGSON3B_R1: + case PRID_REV_LOONGSON3B_R2: + loongson_fdt_blob = __dtb_loongson3_8core_rs780e_begin; + break; + default: + break; + } + } + + + if (!loongson_fdt_blob) + pr_err("Failed to determine built-in Loongson64 dtb\n"); + loongson_sysconf.pci_mem_start_addr = eirq_source->pci_mem_start_addr; loongson_sysconf.pci_mem_end_addr = eirq_source->pci_mem_end_addr; loongson_sysconf.pci_io_base = eirq_source->pci_io_start_addr; diff --git a/arch/mips/loongson64/hpet.c b/arch/mips/loongson64/hpet.c index ed15430ad64f..e4282592575b 100644 --- a/arch/mips/loongson64/hpet.c +++ b/arch/mips/loongson64/hpet.c @@ -187,12 +187,6 @@ static irqreturn_t hpet_irq_handler(int irq, void *data) return IRQ_NONE; } -static struct irqaction hpet_irq = { - .handler = hpet_irq_handler, - .flags = IRQF_NOBALANCING | IRQF_TIMER, - .name = "hpet", -}; - /* * hpet address assignation and irq setting should be done in bios. * but pmon don't do this, we just setup here directly. @@ -224,6 +218,7 @@ static void hpet_setup(void) void __init setup_hpet_timer(void) { + unsigned long flags = IRQF_NOBALANCING | IRQF_TIMER; unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; @@ -247,7 +242,8 @@ void __init setup_hpet_timer(void) cd->min_delta_ticks = HPET_MIN_PROG_DELTA; clockevents_register_device(cd); - setup_irq(HPET_T0_IRQ, &hpet_irq); + if (request_irq(HPET_T0_IRQ, hpet_irq_handler, flags, "hpet", NULL)) + pr_err("Failed to request irq %d (hpet)\n", HPET_T0_IRQ); pr_info("hpet clock event device register\n"); } diff --git a/arch/mips/loongson64/init.c b/arch/mips/loongson64/init.c index 5ac1a0f35ca4..da38944471f4 100644 --- a/arch/mips/loongson64/init.c +++ b/arch/mips/loongson64/init.c @@ -4,6 +4,7 @@ * Author: Wu Zhangjin, wuzhangjin@gmail.com */ +#include <linux/irqchip.h> #include <linux/memblock.h> #include <asm/bootinfo.h> #include <asm/traps.h> @@ -44,3 +45,8 @@ void __init prom_init(void) void __init prom_free_prom_memory(void) { } + +void __init arch_init_irq(void) +{ + irqchip_init(); +} diff --git a/arch/mips/loongson64/irq.c b/arch/mips/loongson64/irq.c deleted file mode 100644 index 79ad797497e4..000000000000 --- a/arch/mips/loongson64/irq.c +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <loongson.h> -#include <irq.h> -#include <linux/interrupt.h> -#include <linux/init.h> - -#include <asm/irq_cpu.h> -#include <asm/i8259.h> -#include <asm/mipsregs.h> - -#include "smp.h" - -extern void loongson3_send_irq_by_ipi(int cpu, int irqs); - -unsigned int irq_cpu[16] = {[0 ... 15] = -1}; -unsigned int ht_irq[] = {0, 1, 3, 4, 5, 6, 7, 8, 12, 14, 15}; -unsigned int local_irq = 1<<0 | 1<<1 | 1<<2 | 1<<7 | 1<<8 | 1<<12; - -int plat_set_irq_affinity(struct irq_data *d, const struct cpumask *affinity, - bool force) -{ - unsigned int cpu; - struct cpumask new_affinity; - - /* I/O devices are connected on package-0 */ - cpumask_copy(&new_affinity, affinity); - for_each_cpu(cpu, affinity) - if (cpu_data[cpu].package > 0) - cpumask_clear_cpu(cpu, &new_affinity); - - if (cpumask_empty(&new_affinity)) - return -EINVAL; - - cpumask_copy(d->common->affinity, &new_affinity); - - return IRQ_SET_MASK_OK_NOCOPY; -} - -static void ht_irqdispatch(void) -{ - unsigned int i, irq; - struct irq_data *irqd; - struct cpumask affinity; - - irq = LOONGSON_HT1_INT_VECTOR(0); - LOONGSON_HT1_INT_VECTOR(0) = irq; /* Acknowledge the IRQs */ - - for (i = 0; i < ARRAY_SIZE(ht_irq); i++) { - if (!(irq & (0x1 << ht_irq[i]))) - continue; - - /* handled by local core */ - if (local_irq & (0x1 << ht_irq[i])) { - do_IRQ(ht_irq[i]); - continue; - } - - irqd = irq_get_irq_data(ht_irq[i]); - cpumask_and(&affinity, irqd->common->affinity, cpu_active_mask); - if (cpumask_empty(&affinity)) { - do_IRQ(ht_irq[i]); - continue; - } - - irq_cpu[ht_irq[i]] = cpumask_next(irq_cpu[ht_irq[i]], &affinity); - if (irq_cpu[ht_irq[i]] >= nr_cpu_ids) - irq_cpu[ht_irq[i]] = cpumask_first(&affinity); - - if (irq_cpu[ht_irq[i]] == 0) { - do_IRQ(ht_irq[i]); - continue; - } - - /* balanced by other cores */ - loongson3_send_irq_by_ipi(irq_cpu[ht_irq[i]], (0x1 << ht_irq[i])); - } -} - -#define UNUSED_IPS (CAUSEF_IP5 | CAUSEF_IP4 | CAUSEF_IP1 | CAUSEF_IP0) - -asmlinkage void plat_irq_dispatch(void) -{ - unsigned int pending; - - pending = read_c0_cause() & read_c0_status() & ST0_IM; - - if (pending & CAUSEF_IP7) - do_IRQ(LOONGSON_TIMER_IRQ); -#if defined(CONFIG_SMP) - if (pending & CAUSEF_IP6) - loongson3_ipi_interrupt(NULL); -#endif - if (pending & CAUSEF_IP3) - ht_irqdispatch(); - if (pending & CAUSEF_IP2) - do_IRQ(LOONGSON_UART_IRQ); - if (pending & UNUSED_IPS) { - pr_err("%s : spurious interrupt\n", __func__); - spurious_interrupt(); - } -} - -static inline void mask_loongson_irq(struct irq_data *d) { } -static inline void unmask_loongson_irq(struct irq_data *d) { } - - /* For MIPS IRQs which shared by all cores */ -static struct irq_chip loongson_irq_chip = { - .name = "Loongson", - .irq_ack = mask_loongson_irq, - .irq_mask = mask_loongson_irq, - .irq_mask_ack = mask_loongson_irq, - .irq_unmask = unmask_loongson_irq, - .irq_eoi = unmask_loongson_irq, -}; - -void irq_router_init(void) -{ - int i; - - /* route LPC int to cpu core0 int 0 */ - LOONGSON_INT_ROUTER_LPC = - LOONGSON_INT_COREx_INTy(loongson_sysconf.boot_cpu_id, 0); - /* route HT1 int0 ~ int7 to cpu core0 INT1*/ - for (i = 0; i < 8; i++) - LOONGSON_INT_ROUTER_HT1(i) = - LOONGSON_INT_COREx_INTy(loongson_sysconf.boot_cpu_id, 1); - /* enable HT1 interrupt */ - LOONGSON_HT1_INTN_EN(0) = 0xffffffff; - /* enable router interrupt intenset */ - LOONGSON_INT_ROUTER_INTENSET = - LOONGSON_INT_ROUTER_INTEN | (0xffff << 16) | 0x1 << 10; -} - -void __init arch_init_irq(void) -{ - struct irq_chip *chip; - - clear_c0_status(ST0_IM | ST0_BEV); - - irq_router_init(); - mips_cpu_irq_init(); - init_i8259_irqs(); - chip = irq_get_chip(I8259A_IRQ_BASE); - chip->irq_set_affinity = plat_set_irq_affinity; - - irq_set_chip_and_handler(LOONGSON_UART_IRQ, - &loongson_irq_chip, handle_percpu_irq); - irq_set_chip_and_handler(LOONGSON_BRIDGE_IRQ, - &loongson_irq_chip, handle_percpu_irq); - - set_c0_status(STATUSF_IP2 | STATUSF_IP3 | STATUSF_IP6); -} - -#ifdef CONFIG_HOTPLUG_CPU - -void fixup_irqs(void) -{ - irq_cpu_offline(); - clear_c0_status(ST0_IM); -} - -#endif diff --git a/arch/mips/loongson64/numa.c b/arch/mips/loongson64/numa.c index e5b40c5e3296..1ae072df4831 100644 --- a/arch/mips/loongson64/numa.c +++ b/arch/mips/loongson64/numa.c @@ -122,7 +122,7 @@ static unsigned long nid_to_addroffset(unsigned int nid) static void __init szmem(unsigned int node) { u32 i, mem_type; - static unsigned long num_physpages = 0; + static unsigned long num_physpages; u64 node_id, node_psize, start_pfn, end_pfn, mem_start, mem_size; /* Parse memory information and activate */ diff --git a/arch/mips/loongson64/setup.c b/arch/mips/loongson64/setup.c index 4fd27f4f90ed..6fe3ffffcaa6 100644 --- a/arch/mips/loongson64/setup.c +++ b/arch/mips/loongson64/setup.c @@ -8,9 +8,15 @@ #include <asm/wbflush.h> #include <asm/bootinfo.h> +#include <linux/libfdt.h> +#include <linux/of_fdt.h> + +#include <asm/prom.h> #include <loongson.h> +void *loongson_fdt_blob; + static void wbflush_loongson(void) { asm(".set\tpush\n\t" @@ -27,4 +33,14 @@ EXPORT_SYMBOL(__wbflush); void __init plat_mem_setup(void) { + if (loongson_fdt_blob) + __dt_setup_arch(loongson_fdt_blob); +} + +void __init device_tree_init(void) +{ + if (!initial_boot_params) + return; + + unflatten_and_copy_device_tree(); } diff --git a/arch/mips/loongson64/smp.c b/arch/mips/loongson64/smp.c index de8e0741ce2d..e1fe8bbb377d 100644 --- a/arch/mips/loongson64/smp.c +++ b/arch/mips/loongson64/smp.c @@ -4,6 +4,7 @@ * Author: Chen Huacai, chenhc@lemote.com */ +#include <irq.h> #include <linux/init.h> #include <linux/cpu.h> #include <linux/sched.h> @@ -25,6 +26,8 @@ DEFINE_PER_CPU(int, cpu_state); +#define LS_IPI_IRQ (MIPS_CPU_IRQ_BASE + 6) + static void *ipi_set0_regs[16]; static void *ipi_clear0_regs[16]; static void *ipi_status0_regs[16]; @@ -302,20 +305,13 @@ loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) ipi_write_action(cpu_logical_map(i), (u32)action); } -#define IPI_IRQ_OFFSET 6 - -void loongson3_send_irq_by_ipi(int cpu, int irqs) -{ - ipi_write_action(cpu_logical_map(cpu), irqs << IPI_IRQ_OFFSET); -} -void loongson3_ipi_interrupt(struct pt_regs *regs) +static irqreturn_t loongson3_ipi_interrupt(int irq, void *dev_id) { int i, cpu = smp_processor_id(); - unsigned int action, c0count, irqs; + unsigned int action, c0count; action = ipi_read_clear(cpu); - irqs = action >> IPI_IRQ_OFFSET; if (action & SMP_RESCHEDULE_YOURSELF) scheduler_ipi(); @@ -335,13 +331,7 @@ void loongson3_ipi_interrupt(struct pt_regs *regs) __wbflush(); /* Let others see the result ASAP */ } - if (irqs) { - int irq; - while ((irq = ffs(irqs))) { - do_IRQ(irq-1); - irqs &= ~(1<<(irq-1)); - } - } + return IRQ_HANDLED; } #define MAX_LOOPS 800 @@ -438,6 +428,9 @@ static void __init loongson3_smp_setup(void) static void __init loongson3_prepare_cpus(unsigned int max_cpus) { + if (request_irq(LS_IPI_IRQ, loongson3_ipi_interrupt, + IRQF_PERCPU | IRQF_NO_SUSPEND, "SMP_IPI", NULL)) + pr_err("Failed to request IPI IRQ\n"); init_cpu_present(cpu_possible_mask); per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; } @@ -484,7 +477,8 @@ static int loongson3_cpu_disable(void) set_cpu_online(cpu, false); calculate_cpu_foreign_map(); local_irq_save(flags); - fixup_irqs(); + irq_cpu_offline(); + clear_c0_status(ST0_IM); local_irq_restore(flags); local_flush_tlb_all(); diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c index 8064821e9805..4baf965e6fe8 100644 --- a/arch/mips/mm/c-octeon.c +++ b/arch/mips/mm/c-octeon.c @@ -237,17 +237,17 @@ static void probe_octeon(void) c->dcache.sets = dcache_size / (c->dcache.linesz * c->dcache.ways); if (smp_processor_id() == 0) { - pr_notice("Primary instruction cache %ldkB, %s, %d way, " - "%d sets, linesize %d bytes.\n", - icache_size >> 10, - cpu_has_vtag_icache ? + pr_info("Primary instruction cache %ldkB, %s, %d way, " + "%d sets, linesize %d bytes.\n", + icache_size >> 10, + cpu_has_vtag_icache ? "virtually tagged" : "physically tagged", - c->icache.ways, c->icache.sets, c->icache.linesz); + c->icache.ways, c->icache.sets, c->icache.linesz); - pr_notice("Primary data cache %ldkB, %d-way, %d sets, " - "linesize %d bytes.\n", - dcache_size >> 10, c->dcache.ways, - c->dcache.sets, c->dcache.linesz); + pr_info("Primary data cache %ldkB, %d-way, %d sets, " + "linesize %d bytes.\n", + dcache_size >> 10, c->dcache.ways, + c->dcache.sets, c->dcache.linesz); } } diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c index 15bb8cf59828..780dd2a567c1 100644 --- a/arch/mips/mm/c-r3k.c +++ b/arch/mips/mm/c-r3k.c @@ -316,9 +316,9 @@ void r3k_cache_init(void) _dma_cache_wback = r3k_dma_cache_wback_inv; _dma_cache_inv = r3k_dma_cache_wback_inv; - printk("Primary instruction cache %ldkB, linesize %ld bytes.\n", + pr_info("Primary instruction cache %ldkB, linesize %ld bytes.\n", icache_size >> 10, icache_lsize); - printk("Primary data cache %ldkB, linesize %ld bytes.\n", + pr_info("Primary data cache %ldkB, linesize %ld bytes.\n", dcache_size >> 10, dcache_lsize); build_clear_page(); diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 5f3d0103b95d..36a311348739 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -901,6 +901,31 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) __sync(); } +static void prefetch_cache_inv(unsigned long addr, unsigned long size) +{ + unsigned int linesz = cpu_scache_line_size(); + unsigned long addr0 = addr, addr1; + + addr0 &= ~(linesz - 1); + addr1 = (addr0 + size - 1) & ~(linesz - 1); + + protected_writeback_scache_line(addr0); + if (likely(addr1 != addr0)) + protected_writeback_scache_line(addr1); + else + return; + + addr0 += linesz; + if (likely(addr1 != addr0)) + protected_writeback_scache_line(addr0); + else + return; + + addr1 -= linesz; + if (likely(addr1 > addr0)) + protected_writeback_scache_line(addr0); +} + static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) { /* Catch bad driver code */ @@ -908,6 +933,10 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) return; preempt_disable(); + + if (current_cpu_type() == CPU_BMIPS5000) + prefetch_cache_inv(addr, size); + if (cpu_has_inclusive_pcaches) { if (size >= scache_size) { if (current_cpu_type() != CPU_LOONGSON64) @@ -1467,17 +1496,17 @@ static void probe_pcache(void) c->icache.ways = 1; } - printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", - icache_size >> 10, - c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", - way_string[c->icache.ways], c->icache.linesz); + pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", + icache_size >> 10, + c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", + way_string[c->icache.ways], c->icache.linesz); - printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", - dcache_size >> 10, way_string[c->dcache.ways], - (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT", - (c->dcache.flags & MIPS_CACHE_ALIASES) ? + pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", + dcache_size >> 10, way_string[c->dcache.ways], + (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT", + (c->dcache.flags & MIPS_CACHE_ALIASES) ? "cache aliases" : "no aliases", - c->dcache.linesz); + c->dcache.linesz); } static void probe_vcache(void) diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c index 686867270627..2d479cc7e66b 100644 --- a/arch/mips/mm/c-tx39.c +++ b/arch/mips/mm/c-tx39.c @@ -410,9 +410,9 @@ void tx39_cache_init(void) current_cpu_data.icache.waybit = 0; current_cpu_data.dcache.waybit = 0; - printk("Primary instruction cache %ldkB, linesize %d bytes\n", + pr_info("Primary instruction cache %ldkB, linesize %d bytes\n", icache_size >> 10, current_cpu_data.icache.linesz); - printk("Primary data cache %ldkB, linesize %d bytes\n", + pr_info("Primary data cache %ldkB, linesize %d bytes\n", dcache_size >> 10, current_cpu_data.dcache.linesz); build_clear_page(); diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 344e6e9ea43b..da407cdc2135 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -1480,6 +1480,7 @@ static void build_r4000_tlb_refill_handler(void) static void setup_pw(void) { + unsigned int pwctl; unsigned long pgd_i, pgd_w; #ifndef __PAGETABLE_PMD_FOLDED unsigned long pmd_i, pmd_w; @@ -1506,6 +1507,7 @@ static void setup_pw(void) pte_i = ilog2(_PAGE_GLOBAL); pte_w = 0; + pwctl = 1 << 30; /* Set PWDirExt */ #ifndef __PAGETABLE_PMD_FOLDED write_c0_pwfield(pgd_i << 24 | pmd_i << 12 | pt_i << 6 | pte_i); @@ -1516,8 +1518,9 @@ static void setup_pw(void) #endif #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT - write_c0_pwctl(1 << 6 | psn); + pwctl |= (1 << 6 | psn); #endif + write_c0_pwctl(pwctl); write_c0_kpgd((long)swapper_pg_dir); kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */ } diff --git a/arch/mips/mti-malta/malta-int.c b/arch/mips/mti-malta/malta-int.c index a840e0c1642c..03d85b2b3eea 100644 --- a/arch/mips/mti-malta/malta-int.c +++ b/arch/mips/mti-malta/malta-int.c @@ -144,12 +144,6 @@ static irqreturn_t corehi_handler(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction corehi_irqaction = { - .handler = corehi_handler, - .name = "CoreHi", - .flags = IRQF_NO_THREAD, -}; - static msc_irqmap_t msc_irqmap[] __initdata = { {MSC01C_INT_TMR, MSC01_IRQ_EDGE, 0}, {MSC01C_INT_PCI, MSC01_IRQ_LEVEL, 0}, @@ -223,5 +217,7 @@ void __init arch_init_irq(void) corehi_irq = MIPS_CPU_IRQ_BASE + MIPSCPU_INT_COREHI; } - setup_irq(corehi_irq, &corehi_irqaction); + if (request_irq(corehi_irq, corehi_handler, IRQF_NO_THREAD, "CoreHi", + NULL)) + pr_err("Failed to request irq %d (CoreHi)\n", corehi_irq); } diff --git a/arch/mips/netlogic/xlr/fmn.c b/arch/mips/netlogic/xlr/fmn.c index d428e8471eec..d7db1533889a 100644 --- a/arch/mips/netlogic/xlr/fmn.c +++ b/arch/mips/netlogic/xlr/fmn.c @@ -110,12 +110,6 @@ static irqreturn_t fmn_message_handler(int irq, void *data) return IRQ_HANDLED; } -struct irqaction fmn_irqaction = { - .handler = fmn_message_handler, - .flags = IRQF_PERCPU, - .name = "fmn", -}; - void xlr_percpu_fmn_init(void) { struct xlr_fmn_info *cpu_fmn_info; @@ -195,8 +189,9 @@ void nlm_setup_fmn_irq(void) { uint32_t flags; - /* setup irq only once */ - setup_irq(IRQ_FMN, &fmn_irqaction); + /* request irq only once */ + if (request_irq(IRQ_FMN, fmn_message_handler, IRQF_PERCPU, "fmn", NULL)) + pr_err("Failed to request irq %d (fmn)\n", IRQ_FMN); flags = nlm_cop2_enable_irqsave(); nlm_fmn_setup_intr(IRQ_FMN, (1 << nlm_threads_per_core) - 1); diff --git a/arch/mips/pic32/pic32mzda/time.c b/arch/mips/pic32/pic32mzda/time.c index 905ec1d86928..7174e9abbb1b 100644 --- a/arch/mips/pic32/pic32mzda/time.c +++ b/arch/mips/pic32/pic32mzda/time.c @@ -3,11 +3,11 @@ * Joshua Henderson <joshua.henderson@microchip.com> * Copyright (C) 2015 Microchip Technology Inc. All rights reserved. */ -#include <linux/clk-provider.h> #include <linux/clocksource.h> #include <linux/init.h> #include <linux/irqdomain.h> #include <linux/of.h> +#include <linux/of_clk.h> #include <linux/of_irq.h> #include <asm/time.h> @@ -23,7 +23,7 @@ static const struct of_device_id pic32_infra_match[] = { static unsigned int pic32_xlate_core_timer_irq(void) { - static struct device_node *node; + struct device_node *node; unsigned int irq; node = of_find_matching_node(NULL, pic32_infra_match); diff --git a/arch/mips/pistachio/time.c b/arch/mips/pistachio/time.c index 3c9235c7710a..de64751dec40 100644 --- a/arch/mips/pistachio/time.c +++ b/arch/mips/pistachio/time.c @@ -6,10 +6,10 @@ */ #include <linux/clk.h> -#include <linux/clk-provider.h> #include <linux/clocksource.h> #include <linux/init.h> #include <linux/of.h> +#include <linux/of_clk.h> #include <asm/mips-cps.h> #include <asm/time.h> diff --git a/arch/mips/pmcs-msp71xx/msp_irq.c b/arch/mips/pmcs-msp71xx/msp_irq.c index 8d53d7a2ed45..d525cc931d89 100644 --- a/arch/mips/pmcs-msp71xx/msp_irq.c +++ b/arch/mips/pmcs-msp71xx/msp_irq.c @@ -107,18 +107,6 @@ asmlinkage void plat_irq_dispatch(void) do_IRQ(MSP_INT_SW1); } -static struct irqaction cic_cascade_msp = { - .handler = no_action, - .name = "MSP CIC cascade", - .flags = IRQF_NO_THREAD, -}; - -static struct irqaction per_cascade_msp = { - .handler = no_action, - .name = "MSP PER cascade", - .flags = IRQF_NO_THREAD, -}; - void __init arch_init_irq(void) { /* assume we'll be using vectored interrupt mode except in UP mode*/ @@ -142,8 +130,12 @@ void __init arch_init_irq(void) #endif /* CONFIG_MIPS_MT_SMP */ #endif /* CONFIG_MIPS_MT */ /* setup the cascaded interrupts */ - setup_irq(MSP_INT_CIC, &cic_cascade_msp); - setup_irq(MSP_INT_PER, &per_cascade_msp); + if (request_irq(MSP_INT_CIC, no_action, IRQF_NO_THREAD, + "MSP CIC cascade", NULL)) + pr_err("Failed to register MSP CIC cascade interrupt\n"); + if (request_irq(MSP_INT_PER, no_action, IRQF_NO_THREAD, + "MSP PER cascade", NULL)) + pr_err("Failed to register MSP PER cascade interrupt\n"); #else /* @@ -153,7 +145,11 @@ void __init arch_init_irq(void) msp_slp_irq_init(); /* setup the cascaded SLP/PER interrupts */ - setup_irq(MSP_INT_SLP, &cic_cascade_msp); - setup_irq(MSP_INT_PER, &per_cascade_msp); + if (request_irq(MSP_INT_SLP, no_action, IRQF_NO_THREAD, + "MSP CIC cascade", NULL)) + pr_err("Failed to register MSP CIC cascade interrupt\n"); + if (request_irq(MSP_INT_PER, no_action, IRQF_NO_THREAD, + "MSP PER cascade", NULL)) + pr_err("Failed to register MSP PER cascade interrupt\n"); #endif } diff --git a/arch/mips/pmcs-msp71xx/msp_smp.c b/arch/mips/pmcs-msp71xx/msp_smp.c index 8f00d26f2a53..00092e2924ec 100644 --- a/arch/mips/pmcs-msp71xx/msp_smp.c +++ b/arch/mips/pmcs-msp71xx/msp_smp.c @@ -38,21 +38,10 @@ static irqreturn_t ipi_call_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq_resched = { - .handler = ipi_resched_interrupt, - .flags = IRQF_PERCPU, - .name = "IPI_resched" -}; - -static struct irqaction irq_call = { - .handler = ipi_call_interrupt, - .flags = IRQF_PERCPU, - .name = "IPI_call" -}; - -void __init arch_init_ipiirq(int irq, struct irqaction *action) +void __init arch_init_ipiirq(int irq, const char *name, irq_handler_t handler) { - setup_irq(irq, action); + if (request_irq(irq, handler, IRQF_PERCPU, name, NULL)) + pr_err("Failed to request irq %d (%s)\n", irq, name); irq_set_handler(irq, handle_percpu_irq); } @@ -60,7 +49,8 @@ void __init msp_vsmp_int_init(void) { set_vi_handler(MIPS_CPU_IPI_RESCHED_IRQ, ipi_resched_dispatch); set_vi_handler(MIPS_CPU_IPI_CALL_IRQ, ipi_call_dispatch); - arch_init_ipiirq(MIPS_CPU_IPI_RESCHED_IRQ, &irq_resched); - arch_init_ipiirq(MIPS_CPU_IPI_CALL_IRQ, &irq_call); + arch_init_ipiirq(MIPS_CPU_IPI_RESCHED_IRQ, "IPI_resched", + ipi_resched_interrupt); + arch_init_ipiirq(MIPS_CPU_IPI_CALL_IRQ, "IPI_call", ipi_call_interrupt); } #endif /* CONFIG_MIPS_MT_SMP */ diff --git a/arch/mips/pmcs-msp71xx/msp_time.c b/arch/mips/pmcs-msp71xx/msp_time.c index d83de01f00b8..9c629829f447 100644 --- a/arch/mips/pmcs-msp71xx/msp_time.c +++ b/arch/mips/pmcs-msp71xx/msp_time.c @@ -27,7 +27,6 @@ #define get_current_vpe() \ ((read_c0_tcbind() >> TCBIND_CURVPE_SHIFT) & TCBIND_CURVPE) -static struct irqaction timer_vpe1; static int tim_installed; void __init plat_time_init(void) @@ -77,10 +76,13 @@ void __init plat_time_init(void) unsigned int get_c0_compare_int(void) { + unsigned long flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED; + /* MIPS_MT modes may want timer for second VPE */ if ((get_current_vpe()) && !tim_installed) { - memcpy(&timer_vpe1, &c0_compare_irqaction, sizeof(timer_vpe1)); - setup_irq(MSP_INT_VPE1_TIMER, &timer_vpe1); + if (request_irq(MSP_INT_VPE1_TIMER, c0_compare_interrupt, flags, + "timer", c0_compare_interrupt)) + pr_err("Failed to register timer interrupt\n"); tim_installed++; } diff --git a/arch/mips/ralink/Kconfig b/arch/mips/ralink/Kconfig index 94e9ce994494..35c2ebd8f094 100644 --- a/arch/mips/ralink/Kconfig +++ b/arch/mips/ralink/Kconfig @@ -52,6 +52,7 @@ choice select COMMON_CLK select CLKSRC_MIPS_GIC select HAVE_PCI if PCI_MT7621 + select SOC_BUS endchoice choice diff --git a/arch/mips/ralink/cevt-rt3352.c b/arch/mips/ralink/cevt-rt3352.c index 61a08943eb2f..269d4877d120 100644 --- a/arch/mips/ralink/cevt-rt3352.c +++ b/arch/mips/ralink/cevt-rt3352.c @@ -82,12 +82,6 @@ static struct systick_device systick = { }, }; -static struct irqaction systick_irqaction = { - .handler = systick_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .dev_id = &systick.dev, -}; - static int systick_shutdown(struct clock_event_device *evt) { struct systick_device *sdev; @@ -95,7 +89,7 @@ static int systick_shutdown(struct clock_event_device *evt) sdev = container_of(evt, struct systick_device, dev); if (sdev->irq_requested) - free_irq(systick.dev.irq, &systick_irqaction); + free_irq(systick.dev.irq, &systick.dev); sdev->irq_requested = 0; iowrite32(0, systick.membase + SYSTICK_CONFIG); @@ -104,12 +98,17 @@ static int systick_shutdown(struct clock_event_device *evt) static int systick_set_oneshot(struct clock_event_device *evt) { + const char *name = systick.dev.name; struct systick_device *sdev; + int irq = systick.dev.irq; sdev = container_of(evt, struct systick_device, dev); - if (!sdev->irq_requested) - setup_irq(systick.dev.irq, &systick_irqaction); + if (!sdev->irq_requested) { + if (request_irq(irq, systick_interrupt, + IRQF_PERCPU | IRQF_TIMER, name, &systick.dev)) + pr_err("Failed to request irq %d (%s)\n", irq, name); + } sdev->irq_requested = 1; iowrite32(CFG_EXT_STK_EN | CFG_CNT_EN, systick.membase + SYSTICK_CONFIG); @@ -125,7 +124,6 @@ static int __init ralink_systick_init(struct device_node *np) if (!systick.membase) return -ENXIO; - systick_irqaction.name = np->name; systick.dev.name = np->name; clockevents_calc_mult_shift(&systick.dev, SYSTICK_FREQ, 60); systick.dev.max_delta_ns = clockevent_delta2ns(0x7fff, &systick.dev); diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c index 9415be0d57b8..0accb80db709 100644 --- a/arch/mips/ralink/mt7621.c +++ b/arch/mips/ralink/mt7621.c @@ -7,6 +7,8 @@ #include <linux/kernel.h> #include <linux/init.h> +#include <linux/slab.h> +#include <linux/sys_soc.h> #include <asm/mipsregs.h> #include <asm/smp-ops.h> @@ -160,6 +162,33 @@ void __init ralink_of_remap(void) panic("Failed to remap core resources"); } +static void soc_dev_init(struct ralink_soc_info *soc_info, u32 rev) +{ + struct soc_device *soc_dev; + struct soc_device_attribute *soc_dev_attr; + + soc_dev_attr = kzalloc(sizeof(*soc_dev_attr), GFP_KERNEL); + if (!soc_dev_attr) + return; + + soc_dev_attr->soc_id = "mt7621"; + soc_dev_attr->family = "Ralink"; + + if (((rev >> CHIP_REV_VER_SHIFT) & CHIP_REV_VER_MASK) == 1 && + (rev & CHIP_REV_ECO_MASK) == 1) + soc_dev_attr->revision = "E2"; + else + soc_dev_attr->revision = "E1"; + + soc_dev_attr->data = soc_info; + + soc_dev = soc_device_register(soc_dev_attr); + if (IS_ERR(soc_dev)) { + kfree(soc_dev_attr); + return; + } +} + void prom_soc_init(struct ralink_soc_info *soc_info) { void __iomem *sysc = (void __iomem *) KSEG1ADDR(MT7621_SYSC_BASE); @@ -214,6 +243,7 @@ void prom_soc_init(struct ralink_soc_info *soc_info) rt2880_pinmux_data = mt7621_pinmux_data; + soc_dev_init(soc_info, rev); if (!register_cps_smp_ops()) return; diff --git a/arch/mips/ralink/timer-gic.c b/arch/mips/ralink/timer-gic.c index 944fbe0fc741..dcf2a44ac51e 100644 --- a/arch/mips/ralink/timer-gic.c +++ b/arch/mips/ralink/timer-gic.c @@ -8,7 +8,7 @@ #include <linux/init.h> #include <linux/of.h> -#include <linux/clk-provider.h> +#include <linux/of_clk.h> #include <linux/clocksource.h> #include "common.h" diff --git a/arch/mips/sgi-ip22/ip22-eisa.c b/arch/mips/sgi-ip22/ip22-eisa.c index a0a79222ce0b..f3b0e90e0135 100644 --- a/arch/mips/sgi-ip22/ip22-eisa.c +++ b/arch/mips/sgi-ip22/ip22-eisa.c @@ -92,11 +92,6 @@ static irqreturn_t ip22_eisa_intr(int irq, void *dev_id) return IRQ_NONE; } -static struct irqaction eisa_action = { - .handler = ip22_eisa_intr, - .name = "EISA", -}; - int __init ip22_eisa_init(void) { int i, c; @@ -136,9 +131,8 @@ int __init ip22_eisa_init(void) init_i8259_irqs(); - /* Cannot use request_irq because of kmalloc not being ready at such - * an early stage. Yes, I've been bitten... */ - setup_irq(SGI_EISA_IRQ, &eisa_action); + if (request_irq(SGI_EISA_IRQ, ip22_eisa_intr, 0, "EISA", NULL)) + pr_err("Failed to request irq %d (EISA)\n", SGI_EISA_IRQ); EISA_bus = 1; return 0; diff --git a/arch/mips/sgi-ip22/ip22-int.c b/arch/mips/sgi-ip22/ip22-int.c index 3804895fa697..96798a4ab2de 100644 --- a/arch/mips/sgi-ip22/ip22-int.c +++ b/arch/mips/sgi-ip22/ip22-int.c @@ -159,36 +159,7 @@ static void __irq_entry indy_buserror_irq(void) irq_exit(); } -static struct irqaction local0_cascade = { - .handler = no_action, - .flags = IRQF_NO_THREAD, - .name = "local0 cascade", -}; - -static struct irqaction local1_cascade = { - .handler = no_action, - .flags = IRQF_NO_THREAD, - .name = "local1 cascade", -}; - -static struct irqaction buserr = { - .handler = no_action, - .flags = IRQF_NO_THREAD, - .name = "Bus Error", -}; - -static struct irqaction map0_cascade = { - .handler = no_action, - .flags = IRQF_NO_THREAD, - .name = "mapable0 cascade", -}; - #ifdef USE_LIO3_IRQ -static struct irqaction map1_cascade = { - .handler = no_action, - .flags = IRQF_NO_THREAD, - .name = "mapable1 cascade", -}; #define SGI_INTERRUPTS SGINT_END #else #define SGI_INTERRUPTS SGINT_LOCAL3 @@ -322,14 +293,24 @@ void __init arch_init_irq(void) } /* vector handler. this register the IRQ as non-sharable */ - setup_irq(SGI_LOCAL_0_IRQ, &local0_cascade); - setup_irq(SGI_LOCAL_1_IRQ, &local1_cascade); - setup_irq(SGI_BUSERR_IRQ, &buserr); + if (request_irq(SGI_LOCAL_0_IRQ, no_action, IRQF_NO_THREAD, + "local0 cascade", NULL)) + pr_err("Failed to register local0 cascade interrupt\n"); + if (request_irq(SGI_LOCAL_1_IRQ, no_action, IRQF_NO_THREAD, + "local1 cascade", NULL)) + pr_err("Failed to register local1 cascade interrupt\n"); + if (request_irq(SGI_BUSERR_IRQ, no_action, IRQF_NO_THREAD, + "Bus Error", NULL)) + pr_err("Failed to register Bus Error interrupt\n"); /* cascade in cascade. i love Indy ;-) */ - setup_irq(SGI_MAP_0_IRQ, &map0_cascade); + if (request_irq(SGI_MAP_0_IRQ, no_action, IRQF_NO_THREAD, + "mapable0 cascade", NULL)) + pr_err("Failed to register mapable0 cascade interrupt\n"); #ifdef USE_LIO3_IRQ - setup_irq(SGI_MAP_1_IRQ, &map1_cascade); + if (request_irq(SGI_MAP_1_IRQ, no_action, IRQF_NO_THREAD, + "mapable1 cascade", NULL)) + pr_err("Failed to register mapable1 cascade interrupt\n"); #endif #ifdef CONFIG_EISA diff --git a/arch/mips/sgi-ip32/ip32-irq.c b/arch/mips/sgi-ip32/ip32-irq.c index a6a0ff7f5aed..1bbd5bfb5458 100644 --- a/arch/mips/sgi-ip32/ip32-irq.c +++ b/arch/mips/sgi-ip32/ip32-irq.c @@ -111,16 +111,6 @@ static inline void flush_mace_bus(void) extern irqreturn_t crime_memerr_intr(int irq, void *dev_id); extern irqreturn_t crime_cpuerr_intr(int irq, void *dev_id); -static struct irqaction memerr_irq = { - .handler = crime_memerr_intr, - .name = "CRIME memory error", -}; - -static struct irqaction cpuerr_irq = { - .handler = crime_cpuerr_intr, - .name = "CRIME CPU error", -}; - /* * This is for pure CRIME interrupts - ie not MACE. The advantage? * We get to split the register in half and do faster lookups. @@ -497,8 +487,12 @@ void __init arch_init_irq(void) break; } } - setup_irq(CRIME_MEMERR_IRQ, &memerr_irq); - setup_irq(CRIME_CPUERR_IRQ, &cpuerr_irq); + if (request_irq(CRIME_MEMERR_IRQ, crime_memerr_intr, 0, + "CRIME memory error", NULL)) + pr_err("Failed to register CRIME memory error interrupt\n"); + if (request_irq(CRIME_CPUERR_IRQ, crime_cpuerr_intr, 0, + "CRIME CPU error", NULL)) + pr_err("Failed to register CRIME CPU error interrupt\n"); #define ALLINTS (IE_IRQ0 | IE_IRQ1 | IE_IRQ2 | IE_IRQ3 | IE_IRQ4 | IE_IRQ5) change_c0_status(ST0_IM, ALLINTS); diff --git a/arch/mips/sni/a20r.c b/arch/mips/sni/a20r.c index f9407e170476..0ecffb65fd6d 100644 --- a/arch/mips/sni/a20r.c +++ b/arch/mips/sni/a20r.c @@ -222,7 +222,9 @@ void __init sni_a20r_irq_init(void) irq_set_chip_and_handler(i, &a20r_irq_type, handle_level_irq); sni_hwint = a20r_hwint; change_c0_status(ST0_IM, IE_IRQ0); - setup_irq(SNI_A20R_IRQ_BASE + 3, &sni_isa_irq); + if (request_irq(SNI_A20R_IRQ_BASE + 3, sni_isa_irq_handler, 0, "ISA", + NULL)) + pr_err("Failed to register ISA interrupt\n"); } void sni_a20r_init(void) diff --git a/arch/mips/sni/irq.c b/arch/mips/sni/irq.c index ac61b90bcc66..dec89afc9886 100644 --- a/arch/mips/sni/irq.c +++ b/arch/mips/sni/irq.c @@ -27,7 +27,7 @@ asmlinkage void plat_irq_dispatch(void) } /* ISA irq handler */ -static irqreturn_t sni_isa_irq_handler(int dummy, void *p) +irqreturn_t sni_isa_irq_handler(int dummy, void *p) { int irq; @@ -39,12 +39,6 @@ static irqreturn_t sni_isa_irq_handler(int dummy, void *p) return IRQ_HANDLED; } -struct irqaction sni_isa_irq = { - .handler = sni_isa_irq_handler, - .name = "ISA", - .flags = IRQF_SHARED -}; - /* * On systems with i8259-style interrupt controllers we assume for * driver compatibility reasons interrupts 0 - 15 to be the i8295 diff --git a/arch/mips/sni/pcit.c b/arch/mips/sni/pcit.c index 05bb51676e82..b331fe22c529 100644 --- a/arch/mips/sni/pcit.c +++ b/arch/mips/sni/pcit.c @@ -244,7 +244,9 @@ void __init sni_pcit_irq_init(void) *(volatile u32 *)SNI_PCIT_INT_REG = 0; sni_hwint = sni_pcit_hwint; change_c0_status(ST0_IM, IE_IRQ1); - setup_irq(SNI_PCIT_INT_START + 6, &sni_isa_irq); + if (request_irq(SNI_PCIT_INT_START + 6, sni_isa_irq_handler, 0, "ISA", + NULL)) + pr_err("Failed to register ISA interrupt\n"); } void __init sni_pcit_cplus_irq_init(void) @@ -257,7 +259,9 @@ void __init sni_pcit_cplus_irq_init(void) *(volatile u32 *)SNI_PCIT_INT_REG = 0x40000000; sni_hwint = sni_pcit_hwint_cplus; change_c0_status(ST0_IM, IE_IRQ0); - setup_irq(MIPS_CPU_IRQ_BASE + 3, &sni_isa_irq); + if (request_irq(MIPS_CPU_IRQ_BASE + 3, sni_isa_irq_handler, 0, "ISA", + NULL)) + pr_err("Failed to register ISA interrupt\n"); } void __init sni_pcit_init(void) diff --git a/arch/mips/sni/rm200.c b/arch/mips/sni/rm200.c index f6fa9afcbfd3..d84744ca871d 100644 --- a/arch/mips/sni/rm200.c +++ b/arch/mips/sni/rm200.c @@ -356,11 +356,6 @@ void sni_rm200_init_8259A(void) /* * IRQ2 is cascade interrupt to second interrupt controller */ -static struct irqaction sni_rm200_irq2 = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; static struct resource sni_rm200_pic1_resource = { .name = "onboard ISA pic1", @@ -389,12 +384,6 @@ static irqreturn_t sni_rm200_i8259A_irq_handler(int dummy, void *p) return IRQ_HANDLED; } -struct irqaction sni_rm200_i8259A_irq = { - .handler = sni_rm200_i8259A_irq_handler, - .name = "onboard ISA", - .flags = IRQF_SHARED -}; - void __init sni_rm200_i8259_irqs(void) { int i; @@ -417,7 +406,9 @@ void __init sni_rm200_i8259_irqs(void) irq_set_chip_and_handler(i, &sni_rm200_i8259A_chip, handle_level_irq); - setup_irq(RM200_I8259A_IRQ_BASE + PIC_CASCADE_IR, &sni_rm200_irq2); + if (request_irq(RM200_I8259A_IRQ_BASE + PIC_CASCADE_IR, no_action, + IRQF_NO_THREAD, "cascade", NULL)) + pr_err("Failed to register cascade interrupt\n"); } @@ -481,8 +472,12 @@ void __init sni_rm200_irq_init(void) irq_set_chip_and_handler(i, &rm200_irq_type, handle_level_irq); sni_hwint = sni_rm200_hwint; change_c0_status(ST0_IM, IE_IRQ0); - setup_irq(SNI_RM200_INT_START + 0, &sni_rm200_i8259A_irq); - setup_irq(SNI_RM200_INT_START + 1, &sni_isa_irq); + if (request_irq(SNI_RM200_INT_START + 0, sni_rm200_i8259A_irq_handler, + 0, "onboard ISA", NULL)) + pr_err("Failed to register onboard ISA interrupt\n"); + if (request_irq(SNI_RM200_INT_START + 1, sni_isa_irq_handler, 0, "ISA", + NULL)) + pr_err("Failed to register ISA interrupt\n"); } void __init sni_rm200_init(void) diff --git a/arch/mips/sni/time.c b/arch/mips/sni/time.c index dbace1f3e1a9..240bb68ec247 100644 --- a/arch/mips/sni/time.c +++ b/arch/mips/sni/time.c @@ -55,12 +55,6 @@ static irqreturn_t a20r_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction a20r_irqaction = { - .handler = a20r_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "a20r-timer", -}; - /* * a20r platform uses 2 counters to divide the input frequency. * Counter 2 output is connected to Counter 0 & 1 input. @@ -68,13 +62,13 @@ static struct irqaction a20r_irqaction = { static void __init sni_a20r_timer_setup(void) { struct clock_event_device *cd = &a20r_clockevent_device; - struct irqaction *action = &a20r_irqaction; unsigned int cpu = smp_processor_id(); cd->cpumask = cpumask_of(cpu); clockevents_register_device(cd); - action->dev_id = cd; - setup_irq(SNI_A20R_IRQ_TIMER, &a20r_irqaction); + if (request_irq(SNI_A20R_IRQ_TIMER, a20r_interrupt, + IRQF_PERCPU | IRQF_TIMER, "a20r-timer", cd)) + pr_err("Failed to register a20r-timer interrupt\n"); } #define SNI_8254_TICK_RATE 1193182UL diff --git a/arch/mips/vr41xx/common/irq.c b/arch/mips/vr41xx/common/irq.c index 7a826fbf9060..8f68446ff2d9 100644 --- a/arch/mips/vr41xx/common/irq.c +++ b/arch/mips/vr41xx/common/irq.c @@ -17,12 +17,6 @@ typedef struct irq_cascade { static irq_cascade_t irq_cascade[NR_IRQS] __cacheline_aligned; -static struct irqaction cascade_irqaction = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; - int cascade_irq(unsigned int irq, int (*get_irq)(unsigned int)) { int retval = 0; @@ -36,7 +30,8 @@ int cascade_irq(unsigned int irq, int (*get_irq)(unsigned int)) irq_cascade[irq].get_irq = get_irq; if (get_irq != NULL) { - retval = setup_irq(irq, &cascade_irqaction); + retval = request_irq(irq, no_action, IRQF_NO_THREAD, + "cascade", NULL); if (retval < 0) irq_cascade[irq].get_irq = NULL; } diff --git a/arch/nds32/include/asm/futex.h b/arch/nds32/include/asm/futex.h index 5213c65c2e0b..4223f473bd36 100644 --- a/arch/nds32/include/asm/futex.h +++ b/arch/nds32/include/asm/futex.h @@ -66,8 +66,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret; - - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: __futex_atomic_op("move %0, %3", ret, oldval, tmp, uaddr, @@ -93,8 +93,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/nds32/include/asm/uaccess.h b/arch/nds32/include/asm/uaccess.h index 8916ad9f9f13..3a9219f53ee0 100644 --- a/arch/nds32/include/asm/uaccess.h +++ b/arch/nds32/include/asm/uaccess.h @@ -11,7 +11,6 @@ #include <asm/errno.h> #include <asm/memory.h> #include <asm/types.h> -#include <linux/mm.h> #define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" diff --git a/arch/openrisc/include/asm/futex.h b/arch/openrisc/include/asm/futex.h index fe894e6331ae..865e9cd0d97b 100644 --- a/arch/openrisc/include/asm/futex.h +++ b/arch/openrisc/include/asm/futex.h @@ -35,7 +35,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -57,8 +58,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 71034b54d74e..3801a2ef9bca 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -79,6 +79,11 @@ config MMU config STACK_GROWSUP def_bool y +config ARCH_DEFCONFIG + string + default "arch/parisc/configs/generic-32bit_defconfig" if !64BIT + default "arch/parisc/configs/generic-64bit_defconfig" if 64BIT + config GENERIC_LOCKBREAK bool default y diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile index dca8f2de8cf5..628cd8bb7ad8 100644 --- a/arch/parisc/Makefile +++ b/arch/parisc/Makefile @@ -34,6 +34,13 @@ CC_ARCHES = hppa hppa2.0 hppa1.1 LD_BFD := elf32-hppa-linux endif +# select defconfig based on actual architecture +ifeq ($(shell uname -m),parisc64) + KBUILD_DEFCONFIG := generic-64bit_defconfig +else + KBUILD_DEFCONFIG := generic-32bit_defconfig +endif + export LD_BFD ifneq ($(SUBARCH),$(UTS_MACHINE)) diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h index d2c3e4106851..c459f656c8c3 100644 --- a/arch/parisc/include/asm/futex.h +++ b/arch/parisc/include/asm/futex.h @@ -40,7 +40,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) u32 tmp; _futex_spin_lock_irqsave(uaddr, &flags); - pagefault_disable(); ret = -EFAULT; if (unlikely(get_user(oldval, uaddr) != 0)) @@ -73,7 +72,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -EFAULT; out_pagefault_enable: - pagefault_enable(); _futex_spin_unlock_irqrestore(uaddr, &flags); if (!ret) diff --git a/arch/parisc/kernel/processor.c b/arch/parisc/kernel/processor.c index 13f771f74ee3..7f2d0c0ecc80 100644 --- a/arch/parisc/kernel/processor.c +++ b/arch/parisc/kernel/processor.c @@ -212,7 +212,7 @@ static int __init processor_probe(struct parisc_device *dev) #ifdef CONFIG_SMP if (cpuid) { set_cpu_present(cpuid, true); - cpu_up(cpuid); + add_cpu(cpuid); } #endif diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 497b7d0b2d7e..6f40af294685 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -182,6 +182,7 @@ config PPC select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13) select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2) select HAVE_CONTEXT_TRACKING if PPC64 + select HAVE_TIF_NOHZ if PPC64 select HAVE_COPY_THREAD_TLS select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h index bc7d9d06a6d9..f187bb5e524e 100644 --- a/arch/powerpc/include/asm/futex.h +++ b/arch/powerpc/include/asm/futex.h @@ -35,8 +35,9 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, { int oldval = 0, ret; + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; allow_read_write_user(uaddr, uaddr, sizeof(*uaddr)); - pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -58,8 +59,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, ret = -ENOSYS; } - pagefault_enable(); - *oval = oldval; prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr)); diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c index 04a7cba58eff..b4184092172a 100644 --- a/arch/powerpc/kexec/core_64.c +++ b/arch/powerpc/kexec/core_64.c @@ -212,7 +212,7 @@ static void wake_offline_cpus(void) if (!cpu_online(cpu)) { printk(KERN_INFO "kexec: Waking offline cpu %d.\n", cpu); - WARN_ON(cpu_up(cpu)); + WARN_ON(add_cpu(cpu)); } } } diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 3086055bf681..3dcfecf858f3 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -518,6 +518,7 @@ static void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events * } } cpuhw->bhrb_stack.nr = u_index; + cpuhw->bhrb_stack.hw_idx = -1ULL; return; } diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c index 2735ec90414d..e87360a0fb40 100644 --- a/arch/powerpc/platforms/ps3/device-init.c +++ b/arch/powerpc/platforms/ps3/device-init.c @@ -13,6 +13,7 @@ #include <linux/init.h> #include <linux/slab.h> #include <linux/reboot.h> +#include <linux/rcuwait.h> #include <asm/firmware.h> #include <asm/lv1call.h> @@ -670,7 +671,8 @@ struct ps3_notification_device { spinlock_t lock; u64 tag; u64 lv1_status; - struct completion done; + struct rcuwait wait; + bool done; }; enum ps3_notify_type { @@ -712,7 +714,8 @@ static irqreturn_t ps3_notification_interrupt(int irq, void *data) pr_debug("%s:%u: completed, status 0x%llx\n", __func__, __LINE__, status); dev->lv1_status = status; - complete(&dev->done); + dev->done = true; + rcuwait_wake_up(&dev->wait); } spin_unlock(&dev->lock); return IRQ_HANDLED; @@ -725,12 +728,12 @@ static int ps3_notification_read_write(struct ps3_notification_device *dev, unsigned long flags; int res; - init_completion(&dev->done); spin_lock_irqsave(&dev->lock, flags); res = write ? lv1_storage_write(dev->sbd.dev_id, 0, 0, 1, 0, lpar, &dev->tag) : lv1_storage_read(dev->sbd.dev_id, 0, 0, 1, 0, lpar, &dev->tag); + dev->done = false; spin_unlock_irqrestore(&dev->lock, flags); if (res) { pr_err("%s:%u: %s failed %d\n", __func__, __LINE__, op, res); @@ -738,14 +741,10 @@ static int ps3_notification_read_write(struct ps3_notification_device *dev, } pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op); - res = wait_event_interruptible(dev->done.wait, - dev->done.done || kthread_should_stop()); + rcuwait_wait_event(&dev->wait, dev->done || kthread_should_stop(), TASK_IDLE); + if (kthread_should_stop()) res = -EINTR; - if (res) { - pr_debug("%s:%u: interrupted %s\n", __func__, __LINE__, op); - return res; - } if (dev->lv1_status) { pr_err("%s:%u: %s not completed, status 0x%llx\n", __func__, @@ -810,6 +809,7 @@ static int ps3_probe_thread(void *data) } spin_lock_init(&dev.lock); + rcuwait_init(&dev.wait); res = request_irq(irq, ps3_notification_interrupt, 0, "ps3_notification", &dev); diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 1a3b5a5276be..cd5db57bfd41 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -50,7 +50,6 @@ config RISCV select PCI_DOMAINS_GENERIC if PCI select PCI_MSI if PCI select RISCV_TIMER - select UACCESS_MEMCPY if !MMU select GENERIC_IRQ_MULTI_HANDLER select GENERIC_ARCH_TOPOLOGY if SMP select ARCH_HAS_PTE_SPECIAL diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 3078b2de0b2d..a131174a0a77 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -12,20 +12,6 @@ config SOC_SIFIVE config SOC_VIRT bool "QEMU Virt Machine" - select VIRTIO_PCI - select VIRTIO_BALLOON - select VIRTIO_MMIO - select VIRTIO_CONSOLE - select VIRTIO_NET - select NET_9P_VIRTIO - select VIRTIO_BLK - select SCSI_VIRTIO - select DRM_VIRTIO_GPU - select HW_RANDOM_VIRTIO - select RPMSG_CHAR - select RPMSG_VIRTIO - select CRYPTO_DEV_VIRTIO - select VIRTIO_INPUT select POWER_RESET_SYSCON select POWER_RESET_SYSCON_POWEROFF select GOLDFISH diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index c8f084203067..2557c5372a25 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -31,6 +31,7 @@ CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y CONFIG_NETLINK_DIAG=y CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCI_HOST_GENERIC=y @@ -38,12 +39,15 @@ CONFIG_PCIE_XILINX=y CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_BLK_DEV_LOOP=y +CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=y +CONFIG_SCSI_VIRTIO=y CONFIG_ATA=y CONFIG_SATA_AHCI=y CONFIG_SATA_AHCI_PLATFORM=y CONFIG_NETDEVICES=y +CONFIG_VIRTIO_NET=y CONFIG_MACB=y CONFIG_E1000E=y CONFIG_R8169=y @@ -54,13 +58,16 @@ CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_EARLYCON_RISCV_SBI=y CONFIG_HVC_RISCV_SBI=y +CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y CONFIG_SPI=y CONFIG_SPI_SIFIVE=y # CONFIG_PTP_1588_CLOCK is not set CONFIG_POWER_RESET=y CONFIG_DRM=y CONFIG_DRM_RADEON=y +CONFIG_DRM_VIRTIO_GPU=y CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_USB=y CONFIG_USB_XHCI_HCD=y @@ -74,6 +81,12 @@ CONFIG_USB_UAS=y CONFIG_MMC=y CONFIG_MMC_SPI=y CONFIG_RTC_CLASS=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=y +CONFIG_VIRTIO_MMIO=y +CONFIG_RPMSG_CHAR=y +CONFIG_RPMSG_VIRTIO=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_AUTOFS4_FS=y @@ -88,16 +101,17 @@ CONFIG_NFS_V4_2=y CONFIG_ROOT_NFS=y CONFIG_9P_FS=y CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_FS=y CONFIG_DEBUG_PAGEALLOC=y +CONFIG_SCHED_STACK_END_CHECK=y CONFIG_DEBUG_VM=y CONFIG_DEBUG_VM_PGFLAGS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DEBUG_PER_CPU_MAPS=y CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_WQ_WATCHDOG=y -CONFIG_SCHED_STACK_END_CHECK=y CONFIG_DEBUG_TIMEKEEPING=y CONFIG_DEBUG_RT_MUTEXES=y CONFIG_DEBUG_SPINLOCK=y diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index a844920a261f..0292879a9690 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -31,6 +31,7 @@ CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y CONFIG_NETLINK_DIAG=y CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCI_HOST_GENERIC=y @@ -38,12 +39,15 @@ CONFIG_PCIE_XILINX=y CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_BLK_DEV_LOOP=y +CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=y +CONFIG_SCSI_VIRTIO=y CONFIG_ATA=y CONFIG_SATA_AHCI=y CONFIG_SATA_AHCI_PLATFORM=y CONFIG_NETDEVICES=y +CONFIG_VIRTIO_NET=y CONFIG_MACB=y CONFIG_E1000E=y CONFIG_R8169=y @@ -54,11 +58,14 @@ CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_EARLYCON_RISCV_SBI=y CONFIG_HVC_RISCV_SBI=y +CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y # CONFIG_PTP_1588_CLOCK is not set CONFIG_POWER_RESET=y CONFIG_DRM=y CONFIG_DRM_RADEON=y +CONFIG_DRM_VIRTIO_GPU=y CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_USB=y CONFIG_USB_XHCI_HCD=y @@ -70,6 +77,12 @@ CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_STORAGE=y CONFIG_USB_UAS=y CONFIG_RTC_CLASS=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=y +CONFIG_VIRTIO_MMIO=y +CONFIG_RPMSG_CHAR=y +CONFIG_RPMSG_VIRTIO=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_AUTOFS4_FS=y @@ -84,16 +97,17 @@ CONFIG_NFS_V4_2=y CONFIG_ROOT_NFS=y CONFIG_9P_FS=y CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_FS=y CONFIG_DEBUG_PAGEALLOC=y +CONFIG_SCHED_STACK_END_CHECK=y CONFIG_DEBUG_VM=y CONFIG_DEBUG_VM_PGFLAGS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DEBUG_PER_CPU_MAPS=y CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_WQ_WATCHDOG=y -CONFIG_SCHED_STACK_END_CHECK=y CONFIG_DEBUG_TIMEKEEPING=y CONFIG_DEBUG_RT_MUTEXES=y CONFIG_DEBUG_SPINLOCK=y diff --git a/arch/riscv/include/asm/clint.h b/arch/riscv/include/asm/clint.h index 6eaa2eedd694..a279b17a6aad 100644 --- a/arch/riscv/include/asm/clint.h +++ b/arch/riscv/include/asm/clint.h @@ -15,12 +15,12 @@ static inline void clint_send_ipi_single(unsigned long hartid) writel(1, clint_ipi_base + hartid); } -static inline void clint_send_ipi_mask(const struct cpumask *hartid_mask) +static inline void clint_send_ipi_mask(const struct cpumask *mask) { - int hartid; + int cpu; - for_each_cpu(hartid, hartid_mask) - clint_send_ipi_single(hartid); + for_each_cpu(cpu, mask) + clint_send_ipi_single(cpuid_to_hartid_map(cpu)); } static inline void clint_clear_ipi(unsigned long hartid) diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index fdfaf7f3df7c..1b00badb9f87 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -46,7 +46,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, ret = 0; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -73,8 +74,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index e43041519edd..393f2014dfee 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -19,6 +19,47 @@ #include <asm/tlbflush.h> #include <linux/mm_types.h> +#ifdef CONFIG_MMU + +#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) +#define VMALLOC_END (PAGE_OFFSET - 1) +#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) + +#define BPF_JIT_REGION_SIZE (SZ_128M) +#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE) +#define BPF_JIT_REGION_END (VMALLOC_END) + +/* + * Roughly size the vmemmap space to be large enough to fit enough + * struct pages to map half the virtual address space. Then + * position vmemmap directly below the VMALLOC region. + */ +#define VMEMMAP_SHIFT \ + (CONFIG_VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT) +#define VMEMMAP_SIZE BIT(VMEMMAP_SHIFT) +#define VMEMMAP_END (VMALLOC_START - 1) +#define VMEMMAP_START (VMALLOC_START - VMEMMAP_SIZE) + +/* + * Define vmemmap for pfn_to_page & page_to_pfn calls. Needed if kernel + * is configured with CONFIG_SPARSEMEM_VMEMMAP enabled. + */ +#define vmemmap ((struct page *)VMEMMAP_START) + +#define PCI_IO_SIZE SZ_16M +#define PCI_IO_END VMEMMAP_START +#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) + +#define FIXADDR_TOP PCI_IO_START +#ifdef CONFIG_64BIT +#define FIXADDR_SIZE PMD_SIZE +#else +#define FIXADDR_SIZE PGDIR_SIZE +#endif +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) + +#endif + #ifdef CONFIG_64BIT #include <asm/pgtable-64.h> #else @@ -90,31 +131,6 @@ extern pgd_t swapper_pg_dir[]; #define __S110 PAGE_SHARED_EXEC #define __S111 PAGE_SHARED_EXEC -#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) -#define VMALLOC_END (PAGE_OFFSET - 1) -#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) - -#define BPF_JIT_REGION_SIZE (SZ_128M) -#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE) -#define BPF_JIT_REGION_END (VMALLOC_END) - -/* - * Roughly size the vmemmap space to be large enough to fit enough - * struct pages to map half the virtual address space. Then - * position vmemmap directly below the VMALLOC region. - */ -#define VMEMMAP_SHIFT \ - (CONFIG_VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT) -#define VMEMMAP_SIZE BIT(VMEMMAP_SHIFT) -#define VMEMMAP_END (VMALLOC_START - 1) -#define VMEMMAP_START (VMALLOC_START - VMEMMAP_SIZE) - -/* - * Define vmemmap for pfn_to_page & page_to_pfn calls. Needed if kernel - * is configured with CONFIG_SPARSEMEM_VMEMMAP enabled. - */ -#define vmemmap ((struct page *)VMEMMAP_START) - static inline int pmd_present(pmd_t pmd) { return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); @@ -432,18 +448,6 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) -#define PCI_IO_SIZE SZ_16M -#define PCI_IO_END VMEMMAP_START -#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) - -#define FIXADDR_TOP PCI_IO_START -#ifdef CONFIG_64BIT -#define FIXADDR_SIZE PMD_SIZE -#else -#define FIXADDR_SIZE PGDIR_SIZE -#endif -#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) - /* * Task size is 0x4000000000 for RV64 or 0x9fc00000 for RV32. * Note that PGDIR_SIZE must evenly divide TASK_SIZE. diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index f462a183a9c2..8ce9d607b53d 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -11,6 +11,24 @@ /* * User space memory access functions */ + +extern unsigned long __must_check __asm_copy_to_user(void __user *to, + const void *from, unsigned long n); +extern unsigned long __must_check __asm_copy_from_user(void *to, + const void __user *from, unsigned long n); + +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return __asm_copy_from_user(to, from, n); +} + +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return __asm_copy_to_user(to, from, n); +} + #ifdef CONFIG_MMU #include <linux/errno.h> #include <linux/compiler.h> @@ -367,24 +385,6 @@ do { \ -EFAULT; \ }) - -extern unsigned long __must_check __asm_copy_to_user(void __user *to, - const void *from, unsigned long n); -extern unsigned long __must_check __asm_copy_from_user(void *to, - const void __user *from, unsigned long n); - -static inline unsigned long -raw_copy_from_user(void *to, const void __user *from, unsigned long n) -{ - return __asm_copy_from_user(to, from, n); -} - -static inline unsigned long -raw_copy_to_user(void __user *to, const void *from, unsigned long n) -{ - return __asm_copy_to_user(to, from, n); -} - extern long strncpy_from_user(char *dest, const char __user *src, long count); extern long __must_check strlen_user(const char __user *str); diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index eb878abcaaf8..e0a6293093f1 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -96,7 +96,7 @@ static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op) if (IS_ENABLED(CONFIG_RISCV_SBI)) sbi_send_ipi(cpumask_bits(&hartid_mask)); else - clint_send_ipi_mask(&hartid_mask); + clint_send_ipi_mask(mask); } static void send_ipi_single(int cpu, enum ipi_message_type op) diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index ffb3d94bf0cc..55ea614d89bf 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -157,5 +157,5 @@ void __init trap_init(void) /* Set the exception vector address */ csr_write(CSR_TVEC, &handle_exception); /* Enable interrupts */ - csr_write(CSR_IE, IE_SIE | IE_EIE); + csr_write(CSR_IE, IE_SIE); } diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 47e7a8204460..0d0db80800c4 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -2,5 +2,5 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o -lib-$(CONFIG_MMU) += uaccess.o +lib-y += uaccess.o lib-$(CONFIG_64BIT) += tishift.o diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h index 5e97a4353147..26f9144562c9 100644 --- a/arch/s390/include/asm/futex.h +++ b/arch/s390/include/asm/futex.h @@ -29,7 +29,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, mm_segment_t old_fs; old_fs = enable_sacf_uaccess(); - pagefault_disable(); switch (op) { case FUTEX_OP_SET: __futex_atomic_op("lr %2,%5\n", @@ -54,7 +53,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, default: ret = -ENOSYS; } - pagefault_enable(); disable_sacf_uaccess(old_fs); if (!ret) diff --git a/arch/sh/boards/mach-cayman/irq.c b/arch/sh/boards/mach-cayman/irq.c index 3b6ea2d99013..0305d0b51730 100644 --- a/arch/sh/boards/mach-cayman/irq.c +++ b/arch/sh/boards/mach-cayman/irq.c @@ -40,16 +40,6 @@ static irqreturn_t cayman_interrupt_pci2(int irq, void *dev_id) return IRQ_NONE; } -static struct irqaction cayman_action_smsc = { - .name = "Cayman SMSC Mux", - .handler = cayman_interrupt_smsc, -}; - -static struct irqaction cayman_action_pci2 = { - .name = "Cayman PCI2 Mux", - .handler = cayman_interrupt_pci2, -}; - static void enable_cayman_irq(struct irq_data *data) { unsigned int irq = data->irq; @@ -149,6 +139,10 @@ void init_cayman_irq(void) } /* Setup the SMSC interrupt */ - setup_irq(SMSC_IRQ, &cayman_action_smsc); - setup_irq(PCI2_IRQ, &cayman_action_pci2); + if (request_irq(SMSC_IRQ, cayman_interrupt_smsc, 0, "Cayman SMSC Mux", + NULL)) + pr_err("Failed to register Cayman SMSC Mux interrupt\n"); + if (request_irq(PCI2_IRQ, cayman_interrupt_pci2, 0, "Cayman PCI2 Mux", + NULL)) + pr_err("Failed to register Cayman PCI2 Mux interrupt\n"); } diff --git a/arch/sh/drivers/dma/dma-pvr2.c b/arch/sh/drivers/dma/dma-pvr2.c index b5dbd1f75768..21c347543e19 100644 --- a/arch/sh/drivers/dma/dma-pvr2.c +++ b/arch/sh/drivers/dma/dma-pvr2.c @@ -64,11 +64,6 @@ static int pvr2_xfer_dma(struct dma_channel *chan) return 0; } -static struct irqaction pvr2_dma_irq = { - .name = "pvr2 DMA handler", - .handler = pvr2_dma_interrupt, -}; - static struct dma_ops pvr2_dma_ops = { .request = pvr2_request_dma, .get_residue = pvr2_get_dma_residue, @@ -84,7 +79,9 @@ static struct dma_info pvr2_dma_info = { static int __init pvr2_dma_init(void) { - setup_irq(HW_EVENT_PVR2_DMA, &pvr2_dma_irq); + if (request_irq(HW_EVENT_PVR2_DMA, pvr2_dma_interrupt, 0, + "pvr2 DMA handler", NULL)) + pr_err("Failed to register pvr2 DMA handler interrupt\n"); request_dma(PVR2_CASCADE_CHAN, "pvr2 cascade"); return register_dmac(&pvr2_dma_info); diff --git a/arch/sh/include/asm/futex.h b/arch/sh/include/asm/futex.h index 3190ec89df81..b39cda09fb95 100644 --- a/arch/sh/include/asm/futex.h +++ b/arch/sh/include/asm/futex.h @@ -34,8 +34,6 @@ static inline int arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 oldval, newval, prev; int ret; - pagefault_disable(); - do { ret = get_user(oldval, uaddr); @@ -67,8 +65,6 @@ static inline int arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, ret = futex_atomic_cmpxchg_inatomic(&prev, uaddr, oldval, newval); } while (!ret && prev != oldval); - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index c1dd6dd642f4..9cc9ab04bd99 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -71,6 +71,7 @@ config SPARC64 select HAVE_FTRACE_MCOUNT_RECORD select HAVE_SYSCALL_TRACEPOINTS select HAVE_CONTEXT_TRACKING + select HAVE_TIF_NOHZ select HAVE_DEBUG_KMEMLEAK select IOMMU_HELPER select SPARSE_IRQ diff --git a/arch/sparc/include/asm/futex_64.h b/arch/sparc/include/asm/futex_64.h index 0865ce77ec00..72de967318d7 100644 --- a/arch/sparc/include/asm/futex_64.h +++ b/arch/sparc/include/asm/futex_64.h @@ -38,8 +38,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, if (unlikely((((unsigned long) uaddr) & 0x3UL))) return -EINVAL; - pagefault_disable(); - switch (op) { case FUTEX_OP_SET: __futex_cas_op("mov\t%4, %1", ret, oldval, uaddr, oparg); @@ -60,8 +58,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c index bbf59b3b4af8..75232cbd58bf 100644 --- a/arch/sparc/kernel/ds.c +++ b/arch/sparc/kernel/ds.c @@ -555,7 +555,7 @@ static int dr_cpu_configure(struct ds_info *dp, struct ds_cap_state *cp, printk(KERN_INFO "ds-%llu: Starting cpu %d...\n", dp->id, cpu); - err = cpu_up(cpu); + err = add_cpu(cpu); if (err) { __u32 res = DR_CPU_RES_FAILURE; __u32 stat = DR_CPU_STAT_UNCONFIGURED; @@ -611,7 +611,7 @@ static int dr_cpu_unconfigure(struct ds_info *dp, printk(KERN_INFO "ds-%llu: Shutting down cpu %d...\n", dp->id, cpu); - err = cpu_down(cpu); + err = remove_cpu(cpu); if (err) dr_cpu_mark(resp, cpu, ncpus, DR_CPU_RES_FAILURE, diff --git a/arch/unicore32/include/asm/io.h b/arch/unicore32/include/asm/io.h index 3ca74e1cde7d..bd4e7c332f85 100644 --- a/arch/unicore32/include/asm/io.h +++ b/arch/unicore32/include/asm/io.h @@ -27,7 +27,7 @@ extern void __uc32_iounmap(volatile void __iomem *addr); * ioremap and friends. * * ioremap takes a PCI memory address, as specified in - * Documentation/io-mapping.txt. + * Documentation/driver-api/io-mapping.rst. * */ #define ioremap(cookie, size) __uc32_ioremap(cookie, size) diff --git a/arch/unicore32/kernel/time.c b/arch/unicore32/kernel/time.c index 8b217a761bf0..c3a37edf4d40 100644 --- a/arch/unicore32/kernel/time.c +++ b/arch/unicore32/kernel/time.c @@ -72,13 +72,6 @@ static struct clocksource cksrc_puv3_oscr = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static struct irqaction puv3_timer_irq = { - .name = "ost0", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = puv3_ost0_interrupt, - .dev_id = &ckevt_puv3_osmr0, -}; - void __init time_init(void) { writel(0, OST_OIER); /* disable any timer interrupts */ @@ -94,7 +87,9 @@ void __init time_init(void) ckevt_puv3_osmr0.min_delta_ticks = MIN_OSCR_DELTA * 2; ckevt_puv3_osmr0.cpumask = cpumask_of(0); - setup_irq(IRQ_TIMER0, &puv3_timer_irq); + if (request_irq(IRQ_TIMER0, puv3_ost0_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, "ost0", &ckevt_puv3_osmr0)) + pr_err("Failed to register ost0 interrupt\n"); clocksource_register_hz(&cksrc_puv3_oscr, CLOCK_TICK_RATE); clockevents_register_device(&ckevt_puv3_osmr0); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index beea77046f9b..ade80caafe6b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -30,7 +30,6 @@ config X86_64 select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE select SWIOTLB - select ARCH_HAS_SYSCALL_WRAPPER config FORCE_DYNAMIC_FTRACE def_bool y @@ -57,7 +56,6 @@ config X86 select ACPI_LEGACY_TABLES_LOOKUP if ACPI select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI select ARCH_32BIT_OFF_T if X86_32 - select ARCH_CLOCKSOURCE_DATA select ARCH_CLOCKSOURCE_INIT select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_DEBUG_VIRTUAL @@ -80,6 +78,7 @@ config X86 select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_STRICT_MODULE_RWX select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE + select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI @@ -128,6 +127,7 @@ config X86 select GENERIC_GETTIMEOFDAY select GENERIC_VDSO_TIME_NS select GUP_GET_PTE_LOW_HIGH if X86_PAE + select HARDIRQS_SW_RESEND select HARDLOCKUP_CHECK_TIMESTAMP if X86_64 select HAVE_ACPI_APEI if ACPI select HAVE_ACPI_APEI_NMI if ACPI @@ -1875,7 +1875,6 @@ config X86_SMAP config X86_UMIP def_bool y - depends on CPU_SUP_INTEL || CPU_SUP_AMD prompt "User Mode Instruction Prevention" if EXPERT ---help--- User Mode Instruction Prevention (UMIP) is a security feature in @@ -2418,7 +2417,7 @@ config CMDLINE config CMDLINE_OVERRIDE bool "Built-in command line overrides boot loader arguments" - depends on CMDLINE_BOOL + depends on CMDLINE_BOOL && CMDLINE != "" ---help--- Set this option to 'Y' to have the kernel ignore the boot loader command line, and use ONLY the built-in command line. diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 012b82fc8617..e17be90ab312 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -68,6 +68,7 @@ clean-files += cpustr.h KBUILD_CFLAGS := $(REALMODE_CFLAGS) -D_SETUP KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=) +KBUILD_CFLAGS += -fno-asynchronous-unwind-tables GCOV_PROFILE := n UBSAN_SANITIZE := n @@ -88,7 +89,7 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE SETUP_OBJS = $(addprefix $(obj)/,$(setup-y)) -sed-zoffset := -e 's/^\([0-9a-fA-F]*\) [a-zA-Z] \(startup_32\|startup_64\|efi32_stub_entry\|efi64_stub_entry\|efi_pe_entry\|input_data\|kernel_info\|_end\|_ehead\|_text\|z_.*\)$$/\#define ZO_\2 0x\1/p' +sed-zoffset := -e 's/^\([0-9a-fA-F]*\) [a-zA-Z] \(startup_32\|startup_64\|efi32_stub_entry\|efi64_stub_entry\|efi_pe_entry\|efi32_pe_entry\|input_data\|kernel_info\|_end\|_ehead\|_text\|z_.*\)$$/\#define ZO_\2 0x\1/p' quiet_cmd_zoffset = ZOFFSET $@ cmd_zoffset = $(NM) $< | sed -n $(sed-zoffset) > $@ diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 26050ae0b27e..5f7c262bcc99 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -39,6 +39,7 @@ KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) KBUILD_CFLAGS += $(call cc-disable-warning, gnu) KBUILD_CFLAGS += -Wno-pointer-sign KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=) +KBUILD_CFLAGS += -fno-asynchronous-unwind-tables KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ GCOV_PROFILE := n @@ -87,10 +88,7 @@ endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o -$(obj)/eboot.o: KBUILD_CFLAGS += -fshort-wchar -mno-red-zone - -vmlinux-objs-$(CONFIG_EFI_STUB) += $(obj)/eboot.o \ - $(objtree)/drivers/firmware/efi/libstub/lib.a +vmlinux-objs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_thunk_$(BITS).o # The compressed kernel is built with -fPIC/-fPIE so that a boot loader diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c deleted file mode 100644 index 287393d725f0..000000000000 --- a/arch/x86/boot/compressed/eboot.c +++ /dev/null @@ -1,889 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - -/* ----------------------------------------------------------------------- - * - * Copyright 2011 Intel Corporation; author Matt Fleming - * - * ----------------------------------------------------------------------- */ - -#pragma GCC visibility push(hidden) - -#include <linux/efi.h> -#include <linux/pci.h> - -#include <asm/efi.h> -#include <asm/e820/types.h> -#include <asm/setup.h> -#include <asm/desc.h> -#include <asm/boot.h> - -#include "../string.h" -#include "eboot.h" - -static efi_system_table_t *sys_table; -extern const bool efi_is64; - -__pure efi_system_table_t *efi_system_table(void) -{ - return sys_table; -} - -__attribute_const__ bool efi_is_64bit(void) -{ - if (IS_ENABLED(CONFIG_EFI_MIXED)) - return efi_is64; - return IS_ENABLED(CONFIG_X86_64); -} - -static efi_status_t -preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom) -{ - struct pci_setup_rom *rom = NULL; - efi_status_t status; - unsigned long size; - uint64_t romsize; - void *romimage; - - /* - * Some firmware images contain EFI function pointers at the place where - * the romimage and romsize fields are supposed to be. Typically the EFI - * code is mapped at high addresses, translating to an unrealistically - * large romsize. The UEFI spec limits the size of option ROMs to 16 - * MiB so we reject any ROMs over 16 MiB in size to catch this. - */ - romimage = efi_table_attr(pci, romimage); - romsize = efi_table_attr(pci, romsize); - if (!romimage || !romsize || romsize > SZ_16M) - return EFI_INVALID_PARAMETER; - - size = romsize + sizeof(*rom); - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)&rom); - if (status != EFI_SUCCESS) { - efi_printk("Failed to allocate memory for 'rom'\n"); - return status; - } - - memset(rom, 0, sizeof(*rom)); - - rom->data.type = SETUP_PCI; - rom->data.len = size - sizeof(struct setup_data); - rom->data.next = 0; - rom->pcilen = pci->romsize; - *__rom = rom; - - status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16, - PCI_VENDOR_ID, 1, &rom->vendor); - - if (status != EFI_SUCCESS) { - efi_printk("Failed to read rom->vendor\n"); - goto free_struct; - } - - status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16, - PCI_DEVICE_ID, 1, &rom->devid); - - if (status != EFI_SUCCESS) { - efi_printk("Failed to read rom->devid\n"); - goto free_struct; - } - - status = efi_call_proto(pci, get_location, &rom->segment, &rom->bus, - &rom->device, &rom->function); - - if (status != EFI_SUCCESS) - goto free_struct; - - memcpy(rom->romdata, romimage, romsize); - return status; - -free_struct: - efi_bs_call(free_pool, rom); - return status; -} - -/* - * There's no way to return an informative status from this function, - * because any analysis (and printing of error messages) needs to be - * done directly at the EFI function call-site. - * - * For example, EFI_INVALID_PARAMETER could indicate a bug or maybe we - * just didn't find any PCI devices, but there's no way to tell outside - * the context of the call. - */ -static void setup_efi_pci(struct boot_params *params) -{ - efi_status_t status; - void **pci_handle = NULL; - efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID; - unsigned long size = 0; - struct setup_data *data; - efi_handle_t h; - int i; - - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &pci_proto, NULL, &size, pci_handle); - - if (status == EFI_BUFFER_TOO_SMALL) { - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)&pci_handle); - - if (status != EFI_SUCCESS) { - efi_printk("Failed to allocate memory for 'pci_handle'\n"); - return; - } - - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &pci_proto, NULL, &size, pci_handle); - } - - if (status != EFI_SUCCESS) - goto free_handle; - - data = (struct setup_data *)(unsigned long)params->hdr.setup_data; - - while (data && data->next) - data = (struct setup_data *)(unsigned long)data->next; - - for_each_efi_handle(h, pci_handle, size, i) { - efi_pci_io_protocol_t *pci = NULL; - struct pci_setup_rom *rom; - - status = efi_bs_call(handle_protocol, h, &pci_proto, - (void **)&pci); - if (status != EFI_SUCCESS || !pci) - continue; - - status = preserve_pci_rom_image(pci, &rom); - if (status != EFI_SUCCESS) - continue; - - if (data) - data->next = (unsigned long)rom; - else - params->hdr.setup_data = (unsigned long)rom; - - data = (struct setup_data *)rom; - } - -free_handle: - efi_bs_call(free_pool, pci_handle); -} - -static void retrieve_apple_device_properties(struct boot_params *boot_params) -{ - efi_guid_t guid = APPLE_PROPERTIES_PROTOCOL_GUID; - struct setup_data *data, *new; - efi_status_t status; - u32 size = 0; - apple_properties_protocol_t *p; - - status = efi_bs_call(locate_protocol, &guid, NULL, (void **)&p); - if (status != EFI_SUCCESS) - return; - - if (efi_table_attr(p, version) != 0x10000) { - efi_printk("Unsupported properties proto version\n"); - return; - } - - efi_call_proto(p, get_all, NULL, &size); - if (!size) - return; - - do { - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, - size + sizeof(struct setup_data), - (void **)&new); - if (status != EFI_SUCCESS) { - efi_printk("Failed to allocate memory for 'properties'\n"); - return; - } - - status = efi_call_proto(p, get_all, new->data, &size); - - if (status == EFI_BUFFER_TOO_SMALL) - efi_bs_call(free_pool, new); - } while (status == EFI_BUFFER_TOO_SMALL); - - new->type = SETUP_APPLE_PROPERTIES; - new->len = size; - new->next = 0; - - data = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data; - if (!data) { - boot_params->hdr.setup_data = (unsigned long)new; - } else { - while (data->next) - data = (struct setup_data *)(unsigned long)data->next; - data->next = (unsigned long)new; - } -} - -static const efi_char16_t apple[] = L"Apple"; - -static void setup_quirks(struct boot_params *boot_params) -{ - efi_char16_t *fw_vendor = (efi_char16_t *)(unsigned long) - efi_table_attr(efi_system_table(), fw_vendor); - - if (!memcmp(fw_vendor, apple, sizeof(apple))) { - if (IS_ENABLED(CONFIG_APPLE_PROPERTIES)) - retrieve_apple_device_properties(boot_params); - } -} - -/* - * See if we have Universal Graphics Adapter (UGA) protocol - */ -static efi_status_t -setup_uga(struct screen_info *si, efi_guid_t *uga_proto, unsigned long size) -{ - efi_status_t status; - u32 width, height; - void **uga_handle = NULL; - efi_uga_draw_protocol_t *uga = NULL, *first_uga; - efi_handle_t handle; - int i; - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)&uga_handle); - if (status != EFI_SUCCESS) - return status; - - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - uga_proto, NULL, &size, uga_handle); - if (status != EFI_SUCCESS) - goto free_handle; - - height = 0; - width = 0; - - first_uga = NULL; - for_each_efi_handle(handle, uga_handle, size, i) { - efi_guid_t pciio_proto = EFI_PCI_IO_PROTOCOL_GUID; - u32 w, h, depth, refresh; - void *pciio; - - status = efi_bs_call(handle_protocol, handle, uga_proto, - (void **)&uga); - if (status != EFI_SUCCESS) - continue; - - pciio = NULL; - efi_bs_call(handle_protocol, handle, &pciio_proto, &pciio); - - status = efi_call_proto(uga, get_mode, &w, &h, &depth, &refresh); - if (status == EFI_SUCCESS && (!first_uga || pciio)) { - width = w; - height = h; - - /* - * Once we've found a UGA supporting PCIIO, - * don't bother looking any further. - */ - if (pciio) - break; - - first_uga = uga; - } - } - - if (!width && !height) - goto free_handle; - - /* EFI framebuffer */ - si->orig_video_isVGA = VIDEO_TYPE_EFI; - - si->lfb_depth = 32; - si->lfb_width = width; - si->lfb_height = height; - - si->red_size = 8; - si->red_pos = 16; - si->green_size = 8; - si->green_pos = 8; - si->blue_size = 8; - si->blue_pos = 0; - si->rsvd_size = 8; - si->rsvd_pos = 24; - -free_handle: - efi_bs_call(free_pool, uga_handle); - - return status; -} - -void setup_graphics(struct boot_params *boot_params) -{ - efi_guid_t graphics_proto = EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID; - struct screen_info *si; - efi_guid_t uga_proto = EFI_UGA_PROTOCOL_GUID; - efi_status_t status; - unsigned long size; - void **gop_handle = NULL; - void **uga_handle = NULL; - - si = &boot_params->screen_info; - memset(si, 0, sizeof(*si)); - - size = 0; - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &graphics_proto, NULL, &size, gop_handle); - if (status == EFI_BUFFER_TOO_SMALL) - status = efi_setup_gop(si, &graphics_proto, size); - - if (status != EFI_SUCCESS) { - size = 0; - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &uga_proto, NULL, &size, uga_handle); - if (status == EFI_BUFFER_TOO_SMALL) - setup_uga(si, &uga_proto, size); - } -} - -void startup_32(struct boot_params *boot_params); - -void __noreturn efi_stub_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg, - struct boot_params *boot_params); - -/* - * Because the x86 boot code expects to be passed a boot_params we - * need to create one ourselves (usually the bootloader would create - * one for us). - */ -efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg) -{ - struct boot_params *boot_params; - struct apm_bios_info *bi; - struct setup_header *hdr; - efi_loaded_image_t *image; - efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; - int options_size = 0; - efi_status_t status; - char *cmdline_ptr; - unsigned long ramdisk_addr; - unsigned long ramdisk_size; - - sys_table = sys_table_arg; - - /* Check if we were booted by the EFI firmware */ - if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) - return EFI_INVALID_PARAMETER; - - status = efi_bs_call(handle_protocol, handle, &proto, (void *)&image); - if (status != EFI_SUCCESS) { - efi_printk("Failed to get handle for LOADED_IMAGE_PROTOCOL\n"); - return status; - } - - status = efi_low_alloc(0x4000, 1, (unsigned long *)&boot_params); - if (status != EFI_SUCCESS) { - efi_printk("Failed to allocate lowmem for boot params\n"); - return status; - } - - memset(boot_params, 0x0, 0x4000); - - hdr = &boot_params->hdr; - bi = &boot_params->apm_bios_info; - - /* Copy the second sector to boot_params */ - memcpy(&hdr->jump, image->image_base + 512, 512); - - /* - * Fill out some of the header fields ourselves because the - * EFI firmware loader doesn't load the first sector. - */ - hdr->root_flags = 1; - hdr->vid_mode = 0xffff; - hdr->boot_flag = 0xAA55; - - hdr->type_of_loader = 0x21; - - /* Convert unicode cmdline to ascii */ - cmdline_ptr = efi_convert_cmdline(image, &options_size); - if (!cmdline_ptr) - goto fail; - - hdr->cmd_line_ptr = (unsigned long)cmdline_ptr; - /* Fill in upper bits of command line address, NOP on 32 bit */ - boot_params->ext_cmd_line_ptr = (u64)(unsigned long)cmdline_ptr >> 32; - - hdr->ramdisk_image = 0; - hdr->ramdisk_size = 0; - - /* Clear APM BIOS info */ - memset(bi, 0, sizeof(*bi)); - - status = efi_parse_options(cmdline_ptr); - if (status != EFI_SUCCESS) - goto fail2; - - status = handle_cmdline_files(image, - (char *)(unsigned long)hdr->cmd_line_ptr, - "initrd=", hdr->initrd_addr_max, - &ramdisk_addr, &ramdisk_size); - - if (status != EFI_SUCCESS && - hdr->xloadflags & XLF_CAN_BE_LOADED_ABOVE_4G) { - efi_printk("Trying to load files to higher address\n"); - status = handle_cmdline_files(image, - (char *)(unsigned long)hdr->cmd_line_ptr, - "initrd=", -1UL, - &ramdisk_addr, &ramdisk_size); - } - - if (status != EFI_SUCCESS) - goto fail2; - hdr->ramdisk_image = ramdisk_addr & 0xffffffff; - hdr->ramdisk_size = ramdisk_size & 0xffffffff; - boot_params->ext_ramdisk_image = (u64)ramdisk_addr >> 32; - boot_params->ext_ramdisk_size = (u64)ramdisk_size >> 32; - - hdr->code32_start = (u32)(unsigned long)startup_32; - - efi_stub_entry(handle, sys_table, boot_params); - /* not reached */ - -fail2: - efi_free(options_size, hdr->cmd_line_ptr); -fail: - efi_free(0x4000, (unsigned long)boot_params); - - return status; -} - -static void add_e820ext(struct boot_params *params, - struct setup_data *e820ext, u32 nr_entries) -{ - struct setup_data *data; - - e820ext->type = SETUP_E820_EXT; - e820ext->len = nr_entries * sizeof(struct boot_e820_entry); - e820ext->next = 0; - - data = (struct setup_data *)(unsigned long)params->hdr.setup_data; - - while (data && data->next) - data = (struct setup_data *)(unsigned long)data->next; - - if (data) - data->next = (unsigned long)e820ext; - else - params->hdr.setup_data = (unsigned long)e820ext; -} - -static efi_status_t -setup_e820(struct boot_params *params, struct setup_data *e820ext, u32 e820ext_size) -{ - struct boot_e820_entry *entry = params->e820_table; - struct efi_info *efi = ¶ms->efi_info; - struct boot_e820_entry *prev = NULL; - u32 nr_entries; - u32 nr_desc; - int i; - - nr_entries = 0; - nr_desc = efi->efi_memmap_size / efi->efi_memdesc_size; - - for (i = 0; i < nr_desc; i++) { - efi_memory_desc_t *d; - unsigned int e820_type = 0; - unsigned long m = efi->efi_memmap; - -#ifdef CONFIG_X86_64 - m |= (u64)efi->efi_memmap_hi << 32; -#endif - - d = efi_early_memdesc_ptr(m, efi->efi_memdesc_size, i); - switch (d->type) { - case EFI_RESERVED_TYPE: - case EFI_RUNTIME_SERVICES_CODE: - case EFI_RUNTIME_SERVICES_DATA: - case EFI_MEMORY_MAPPED_IO: - case EFI_MEMORY_MAPPED_IO_PORT_SPACE: - case EFI_PAL_CODE: - e820_type = E820_TYPE_RESERVED; - break; - - case EFI_UNUSABLE_MEMORY: - e820_type = E820_TYPE_UNUSABLE; - break; - - case EFI_ACPI_RECLAIM_MEMORY: - e820_type = E820_TYPE_ACPI; - break; - - case EFI_LOADER_CODE: - case EFI_LOADER_DATA: - case EFI_BOOT_SERVICES_CODE: - case EFI_BOOT_SERVICES_DATA: - case EFI_CONVENTIONAL_MEMORY: - if (efi_soft_reserve_enabled() && - (d->attribute & EFI_MEMORY_SP)) - e820_type = E820_TYPE_SOFT_RESERVED; - else - e820_type = E820_TYPE_RAM; - break; - - case EFI_ACPI_MEMORY_NVS: - e820_type = E820_TYPE_NVS; - break; - - case EFI_PERSISTENT_MEMORY: - e820_type = E820_TYPE_PMEM; - break; - - default: - continue; - } - - /* Merge adjacent mappings */ - if (prev && prev->type == e820_type && - (prev->addr + prev->size) == d->phys_addr) { - prev->size += d->num_pages << 12; - continue; - } - - if (nr_entries == ARRAY_SIZE(params->e820_table)) { - u32 need = (nr_desc - i) * sizeof(struct e820_entry) + - sizeof(struct setup_data); - - if (!e820ext || e820ext_size < need) - return EFI_BUFFER_TOO_SMALL; - - /* boot_params map full, switch to e820 extended */ - entry = (struct boot_e820_entry *)e820ext->data; - } - - entry->addr = d->phys_addr; - entry->size = d->num_pages << PAGE_SHIFT; - entry->type = e820_type; - prev = entry++; - nr_entries++; - } - - if (nr_entries > ARRAY_SIZE(params->e820_table)) { - u32 nr_e820ext = nr_entries - ARRAY_SIZE(params->e820_table); - - add_e820ext(params, e820ext, nr_e820ext); - nr_entries -= nr_e820ext; - } - - params->e820_entries = (u8)nr_entries; - - return EFI_SUCCESS; -} - -static efi_status_t alloc_e820ext(u32 nr_desc, struct setup_data **e820ext, - u32 *e820ext_size) -{ - efi_status_t status; - unsigned long size; - - size = sizeof(struct setup_data) + - sizeof(struct e820_entry) * nr_desc; - - if (*e820ext) { - efi_bs_call(free_pool, *e820ext); - *e820ext = NULL; - *e820ext_size = 0; - } - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)e820ext); - if (status == EFI_SUCCESS) - *e820ext_size = size; - - return status; -} - -static efi_status_t allocate_e820(struct boot_params *params, - struct setup_data **e820ext, - u32 *e820ext_size) -{ - unsigned long map_size, desc_size, buff_size; - struct efi_boot_memmap boot_map; - efi_memory_desc_t *map; - efi_status_t status; - __u32 nr_desc; - - boot_map.map = ↦ - boot_map.map_size = &map_size; - boot_map.desc_size = &desc_size; - boot_map.desc_ver = NULL; - boot_map.key_ptr = NULL; - boot_map.buff_size = &buff_size; - - status = efi_get_memory_map(&boot_map); - if (status != EFI_SUCCESS) - return status; - - nr_desc = buff_size / desc_size; - - if (nr_desc > ARRAY_SIZE(params->e820_table)) { - u32 nr_e820ext = nr_desc - ARRAY_SIZE(params->e820_table); - - status = alloc_e820ext(nr_e820ext, e820ext, e820ext_size); - if (status != EFI_SUCCESS) - return status; - } - - return EFI_SUCCESS; -} - -struct exit_boot_struct { - struct boot_params *boot_params; - struct efi_info *efi; -}; - -static efi_status_t exit_boot_func(struct efi_boot_memmap *map, - void *priv) -{ - const char *signature; - struct exit_boot_struct *p = priv; - - signature = efi_is_64bit() ? EFI64_LOADER_SIGNATURE - : EFI32_LOADER_SIGNATURE; - memcpy(&p->efi->efi_loader_signature, signature, sizeof(__u32)); - - p->efi->efi_systab = (unsigned long)efi_system_table(); - p->efi->efi_memdesc_size = *map->desc_size; - p->efi->efi_memdesc_version = *map->desc_ver; - p->efi->efi_memmap = (unsigned long)*map->map; - p->efi->efi_memmap_size = *map->map_size; - -#ifdef CONFIG_X86_64 - p->efi->efi_systab_hi = (unsigned long)efi_system_table() >> 32; - p->efi->efi_memmap_hi = (unsigned long)*map->map >> 32; -#endif - - return EFI_SUCCESS; -} - -static efi_status_t exit_boot(struct boot_params *boot_params, void *handle) -{ - unsigned long map_sz, key, desc_size, buff_size; - efi_memory_desc_t *mem_map; - struct setup_data *e820ext = NULL; - __u32 e820ext_size = 0; - efi_status_t status; - __u32 desc_version; - struct efi_boot_memmap map; - struct exit_boot_struct priv; - - map.map = &mem_map; - map.map_size = &map_sz; - map.desc_size = &desc_size; - map.desc_ver = &desc_version; - map.key_ptr = &key; - map.buff_size = &buff_size; - priv.boot_params = boot_params; - priv.efi = &boot_params->efi_info; - - status = allocate_e820(boot_params, &e820ext, &e820ext_size); - if (status != EFI_SUCCESS) - return status; - - /* Might as well exit boot services now */ - status = efi_exit_boot_services(handle, &map, &priv, exit_boot_func); - if (status != EFI_SUCCESS) - return status; - - /* Historic? */ - boot_params->alt_mem_k = 32 * 1024; - - status = setup_e820(boot_params, e820ext, e820ext_size); - if (status != EFI_SUCCESS) - return status; - - return EFI_SUCCESS; -} - -/* - * On success we return a pointer to a boot_params structure, and NULL - * on failure. - */ -struct boot_params *efi_main(efi_handle_t handle, - efi_system_table_t *sys_table_arg, - struct boot_params *boot_params) -{ - struct desc_ptr *gdt = NULL; - struct setup_header *hdr = &boot_params->hdr; - efi_status_t status; - struct desc_struct *desc; - unsigned long cmdline_paddr; - - sys_table = sys_table_arg; - - /* Check if we were booted by the EFI firmware */ - if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) - goto fail; - - /* - * make_boot_params() may have been called before efi_main(), in which - * case this is the second time we parse the cmdline. This is ok, - * parsing the cmdline multiple times does not have side-effects. - */ - cmdline_paddr = ((u64)hdr->cmd_line_ptr | - ((u64)boot_params->ext_cmd_line_ptr << 32)); - efi_parse_options((char *)cmdline_paddr); - - /* - * If the boot loader gave us a value for secure_boot then we use that, - * otherwise we ask the BIOS. - */ - if (boot_params->secure_boot == efi_secureboot_mode_unset) - boot_params->secure_boot = efi_get_secureboot(); - - /* Ask the firmware to clear memory on unclean shutdown */ - efi_enable_reset_attack_mitigation(); - - efi_random_get_seed(); - - efi_retrieve_tpm2_eventlog(); - - setup_graphics(boot_params); - - setup_efi_pci(boot_params); - - setup_quirks(boot_params); - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, sizeof(*gdt), - (void **)&gdt); - if (status != EFI_SUCCESS) { - efi_printk("Failed to allocate memory for 'gdt' structure\n"); - goto fail; - } - - gdt->size = 0x800; - status = efi_low_alloc(gdt->size, 8, (unsigned long *)&gdt->address); - if (status != EFI_SUCCESS) { - efi_printk("Failed to allocate memory for 'gdt'\n"); - goto fail; - } - - /* - * If the kernel isn't already loaded at the preferred load - * address, relocate it. - */ - if (hdr->pref_address != hdr->code32_start) { - unsigned long bzimage_addr = hdr->code32_start; - status = efi_relocate_kernel(&bzimage_addr, - hdr->init_size, hdr->init_size, - hdr->pref_address, - hdr->kernel_alignment, - LOAD_PHYSICAL_ADDR); - if (status != EFI_SUCCESS) { - efi_printk("efi_relocate_kernel() failed!\n"); - goto fail; - } - - hdr->pref_address = hdr->code32_start; - hdr->code32_start = bzimage_addr; - } - - status = exit_boot(boot_params, handle); - if (status != EFI_SUCCESS) { - efi_printk("exit_boot() failed!\n"); - goto fail; - } - - memset((char *)gdt->address, 0x0, gdt->size); - desc = (struct desc_struct *)gdt->address; - - /* The first GDT is a dummy. */ - desc++; - - if (IS_ENABLED(CONFIG_X86_64)) { - /* __KERNEL32_CS */ - desc->limit0 = 0xffff; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_CODE | SEG_TYPE_EXEC_READ; - desc->s = DESC_TYPE_CODE_DATA; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0xf; - desc->avl = 0; - desc->l = 0; - desc->d = SEG_OP_SIZE_32BIT; - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; - - desc++; - } else { - /* Second entry is unused on 32-bit */ - desc++; - } - - /* __KERNEL_CS */ - desc->limit0 = 0xffff; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_CODE | SEG_TYPE_EXEC_READ; - desc->s = DESC_TYPE_CODE_DATA; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0xf; - desc->avl = 0; - - if (IS_ENABLED(CONFIG_X86_64)) { - desc->l = 1; - desc->d = 0; - } else { - desc->l = 0; - desc->d = SEG_OP_SIZE_32BIT; - } - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; - desc++; - - /* __KERNEL_DS */ - desc->limit0 = 0xffff; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_DATA | SEG_TYPE_READ_WRITE; - desc->s = DESC_TYPE_CODE_DATA; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0xf; - desc->avl = 0; - desc->l = 0; - desc->d = SEG_OP_SIZE_32BIT; - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; - desc++; - - if (IS_ENABLED(CONFIG_X86_64)) { - /* Task segment value */ - desc->limit0 = 0x0000; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_TSS; - desc->s = 0; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0x0; - desc->avl = 0; - desc->l = 0; - desc->d = 0; - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; - desc++; - } - - asm volatile("cli"); - asm volatile ("lgdt %0" : : "m" (*gdt)); - - return boot_params; -fail: - efi_printk("efi_main() failed!\n"); - - for (;;) - asm("hlt"); -} diff --git a/arch/x86/boot/compressed/eboot.h b/arch/x86/boot/compressed/eboot.h deleted file mode 100644 index 99f35343d443..000000000000 --- a/arch/x86/boot/compressed/eboot.h +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef BOOT_COMPRESSED_EBOOT_H -#define BOOT_COMPRESSED_EBOOT_H - -#define SEG_TYPE_DATA (0 << 3) -#define SEG_TYPE_READ_WRITE (1 << 1) -#define SEG_TYPE_CODE (1 << 3) -#define SEG_TYPE_EXEC_READ (1 << 1) -#define SEG_TYPE_TSS ((1 << 3) | (1 << 0)) -#define SEG_OP_SIZE_32BIT (1 << 0) -#define SEG_GRANULARITY_4KB (1 << 0) - -#define DESC_TYPE_CODE_DATA (1 << 0) - -typedef union efi_uga_draw_protocol efi_uga_draw_protocol_t; - -union efi_uga_draw_protocol { - struct { - efi_status_t (__efiapi *get_mode)(efi_uga_draw_protocol_t *, - u32*, u32*, u32*, u32*); - void *set_mode; - void *blt; - }; - struct { - u32 get_mode; - u32 set_mode; - u32 blt; - } mixed_mode; -}; - -#endif /* BOOT_COMPRESSED_EBOOT_H */ diff --git a/arch/x86/boot/compressed/efi_thunk_64.S b/arch/x86/boot/compressed/efi_thunk_64.S index 8fb7f6799c52..2b2049259619 100644 --- a/arch/x86/boot/compressed/efi_thunk_64.S +++ b/arch/x86/boot/compressed/efi_thunk_64.S @@ -54,11 +54,16 @@ SYM_FUNC_START(__efi64_thunk) * Switch to gdt with 32-bit segments. This is the firmware GDT * that was installed when the kernel started executing. This * pointer was saved at the EFI stub entry point in head_64.S. + * + * Pass the saved DS selector to the 32-bit code, and use far return to + * restore the saved CS selector. */ leaq efi32_boot_gdt(%rip), %rax lgdt (%rax) - pushq $__KERNEL_CS + movzwl efi32_boot_ds(%rip), %edx + movzwq efi32_boot_cs(%rip), %rax + pushq %rax leaq efi_enter32(%rip), %rax pushq %rax lretq @@ -73,6 +78,10 @@ SYM_FUNC_START(__efi64_thunk) movl %ebx, %es pop %rbx movl %ebx, %ds + /* Clear out 32-bit selector from FS and GS */ + xorl %ebx, %ebx + movl %ebx, %fs + movl %ebx, %gs /* * Convert 32-bit status code into 64-bit. @@ -92,10 +101,12 @@ SYM_FUNC_END(__efi64_thunk) * The stack should represent the 32-bit calling convention. */ SYM_FUNC_START_LOCAL(efi_enter32) - movl $__KERNEL_DS, %eax - movl %eax, %ds - movl %eax, %es - movl %eax, %ss + /* Load firmware selector into data and stack segment registers */ + movl %edx, %ds + movl %edx, %es + movl %edx, %fs + movl %edx, %gs + movl %edx, %ss /* Reload pgtables */ movl %cr3, %eax @@ -157,6 +168,14 @@ SYM_DATA_START(efi32_boot_gdt) .quad 0 SYM_DATA_END(efi32_boot_gdt) +SYM_DATA_START(efi32_boot_cs) + .word 0 +SYM_DATA_END(efi32_boot_cs) + +SYM_DATA_START(efi32_boot_ds) + .word 0 +SYM_DATA_END(efi32_boot_ds) + SYM_DATA_START(efi_gdt64) .word efi_gdt64_end - efi_gdt64 .long 0 /* Filled out by user */ diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S index 73f17d0544dd..ab3307036ba4 100644 --- a/arch/x86/boot/compressed/head_32.S +++ b/arch/x86/boot/compressed/head_32.S @@ -63,21 +63,7 @@ __HEAD SYM_FUNC_START(startup_32) cld - /* - * Test KEEP_SEGMENTS flag to see if the bootloader is asking - * us to not reload segments - */ - testb $KEEP_SEGMENTS, BP_loadflags(%esi) - jnz 1f - cli - movl $__BOOT_DS, %eax - movl %eax, %ds - movl %eax, %es - movl %eax, %fs - movl %eax, %gs - movl %eax, %ss -1: /* * Calculate the delta between where we were compiled to run @@ -89,32 +75,59 @@ SYM_FUNC_START(startup_32) */ leal (BP_scratch+4)(%esi), %esp call 1f -1: popl %ebp - subl $1b, %ebp +1: popl %edx + subl $1b, %edx + + /* Load new GDT */ + leal gdt(%edx), %eax + movl %eax, 2(%eax) + lgdt (%eax) + + /* Load segment registers with our descriptors */ + movl $__BOOT_DS, %eax + movl %eax, %ds + movl %eax, %es + movl %eax, %fs + movl %eax, %gs + movl %eax, %ss /* - * %ebp contains the address we are loaded at by the boot loader and %ebx + * %edx contains the address we are loaded at by the boot loader and %ebx * contains the address where we should move the kernel image temporarily - * for safe in-place decompression. + * for safe in-place decompression. %ebp contains the address that the kernel + * will be decompressed to. */ #ifdef CONFIG_RELOCATABLE - movl %ebp, %ebx + movl %edx, %ebx + +#ifdef CONFIG_EFI_STUB +/* + * If we were loaded via the EFI LoadImage service, startup_32() will be at an + * offset to the start of the space allocated for the image. efi_pe_entry() will + * set up image_offset to tell us where the image actually starts, so that we + * can use the full available buffer. + * image_offset = startup_32 - image_base + * Otherwise image_offset will be zero and has no effect on the calculations. + */ + subl image_offset(%edx), %ebx +#endif + movl BP_kernel_alignment(%esi), %eax decl %eax addl %eax, %ebx notl %eax andl %eax, %ebx cmpl $LOAD_PHYSICAL_ADDR, %ebx - jge 1f + jae 1f #endif movl $LOAD_PHYSICAL_ADDR, %ebx 1: + movl %ebx, %ebp // Save the output address for later /* Target address to relocate to for decompression */ - movl BP_init_size(%esi), %eax - subl $_end, %eax - addl %eax, %ebx + addl BP_init_size(%esi), %ebx + subl $_end, %ebx /* Set up the stack */ leal boot_stack_end(%ebx), %esp @@ -128,7 +141,7 @@ SYM_FUNC_START(startup_32) * where decompression in place becomes safe. */ pushl %esi - leal (_bss-4)(%ebp), %esi + leal (_bss-4)(%edx), %esi leal (_bss-4)(%ebx), %edi movl $(_bss - startup_32), %ecx shrl $2, %ecx @@ -137,6 +150,15 @@ SYM_FUNC_START(startup_32) cld popl %esi + /* + * The GDT may get overwritten either during the copy we just did or + * during extract_kernel below. To avoid any issues, repoint the GDTR + * to the new copy of the GDT. + */ + leal gdt(%ebx), %eax + movl %eax, 2(%eax) + lgdt (%eax) + /* * Jump to the relocated address. */ @@ -148,9 +170,8 @@ SYM_FUNC_END(startup_32) SYM_FUNC_START(efi32_stub_entry) SYM_FUNC_START_ALIAS(efi_stub_entry) add $0x4, %esp + movl 8(%esp), %esi /* save boot_params pointer */ call efi_main - movl %eax, %esi - movl BP_code32_start(%esi), %eax leal startup_32(%eax), %eax jmp *%eax SYM_FUNC_END(efi32_stub_entry) @@ -189,9 +210,7 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated) /* push arguments for extract_kernel: */ pushl $z_output_len /* decompressed length, end of relocs */ - leal _end(%ebx), %eax - subl BP_init_size(%esi), %eax - pushl %eax /* output address */ + pushl %ebp /* output address */ pushl $z_input_len /* input_len */ leal input_data(%ebx), %eax @@ -209,6 +228,21 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated) jmp *%eax SYM_FUNC_END(.Lrelocated) + .data + .balign 8 +SYM_DATA_START_LOCAL(gdt) + .word gdt_end - gdt - 1 + .long 0 + .word 0 + .quad 0x0000000000000000 /* Reserved */ + .quad 0x00cf9a000000ffff /* __KERNEL_CS */ + .quad 0x00cf92000000ffff /* __KERNEL_DS */ +SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt_end) + +#ifdef CONFIG_EFI_STUB +SYM_DATA(image_offset, .long 0) +#endif + /* * Stack and heap for uncompression */ diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 1f1f6c8139b3..4f7e6b84be07 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -53,19 +53,7 @@ SYM_FUNC_START(startup_32) * all need to be under the 4G limit. */ cld - /* - * Test KEEP_SEGMENTS flag to see if the bootloader is asking - * us to not reload segments - */ - testb $KEEP_SEGMENTS, BP_loadflags(%esi) - jnz 1f - cli - movl $(__BOOT_DS), %eax - movl %eax, %ds - movl %eax, %es - movl %eax, %ss -1: /* * Calculate the delta between where we were compiled to run @@ -80,10 +68,21 @@ SYM_FUNC_START(startup_32) 1: popl %ebp subl $1b, %ebp + /* Load new GDT with the 64bit segments using 32bit descriptor */ + leal gdt(%ebp), %eax + movl %eax, 2(%eax) + lgdt (%eax) + + /* Load segment registers with our descriptors */ + movl $__BOOT_DS, %eax + movl %eax, %ds + movl %eax, %es + movl %eax, %fs + movl %eax, %gs + movl %eax, %ss + /* setup a stack and make sure cpu supports long mode. */ - movl $boot_stack_end, %eax - addl %ebp, %eax - movl %eax, %esp + leal boot_stack_end(%ebp), %esp call verify_cpu testl %eax, %eax @@ -100,30 +99,38 @@ SYM_FUNC_START(startup_32) #ifdef CONFIG_RELOCATABLE movl %ebp, %ebx + +#ifdef CONFIG_EFI_STUB +/* + * If we were loaded via the EFI LoadImage service, startup_32 will be at an + * offset to the start of the space allocated for the image. efi_pe_entry will + * set up image_offset to tell us where the image actually starts, so that we + * can use the full available buffer. + * image_offset = startup_32 - image_base + * Otherwise image_offset will be zero and has no effect on the calculations. + */ + subl image_offset(%ebp), %ebx +#endif + movl BP_kernel_alignment(%esi), %eax decl %eax addl %eax, %ebx notl %eax andl %eax, %ebx cmpl $LOAD_PHYSICAL_ADDR, %ebx - jge 1f + jae 1f #endif movl $LOAD_PHYSICAL_ADDR, %ebx 1: /* Target address to relocate to for decompression */ - movl BP_init_size(%esi), %eax - subl $_end, %eax - addl %eax, %ebx + addl BP_init_size(%esi), %ebx + subl $_end, %ebx /* * Prepare for entering 64 bit mode */ - /* Load new GDT with the 64bit segments using 32bit descriptor */ - addl %ebp, gdt+2(%ebp) - lgdt gdt(%ebp) - /* Enable PAE mode */ movl %cr4, %eax orl $X86_CR4_PAE, %eax @@ -212,8 +219,13 @@ SYM_FUNC_START(startup_32) cmp $0, %edi jz 1f leal efi64_stub_entry(%ebp), %eax - movl %esi, %edx movl efi32_boot_args+4(%ebp), %esi + movl efi32_boot_args+8(%ebp), %edx // saved bootparams pointer + cmpl $0, %edx + jnz 1f + leal efi_pe_entry(%ebp), %eax + movl %edi, %ecx // MS calling convention + movl %esi, %edx 1: #endif pushl %eax @@ -238,11 +250,17 @@ SYM_FUNC_START(efi32_stub_entry) 1: pop %ebp subl $1b, %ebp + movl %esi, efi32_boot_args+8(%ebp) +SYM_INNER_LABEL(efi32_pe_stub_entry, SYM_L_LOCAL) movl %ecx, efi32_boot_args(%ebp) movl %edx, efi32_boot_args+4(%ebp) - sgdtl efi32_boot_gdt(%ebp) movb $0, efi_is64(%ebp) + /* Save firmware GDTR and code/data selectors */ + sgdtl efi32_boot_gdt(%ebp) + movw %cs, efi32_boot_cs(%ebp) + movw %ds, efi32_boot_ds(%ebp) + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -266,6 +284,9 @@ SYM_CODE_START(startup_64) * and command line. */ + cld + cli + /* Setup data segments. */ xorl %eax, %eax movl %eax, %ds @@ -290,13 +311,27 @@ SYM_CODE_START(startup_64) /* Start with the delta to where the kernel will run at. */ #ifdef CONFIG_RELOCATABLE leaq startup_32(%rip) /* - $startup_32 */, %rbp + +#ifdef CONFIG_EFI_STUB +/* + * If we were loaded via the EFI LoadImage service, startup_32 will be at an + * offset to the start of the space allocated for the image. efi_pe_entry will + * set up image_offset to tell us where the image actually starts, so that we + * can use the full available buffer. + * image_offset = startup_32 - image_base + * Otherwise image_offset will be zero and has no effect on the calculations. + */ + movl image_offset(%rip), %eax + subq %rax, %rbp +#endif + movl BP_kernel_alignment(%rsi), %eax decl %eax addq %rax, %rbp notq %rax andq %rax, %rbp cmpq $LOAD_PHYSICAL_ADDR, %rbp - jge 1f + jae 1f #endif movq $LOAD_PHYSICAL_ADDR, %rbp 1: @@ -354,9 +389,9 @@ SYM_CODE_START(startup_64) */ /* Make sure we have GDT with 32-bit code segment */ - leaq gdt(%rip), %rax - movq %rax, gdt64+2(%rip) - lgdt gdt64(%rip) + leaq gdt64(%rip), %rax + addq %rax, 2(%rax) + lgdt (%rax) /* * paging_prepare() sets up the trampoline and checks if we need to @@ -441,6 +476,16 @@ trampoline_return: cld popq %rsi + /* + * The GDT may get overwritten either during the copy we just did or + * during extract_kernel below. To avoid any issues, repoint the GDTR + * to the new copy of the GDT. + */ + leaq gdt64(%rbx), %rax + leaq gdt(%rbx), %rdx + movq %rdx, 2(%rax) + lgdt (%rax) + /* * Jump to the relocated address. */ @@ -453,9 +498,9 @@ SYM_CODE_END(startup_64) SYM_FUNC_START(efi64_stub_entry) SYM_FUNC_START_ALIAS(efi_stub_entry) and $~0xf, %rsp /* realign the stack */ + movq %rdx, %rbx /* save boot_params pointer */ call efi_main - movq %rax,%rsi - movl BP_code32_start(%esi), %eax + movq %rbx,%rsi leaq startup_64(%rax), %rax jmp *%rax SYM_FUNC_END(efi64_stub_entry) @@ -484,7 +529,7 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated) leaq input_data(%rip), %rdx /* input_data */ movl $z_input_len, %ecx /* input_len */ movq %rbp, %r8 /* output target address */ - movq $z_output_len, %r9 /* decompressed length, end of relocs */ + movl $z_output_len, %r9d /* decompressed length, end of relocs */ call extract_kernel /* returns kernel location in %rax */ popq %rsi @@ -613,13 +658,13 @@ SYM_FUNC_END(.Lno_longmode) .data SYM_DATA_START_LOCAL(gdt64) - .word gdt_end - gdt - .quad 0 + .word gdt_end - gdt - 1 + .quad gdt - gdt64 SYM_DATA_END(gdt64) .balign 8 SYM_DATA_START_LOCAL(gdt) - .word gdt_end - gdt - .long gdt + .word gdt_end - gdt - 1 + .long 0 .word 0 .quad 0x00cf9a000000ffff /* __KERNEL32_CS */ .quad 0x00af9a000000ffff /* __KERNEL_CS */ @@ -628,9 +673,97 @@ SYM_DATA_START_LOCAL(gdt) .quad 0x0000000000000000 /* TS continued */ SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt_end) +#ifdef CONFIG_EFI_STUB +SYM_DATA(image_offset, .long 0) +#endif + #ifdef CONFIG_EFI_MIXED -SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0) +SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0) SYM_DATA(efi_is64, .byte 1) + +#define ST32_boottime 60 // offsetof(efi_system_table_32_t, boottime) +#define BS32_handle_protocol 88 // offsetof(efi_boot_services_32_t, handle_protocol) +#define LI32_image_base 32 // offsetof(efi_loaded_image_32_t, image_base) + + .text + .code32 +SYM_FUNC_START(efi32_pe_entry) +/* + * efi_status_t efi32_pe_entry(efi_handle_t image_handle, + * efi_system_table_32_t *sys_table) + */ + + pushl %ebp + movl %esp, %ebp + pushl %eax // dummy push to allocate loaded_image + + pushl %ebx // save callee-save registers + pushl %edi + + call verify_cpu // check for long mode support + testl %eax, %eax + movl $0x80000003, %eax // EFI_UNSUPPORTED + jnz 2f + + call 1f +1: pop %ebx + subl $1b, %ebx + + /* Get the loaded image protocol pointer from the image handle */ + leal -4(%ebp), %eax + pushl %eax // &loaded_image + leal loaded_image_proto(%ebx), %eax + pushl %eax // pass the GUID address + pushl 8(%ebp) // pass the image handle + + /* + * Note the alignment of the stack frame. + * sys_table + * handle <-- 16-byte aligned on entry by ABI + * return address + * frame pointer + * loaded_image <-- local variable + * saved %ebx <-- 16-byte aligned here + * saved %edi + * &loaded_image + * &loaded_image_proto + * handle <-- 16-byte aligned for call to handle_protocol + */ + + movl 12(%ebp), %eax // sys_table + movl ST32_boottime(%eax), %eax // sys_table->boottime + call *BS32_handle_protocol(%eax) // sys_table->boottime->handle_protocol + addl $12, %esp // restore argument space + testl %eax, %eax + jnz 2f + + movl 8(%ebp), %ecx // image_handle + movl 12(%ebp), %edx // sys_table + movl -4(%ebp), %esi // loaded_image + movl LI32_image_base(%esi), %esi // loaded_image->image_base + movl %ebx, %ebp // startup_32 for efi32_pe_stub_entry + /* + * We need to set the image_offset variable here since startup_32() will + * use it before we get to the 64-bit efi_pe_entry() in C code. + */ + subl %esi, %ebx + movl %ebx, image_offset(%ebp) // save image_offset + jmp efi32_pe_stub_entry + +2: popl %edi // restore callee-save registers + popl %ebx + leave + ret +SYM_FUNC_END(efi32_pe_entry) + + .section ".rodata" + /* EFI loaded image protocol GUID */ + .balign 4 +SYM_DATA_START_LOCAL(loaded_image_proto) + .long 0x5b1b31a1 + .word 0x9562, 0x11d2 + .byte 0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b +SYM_DATA_END(loaded_image_proto) #endif /* @@ -647,7 +780,7 @@ SYM_DATA_END_LABEL(boot_stack, SYM_L_LOCAL, boot_stack_end) /* * Space for page tables (not in .bss so not zeroed) */ - .section ".pgtable","a",@nobits + .section ".pgtable","aw",@nobits .balign 4096 SYM_DATA_LOCAL(pgtable, .fill BOOT_PGT_SIZE, 1, 0) diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index c8181392f70d..726e264410ff 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -59,7 +59,7 @@ void __puthex(unsigned long value); static inline void debug_putstr(const char *s) { } -static inline void debug_puthex(const char *s) +static inline void debug_puthex(unsigned long value) { } #define debug_putaddr(x) /* */ diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 97d9b6d6c1af..735ad7f21ab0 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -15,7 +15,7 @@ * hex while segment addresses are written as segment:offset. * */ - +#include <linux/pe.h> #include <asm/segment.h> #include <asm/boot.h> #include <asm/page_types.h> @@ -43,8 +43,7 @@ SYSSEG = 0x1000 /* historical load address >> 4 */ bootsect_start: #ifdef CONFIG_EFI_STUB # "MZ", MS-DOS header - .byte 0x4d - .byte 0x5a + .word MZ_MAGIC #endif # Normalize the start address @@ -97,39 +96,30 @@ bugger_off_msg: #ifdef CONFIG_EFI_STUB pe_header: - .ascii "PE" - .word 0 + .long PE_MAGIC coff_header: #ifdef CONFIG_X86_32 - .word 0x14c # i386 + .set image_file_add_flags, IMAGE_FILE_32BIT_MACHINE + .set pe_opt_magic, PE_OPT_MAGIC_PE32 + .word IMAGE_FILE_MACHINE_I386 #else - .word 0x8664 # x86-64 + .set image_file_add_flags, 0 + .set pe_opt_magic, PE_OPT_MAGIC_PE32PLUS + .word IMAGE_FILE_MACHINE_AMD64 #endif - .word 4 # nr_sections + .word section_count # nr_sections .long 0 # TimeDateStamp .long 0 # PointerToSymbolTable .long 1 # NumberOfSymbols .word section_table - optional_header # SizeOfOptionalHeader -#ifdef CONFIG_X86_32 - .word 0x306 # Characteristics. - # IMAGE_FILE_32BIT_MACHINE | - # IMAGE_FILE_DEBUG_STRIPPED | - # IMAGE_FILE_EXECUTABLE_IMAGE | - # IMAGE_FILE_LINE_NUMS_STRIPPED -#else - .word 0x206 # Characteristics - # IMAGE_FILE_DEBUG_STRIPPED | - # IMAGE_FILE_EXECUTABLE_IMAGE | - # IMAGE_FILE_LINE_NUMS_STRIPPED -#endif + .word IMAGE_FILE_EXECUTABLE_IMAGE | \ + image_file_add_flags | \ + IMAGE_FILE_DEBUG_STRIPPED | \ + IMAGE_FILE_LINE_NUMS_STRIPPED # Characteristics optional_header: -#ifdef CONFIG_X86_32 - .word 0x10b # PE32 format -#else - .word 0x20b # PE32+ format -#endif + .word pe_opt_magic .byte 0x02 # MajorLinkerVersion .byte 0x14 # MinorLinkerVersion @@ -148,17 +138,19 @@ optional_header: #endif extra_header_fields: + # PE specification requires ImageBase to be 64k aligned + .set image_base, (LOAD_PHYSICAL_ADDR + 0xffff) & ~0xffff #ifdef CONFIG_X86_32 - .long 0 # ImageBase + .long image_base # ImageBase #else - .quad 0 # ImageBase + .quad image_base # ImageBase #endif .long 0x20 # SectionAlignment .long 0x20 # FileAlignment .word 0 # MajorOperatingSystemVersion .word 0 # MinorOperatingSystemVersion - .word 0 # MajorImageVersion - .word 0 # MinorImageVersion + .word LINUX_EFISTUB_MAJOR_VERSION # MajorImageVersion + .word LINUX_EFISTUB_MINOR_VERSION # MinorImageVersion .word 0 # MajorSubsystemVersion .word 0 # MinorSubsystemVersion .long 0 # Win32VersionValue @@ -170,7 +162,7 @@ extra_header_fields: .long 0x200 # SizeOfHeaders .long 0 # CheckSum - .word 0xa # Subsystem (EFI application) + .word IMAGE_SUBSYSTEM_EFI_APPLICATION # Subsystem (EFI application) .word 0 # DllCharacteristics #ifdef CONFIG_X86_32 .long 0 # SizeOfStackReserve @@ -184,7 +176,7 @@ extra_header_fields: .quad 0 # SizeOfHeapCommit #endif .long 0 # LoaderFlags - .long 0x6 # NumberOfRvaAndSizes + .long (section_table - .) / 8 # NumberOfRvaAndSizes .quad 0 # ExportTable .quad 0 # ImportTable @@ -210,7 +202,10 @@ section_table: .long 0 # PointerToLineNumbers .word 0 # NumberOfRelocations .word 0 # NumberOfLineNumbers - .long 0x60500020 # Characteristics (section flags) + .long IMAGE_SCN_CNT_CODE | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_EXECUTE | \ + IMAGE_SCN_ALIGN_16BYTES # Characteristics # # The EFI application loader requires a relocation section @@ -228,45 +223,53 @@ section_table: .long 0 # PointerToLineNumbers .word 0 # NumberOfRelocations .word 0 # NumberOfLineNumbers - .long 0x42100040 # Characteristics (section flags) + .long IMAGE_SCN_CNT_INITIALIZED_DATA | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_DISCARDABLE | \ + IMAGE_SCN_ALIGN_1BYTES # Characteristics +#ifdef CONFIG_EFI_MIXED # # The offset & size fields are filled in by build.c. # - .ascii ".text" - .byte 0 - .byte 0 - .byte 0 + .asciz ".compat" .long 0 - .long 0x0 # startup_{32,64} + .long 0x0 .long 0 # Size of initialized data # on disk - .long 0x0 # startup_{32,64} + .long 0x0 .long 0 # PointerToRelocations .long 0 # PointerToLineNumbers .word 0 # NumberOfRelocations .word 0 # NumberOfLineNumbers - .long 0x60500020 # Characteristics (section flags) + .long IMAGE_SCN_CNT_INITIALIZED_DATA | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_DISCARDABLE | \ + IMAGE_SCN_ALIGN_1BYTES # Characteristics +#endif # # The offset & size fields are filled in by build.c. # - .ascii ".bss" - .byte 0 + .ascii ".text" .byte 0 .byte 0 .byte 0 .long 0 - .long 0x0 + .long 0x0 # startup_{32,64} .long 0 # Size of initialized data # on disk - .long 0x0 + .long 0x0 # startup_{32,64} .long 0 # PointerToRelocations .long 0 # PointerToLineNumbers .word 0 # NumberOfRelocations .word 0 # NumberOfLineNumbers - .long 0xc8000080 # Characteristics (section flags) + .long IMAGE_SCN_CNT_CODE | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_EXECUTE | \ + IMAGE_SCN_ALIGN_16BYTES # Characteristics + .set section_count, (. - section_table) / 40 #endif /* CONFIG_EFI_STUB */ # Kernel attributes; used by setup. This is part 1 of the diff --git a/arch/x86/boot/setup.ld b/arch/x86/boot/setup.ld index 3da1c37c6dd5..24c95522f231 100644 --- a/arch/x86/boot/setup.ld +++ b/arch/x86/boot/setup.ld @@ -52,7 +52,6 @@ SECTIONS _end = .; /DISCARD/ : { - *(.eh_frame) *(.note*) } diff --git a/arch/x86/boot/tools/build.c b/arch/x86/boot/tools/build.c index 55e669d29e54..8f8c8e386cea 100644 --- a/arch/x86/boot/tools/build.c +++ b/arch/x86/boot/tools/build.c @@ -53,11 +53,20 @@ u8 buf[SETUP_SECT_MAX*512]; #define PECOFF_RELOC_RESERVE 0x20 +#ifdef CONFIG_EFI_MIXED +#define PECOFF_COMPAT_RESERVE 0x20 +#else +#define PECOFF_COMPAT_RESERVE 0x0 +#endif + unsigned long efi32_stub_entry; unsigned long efi64_stub_entry; unsigned long efi_pe_entry; +unsigned long efi32_pe_entry; unsigned long kernel_info; unsigned long startup_64; +unsigned long _ehead; +unsigned long _end; /*----------------------------------------------------------------------*/ @@ -189,7 +198,10 @@ static void update_pecoff_section_header(char *section_name, u32 offset, u32 siz static void update_pecoff_setup_and_reloc(unsigned int size) { u32 setup_offset = 0x200; - u32 reloc_offset = size - PECOFF_RELOC_RESERVE; + u32 reloc_offset = size - PECOFF_RELOC_RESERVE - PECOFF_COMPAT_RESERVE; +#ifdef CONFIG_EFI_MIXED + u32 compat_offset = reloc_offset + PECOFF_RELOC_RESERVE; +#endif u32 setup_size = reloc_offset - setup_offset; update_pecoff_section_header(".setup", setup_offset, setup_size); @@ -201,43 +213,59 @@ static void update_pecoff_setup_and_reloc(unsigned int size) */ put_unaligned_le32(reloc_offset + 10, &buf[reloc_offset]); put_unaligned_le32(10, &buf[reloc_offset + 4]); + +#ifdef CONFIG_EFI_MIXED + update_pecoff_section_header(".compat", compat_offset, PECOFF_COMPAT_RESERVE); + + /* + * Put the IA-32 machine type (0x14c) and the associated entry point + * address in the .compat section, so loaders can figure out which other + * execution modes this image supports. + */ + buf[compat_offset] = 0x1; + buf[compat_offset + 1] = 0x8; + put_unaligned_le16(0x14c, &buf[compat_offset + 2]); + put_unaligned_le32(efi32_pe_entry + size, &buf[compat_offset + 4]); +#endif } -static void update_pecoff_text(unsigned int text_start, unsigned int file_sz) +static void update_pecoff_text(unsigned int text_start, unsigned int file_sz, + unsigned int init_sz) { unsigned int pe_header; unsigned int text_sz = file_sz - text_start; + unsigned int bss_sz = init_sz - file_sz; pe_header = get_unaligned_le32(&buf[0x3c]); /* + * The PE/COFF loader may load the image at an address which is + * misaligned with respect to the kernel_alignment field in the setup + * header. + * + * In order to avoid relocating the kernel to correct the misalignment, + * add slack to allow the buffer to be aligned within the declared size + * of the image. + */ + bss_sz += CONFIG_PHYSICAL_ALIGN; + init_sz += CONFIG_PHYSICAL_ALIGN; + + /* * Size of code: Subtract the size of the first sector (512 bytes) * which includes the header. */ - put_unaligned_le32(file_sz - 512, &buf[pe_header + 0x1c]); + put_unaligned_le32(file_sz - 512 + bss_sz, &buf[pe_header + 0x1c]); + + /* Size of image */ + put_unaligned_le32(init_sz, &buf[pe_header + 0x50]); /* * Address of entry point for PE/COFF executable */ put_unaligned_le32(text_start + efi_pe_entry, &buf[pe_header + 0x28]); - update_pecoff_section_header(".text", text_start, text_sz); -} - -static void update_pecoff_bss(unsigned int file_sz, unsigned int init_sz) -{ - unsigned int pe_header; - unsigned int bss_sz = init_sz - file_sz; - - pe_header = get_unaligned_le32(&buf[0x3c]); - - /* Size of uninitialized data */ - put_unaligned_le32(bss_sz, &buf[pe_header + 0x24]); - - /* Size of image */ - put_unaligned_le32(init_sz, &buf[pe_header + 0x50]); - - update_pecoff_section_header_fields(".bss", file_sz, bss_sz, 0, 0); + update_pecoff_section_header_fields(".text", text_start, text_sz + bss_sz, + text_sz, text_start); } static int reserve_pecoff_reloc_section(int c) @@ -278,9 +306,8 @@ static void efi_stub_entry_update(void) static inline void update_pecoff_setup_and_reloc(unsigned int size) {} static inline void update_pecoff_text(unsigned int text_start, - unsigned int file_sz) {} -static inline void update_pecoff_bss(unsigned int file_sz, - unsigned int init_sz) {} + unsigned int file_sz, + unsigned int init_sz) {} static inline void efi_stub_defaults(void) {} static inline void efi_stub_entry_update(void) {} @@ -290,6 +317,12 @@ static inline int reserve_pecoff_reloc_section(int c) } #endif /* CONFIG_EFI_STUB */ +static int reserve_pecoff_compat_section(int c) +{ + /* Reserve 0x20 bytes for .compat section */ + memset(buf+c, 0, PECOFF_COMPAT_RESERVE); + return PECOFF_COMPAT_RESERVE; +} /* * Parse zoffset.h and find the entry points. We could just #include zoffset.h @@ -322,8 +355,11 @@ static void parse_zoffset(char *fname) PARSE_ZOFS(p, efi32_stub_entry); PARSE_ZOFS(p, efi64_stub_entry); PARSE_ZOFS(p, efi_pe_entry); + PARSE_ZOFS(p, efi32_pe_entry); PARSE_ZOFS(p, kernel_info); PARSE_ZOFS(p, startup_64); + PARSE_ZOFS(p, _ehead); + PARSE_ZOFS(p, _end); p = strchr(p, '\n'); while (p && (*p == '\r' || *p == '\n')) @@ -365,6 +401,7 @@ int main(int argc, char ** argv) die("Boot block hasn't got boot flag (0xAA55)"); fclose(file); + c += reserve_pecoff_compat_section(c); c += reserve_pecoff_reloc_section(c); /* Pad unused space with zeros */ @@ -406,9 +443,28 @@ int main(int argc, char ** argv) buf[0x1f1] = setup_sectors-1; put_unaligned_le32(sys_size, &buf[0x1f4]); - update_pecoff_text(setup_sectors * 512, i + (sys_size * 16)); init_sz = get_unaligned_le32(&buf[0x260]); - update_pecoff_bss(i + (sys_size * 16), init_sz); +#ifdef CONFIG_EFI_STUB + /* + * The decompression buffer will start at ImageBase. When relocating + * the compressed kernel to its end, we must ensure that the head + * section does not get overwritten. The head section occupies + * [i, i + _ehead), and the destination is [init_sz - _end, init_sz). + * + * At present these should never overlap, because 'i' is at most 32k + * because of SETUP_SECT_MAX, '_ehead' is less than 1k, and the + * calculation of INIT_SIZE in boot/header.S ensures that + * 'init_sz - _end' is at least 64k. + * + * For future-proofing, increase init_sz if necessary. + */ + + if (init_sz - _end < i + _ehead) { + init_sz = (i + _ehead + _end + 4095) & ~4095; + put_unaligned_le32(init_sz, &buf[0x260]); + } +#endif + update_pecoff_text(setup_sectors * 512, i + (sys_size * 16), init_sz); efi_stub_entry_update(); diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index 59ce9ed58430..5b602beb0b72 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -125,7 +125,6 @@ CONFIG_IP6_NF_MANGLE=y CONFIG_NET_SCHED=y CONFIG_NET_EMATCH=y CONFIG_NET_CLS_ACT=y -CONFIG_HAMRADIO=y CONFIG_CFG80211=y CONFIG_MAC80211=y CONFIG_MAC80211_LEDS=y @@ -171,7 +170,6 @@ CONFIG_FORCEDETH=y CONFIG_8139TOO=y # CONFIG_8139TOO_PIO is not set CONFIG_R8169=y -CONFIG_FDDI=y CONFIG_INPUT_POLLDEV=y # CONFIG_INPUT_MOUSEDEV_PSAUX is not set CONFIG_INPUT_EVDEV=y diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 0b9654c7a05c..f3d1f36103b1 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -123,7 +123,6 @@ CONFIG_IP6_NF_MANGLE=y CONFIG_NET_SCHED=y CONFIG_NET_EMATCH=y CONFIG_NET_CLS_ACT=y -CONFIG_HAMRADIO=y CONFIG_CFG80211=y CONFIG_MAC80211=y CONFIG_MAC80211_LEDS=y @@ -164,7 +163,6 @@ CONFIG_SKY2=y CONFIG_FORCEDETH=y CONFIG_8139TOO=y CONFIG_R8169=y -CONFIG_FDDI=y CONFIG_INPUT_POLLDEV=y # CONFIG_INPUT_MOUSEDEV_PSAUX is not set CONFIG_INPUT_EVDEV=y diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index bbbebbd35b5d..75b6ea20491e 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -1064,7 +1064,7 @@ static struct aead_alg aesni_aeads[0]; static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)]; static const struct x86_cpu_id aesni_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_AES), + X86_MATCH_FEATURE(X86_FEATURE_AES, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c index 418bd88acac8..7c4c7b2fbf05 100644 --- a/arch/x86/crypto/crc32-pclmul_glue.c +++ b/arch/x86/crypto/crc32-pclmul_glue.c @@ -170,7 +170,7 @@ static struct shash_alg alg = { }; static const struct x86_cpu_id crc32pclmul_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), + X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id); diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c index c20d1b8a82c3..d2d069bd459b 100644 --- a/arch/x86/crypto/crc32c-intel_glue.c +++ b/arch/x86/crypto/crc32c-intel_glue.c @@ -221,7 +221,7 @@ static struct shash_alg alg = { }; static const struct x86_cpu_id crc32c_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_XMM4_2), + X86_MATCH_FEATURE(X86_FEATURE_XMM4_2, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id); diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c index 3c81e15b0873..71291d5af9f4 100644 --- a/arch/x86/crypto/crct10dif-pclmul_glue.c +++ b/arch/x86/crypto/crct10dif-pclmul_glue.c @@ -114,7 +114,7 @@ static struct shash_alg alg = { }; static const struct x86_cpu_id crct10dif_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), + X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id); diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index a4b728518e28..1f1a95f3dd0c 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -313,7 +313,7 @@ static struct ahash_alg ghash_async_alg = { }; static const struct x86_cpu_id pcmul_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */ + X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */ {} }; MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile index 06fc70cf5433..85eb381259c2 100644 --- a/arch/x86/entry/Makefile +++ b/arch/x86/entry/Makefile @@ -14,4 +14,5 @@ obj-y += vdso/ obj-y += vsyscall/ obj-$(CONFIG_IA32_EMULATION) += entry_64_compat.o syscall_32.o +obj-$(CONFIG_X86_X32_ABI) += syscall_x32.o diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 9747876980b5..76735ec813e6 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -34,6 +34,7 @@ #include <asm/fpu/api.h> #include <asm/nospec-branch.h> #include <asm/io_bitmap.h> +#include <asm/syscall.h> #define CREATE_TRACE_POINTS #include <trace/events/syscalls.h> @@ -333,20 +334,7 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs) if (likely(nr < IA32_NR_syscalls)) { nr = array_index_nospec(nr, IA32_NR_syscalls); -#ifdef CONFIG_IA32_EMULATION regs->ax = ia32_sys_call_table[nr](regs); -#else - /* - * It's possible that a 32-bit syscall implementation - * takes a 64-bit parameter but nonetheless assumes that - * the high bits are zero. Make sure we zero-extend all - * of the args. - */ - regs->ax = ia32_sys_call_table[nr]( - (unsigned int)regs->bx, (unsigned int)regs->cx, - (unsigned int)regs->dx, (unsigned int)regs->si, - (unsigned int)regs->di, (unsigned int)regs->bp); -#endif /* CONFIG_IA32_EMULATION */ } syscall_return_slowpath(regs); @@ -438,3 +426,8 @@ __visible long do_fast_syscall_32(struct pt_regs *regs) #endif } #endif + +SYSCALL_DEFINE0(ni_syscall) +{ + return -ENOSYS; +} diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 7e0560442538..b67bae7091d7 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -1088,10 +1088,10 @@ SYM_FUNC_START(entry_INT80_32) STACKLEAK_ERASE restore_all: - TRACE_IRQS_IRET + TRACE_IRQS_ON SWITCH_TO_ENTRY_STACK CHECK_AND_APPLY_ESPFIX -.Lrestore_nocheck: + /* Switch back to user CR3 */ SWITCH_TO_USER_CR3 scratch_reg=%eax @@ -1290,7 +1290,7 @@ SYM_CODE_END(simd_coprocessor_error) SYM_CODE_START(device_not_available) ASM_CLAC - pushl $-1 # mark this as an int + pushl $0 pushl $do_device_not_available jmp common_exception SYM_CODE_END(device_not_available) @@ -1365,7 +1365,7 @@ SYM_CODE_END(divide_error) SYM_CODE_START(machine_check) ASM_CLAC pushl $0 - pushl machine_check_vector + pushl $do_mce jmp common_exception SYM_CODE_END(machine_check) #endif @@ -1531,7 +1531,7 @@ SYM_CODE_START(debug) * Entry from sysenter is now handled in common_exception */ ASM_CLAC - pushl $-1 # mark this as an int + pushl $0 pushl $do_debug jmp common_exception SYM_CODE_END(debug) @@ -1682,18 +1682,13 @@ SYM_CODE_END(nmi) SYM_CODE_START(int3) ASM_CLAC - pushl $-1 # mark this as an int - - SAVE_ALL switch_stacks=1 - ENCODE_FRAME_POINTER - TRACE_IRQS_OFF - xorl %edx, %edx # zero error code - movl %esp, %eax # pt_regs pointer - call do_int3 - jmp ret_from_exception + pushl $0 + pushl $do_int3 + jmp common_exception SYM_CODE_END(int3) SYM_CODE_START(general_protection) + ASM_CLAC pushl $do_general_protection jmp common_exception SYM_CODE_END(general_protection) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index f2bb91e87877..0e9504fabe52 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -174,7 +174,7 @@ SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) movq %rsp, %rsi call do_syscall_64 /* returns with IRQs disabled */ - TRACE_IRQS_IRETQ /* we're about to change IF */ + TRACE_IRQS_ON /* return enables interrupts */ /* * Try to use SYSRET instead of IRET if we're returning to @@ -619,7 +619,7 @@ ret_from_intr: .Lretint_user: mov %rsp,%rdi call prepare_exit_to_usermode - TRACE_IRQS_IRETQ + TRACE_IRQS_ON SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) #ifdef CONFIG_DEBUG_ENTRY diff --git a/arch/x86/entry/syscall_32.c b/arch/x86/entry/syscall_32.c index 7d17b3addbbb..86eb0d89d46f 100644 --- a/arch/x86/entry/syscall_32.c +++ b/arch/x86/entry/syscall_32.c @@ -4,29 +4,22 @@ #include <linux/linkage.h> #include <linux/sys.h> #include <linux/cache.h> -#include <asm/asm-offsets.h> +#include <linux/syscalls.h> +#include <asm/unistd.h> #include <asm/syscall.h> -#ifdef CONFIG_IA32_EMULATION -/* On X86_64, we use struct pt_regs * to pass parameters to syscalls */ -#define __SYSCALL_I386(nr, sym, qual) extern asmlinkage long sym(const struct pt_regs *); -#define __sys_ni_syscall __ia32_sys_ni_syscall -#else /* CONFIG_IA32_EMULATION */ -#define __SYSCALL_I386(nr, sym, qual) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); -extern asmlinkage long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); -#define __sys_ni_syscall sys_ni_syscall -#endif /* CONFIG_IA32_EMULATION */ +#define __SYSCALL_I386(nr, sym) extern long __ia32_##sym(const struct pt_regs *); #include <asm/syscalls_32.h> #undef __SYSCALL_I386 -#define __SYSCALL_I386(nr, sym, qual) [nr] = sym, +#define __SYSCALL_I386(nr, sym) [nr] = __ia32_##sym, -__visible const sys_call_ptr_t ia32_sys_call_table[__NR_syscall_compat_max+1] = { +__visible const sys_call_ptr_t ia32_sys_call_table[__NR_ia32_syscall_max+1] = { /* * Smells like a compiler bug -- it doesn't work * when the & below is removed. */ - [0 ... __NR_syscall_compat_max] = &__sys_ni_syscall, + [0 ... __NR_ia32_syscall_max] = &__ia32_sys_ni_syscall, #include <asm/syscalls_32.h> }; diff --git a/arch/x86/entry/syscall_64.c b/arch/x86/entry/syscall_64.c index adf619a856e8..1594ec72bcbb 100644 --- a/arch/x86/entry/syscall_64.c +++ b/arch/x86/entry/syscall_64.c @@ -5,24 +5,17 @@ #include <linux/sys.h> #include <linux/cache.h> #include <linux/syscalls.h> -#include <asm/asm-offsets.h> +#include <asm/unistd.h> #include <asm/syscall.h> -extern asmlinkage long sys_ni_syscall(void); +#define __SYSCALL_X32(nr, sym) +#define __SYSCALL_COMMON(nr, sym) __SYSCALL_64(nr, sym) -SYSCALL_DEFINE0(ni_syscall) -{ - return sys_ni_syscall(); -} - -#define __SYSCALL_64(nr, sym, qual) extern asmlinkage long sym(const struct pt_regs *); -#define __SYSCALL_X32(nr, sym, qual) __SYSCALL_64(nr, sym, qual) +#define __SYSCALL_64(nr, sym) extern long __x64_##sym(const struct pt_regs *); #include <asm/syscalls_64.h> #undef __SYSCALL_64 -#undef __SYSCALL_X32 -#define __SYSCALL_64(nr, sym, qual) [nr] = sym, -#define __SYSCALL_X32(nr, sym, qual) +#define __SYSCALL_64(nr, sym) [nr] = __x64_##sym, asmlinkage const sys_call_ptr_t sys_call_table[__NR_syscall_max+1] = { /* @@ -32,25 +25,3 @@ asmlinkage const sys_call_ptr_t sys_call_table[__NR_syscall_max+1] = { [0 ... __NR_syscall_max] = &__x64_sys_ni_syscall, #include <asm/syscalls_64.h> }; - -#undef __SYSCALL_64 -#undef __SYSCALL_X32 - -#ifdef CONFIG_X86_X32_ABI - -#define __SYSCALL_64(nr, sym, qual) -#define __SYSCALL_X32(nr, sym, qual) [nr] = sym, - -asmlinkage const sys_call_ptr_t x32_sys_call_table[__NR_syscall_x32_max+1] = { - /* - * Smells like a compiler bug -- it doesn't work - * when the & below is removed. - */ - [0 ... __NR_syscall_x32_max] = &__x64_sys_ni_syscall, -#include <asm/syscalls_64.h> -}; - -#undef __SYSCALL_64 -#undef __SYSCALL_X32 - -#endif diff --git a/arch/x86/entry/syscall_x32.c b/arch/x86/entry/syscall_x32.c new file mode 100644 index 000000000000..3d8d70d3896c --- /dev/null +++ b/arch/x86/entry/syscall_x32.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* System call table for x32 ABI. */ + +#include <linux/linkage.h> +#include <linux/sys.h> +#include <linux/cache.h> +#include <linux/syscalls.h> +#include <asm/unistd.h> +#include <asm/syscall.h> + +#define __SYSCALL_64(nr, sym) + +#define __SYSCALL_X32(nr, sym) extern long __x32_##sym(const struct pt_regs *); +#define __SYSCALL_COMMON(nr, sym) extern long __x64_##sym(const struct pt_regs *); +#include <asm/syscalls_64.h> +#undef __SYSCALL_X32 +#undef __SYSCALL_COMMON + +#define __SYSCALL_X32(nr, sym) [nr] = __x32_##sym, +#define __SYSCALL_COMMON(nr, sym) [nr] = __x64_##sym, + +asmlinkage const sys_call_ptr_t x32_sys_call_table[__NR_x32_syscall_max+1] = { + /* + * Smells like a compiler bug -- it doesn't work + * when the & below is removed. + */ + [0 ... __NR_x32_syscall_max] = &__x64_sys_ni_syscall, +#include <asm/syscalls_64.h> +}; diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index c17cb77eb150..54581ac671b4 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -11,434 +11,434 @@ # # The abi is always "i386" for this file. # -0 i386 restart_syscall sys_restart_syscall __ia32_sys_restart_syscall -1 i386 exit sys_exit __ia32_sys_exit -2 i386 fork sys_fork __ia32_sys_fork -3 i386 read sys_read __ia32_sys_read -4 i386 write sys_write __ia32_sys_write -5 i386 open sys_open __ia32_compat_sys_open -6 i386 close sys_close __ia32_sys_close -7 i386 waitpid sys_waitpid __ia32_sys_waitpid -8 i386 creat sys_creat __ia32_sys_creat -9 i386 link sys_link __ia32_sys_link -10 i386 unlink sys_unlink __ia32_sys_unlink -11 i386 execve sys_execve __ia32_compat_sys_execve -12 i386 chdir sys_chdir __ia32_sys_chdir -13 i386 time sys_time32 __ia32_sys_time32 -14 i386 mknod sys_mknod __ia32_sys_mknod -15 i386 chmod sys_chmod __ia32_sys_chmod -16 i386 lchown sys_lchown16 __ia32_sys_lchown16 +0 i386 restart_syscall sys_restart_syscall +1 i386 exit sys_exit +2 i386 fork sys_fork +3 i386 read sys_read +4 i386 write sys_write +5 i386 open sys_open compat_sys_open +6 i386 close sys_close +7 i386 waitpid sys_waitpid +8 i386 creat sys_creat +9 i386 link sys_link +10 i386 unlink sys_unlink +11 i386 execve sys_execve compat_sys_execve +12 i386 chdir sys_chdir +13 i386 time sys_time32 +14 i386 mknod sys_mknod +15 i386 chmod sys_chmod +16 i386 lchown sys_lchown16 17 i386 break -18 i386 oldstat sys_stat __ia32_sys_stat -19 i386 lseek sys_lseek __ia32_compat_sys_lseek -20 i386 getpid sys_getpid __ia32_sys_getpid -21 i386 mount sys_mount __ia32_compat_sys_mount -22 i386 umount sys_oldumount __ia32_sys_oldumount -23 i386 setuid sys_setuid16 __ia32_sys_setuid16 -24 i386 getuid sys_getuid16 __ia32_sys_getuid16 -25 i386 stime sys_stime32 __ia32_sys_stime32 -26 i386 ptrace sys_ptrace __ia32_compat_sys_ptrace -27 i386 alarm sys_alarm __ia32_sys_alarm -28 i386 oldfstat sys_fstat __ia32_sys_fstat -29 i386 pause sys_pause __ia32_sys_pause -30 i386 utime sys_utime32 __ia32_sys_utime32 +18 i386 oldstat sys_stat +19 i386 lseek sys_lseek compat_sys_lseek +20 i386 getpid sys_getpid +21 i386 mount sys_mount compat_sys_mount +22 i386 umount sys_oldumount +23 i386 setuid sys_setuid16 +24 i386 getuid sys_getuid16 +25 i386 stime sys_stime32 +26 i386 ptrace sys_ptrace compat_sys_ptrace +27 i386 alarm sys_alarm +28 i386 oldfstat sys_fstat +29 i386 pause sys_pause +30 i386 utime sys_utime32 31 i386 stty 32 i386 gtty -33 i386 access sys_access __ia32_sys_access -34 i386 nice sys_nice __ia32_sys_nice +33 i386 access sys_access +34 i386 nice sys_nice 35 i386 ftime -36 i386 sync sys_sync __ia32_sys_sync -37 i386 kill sys_kill __ia32_sys_kill -38 i386 rename sys_rename __ia32_sys_rename -39 i386 mkdir sys_mkdir __ia32_sys_mkdir -40 i386 rmdir sys_rmdir __ia32_sys_rmdir -41 i386 dup sys_dup __ia32_sys_dup -42 i386 pipe sys_pipe __ia32_sys_pipe -43 i386 times sys_times __ia32_compat_sys_times +36 i386 sync sys_sync +37 i386 kill sys_kill +38 i386 rename sys_rename +39 i386 mkdir sys_mkdir +40 i386 rmdir sys_rmdir +41 i386 dup sys_dup +42 i386 pipe sys_pipe +43 i386 times sys_times compat_sys_times 44 i386 prof -45 i386 brk sys_brk __ia32_sys_brk -46 i386 setgid sys_setgid16 __ia32_sys_setgid16 -47 i386 getgid sys_getgid16 __ia32_sys_getgid16 -48 i386 signal sys_signal __ia32_sys_signal -49 i386 geteuid sys_geteuid16 __ia32_sys_geteuid16 -50 i386 getegid sys_getegid16 __ia32_sys_getegid16 -51 i386 acct sys_acct __ia32_sys_acct -52 i386 umount2 sys_umount __ia32_sys_umount +45 i386 brk sys_brk +46 i386 setgid sys_setgid16 +47 i386 getgid sys_getgid16 +48 i386 signal sys_signal +49 i386 geteuid sys_geteuid16 +50 i386 getegid sys_getegid16 +51 i386 acct sys_acct +52 i386 umount2 sys_umount 53 i386 lock -54 i386 ioctl sys_ioctl __ia32_compat_sys_ioctl -55 i386 fcntl sys_fcntl __ia32_compat_sys_fcntl64 +54 i386 ioctl sys_ioctl compat_sys_ioctl +55 i386 fcntl sys_fcntl compat_sys_fcntl64 56 i386 mpx -57 i386 setpgid sys_setpgid __ia32_sys_setpgid +57 i386 setpgid sys_setpgid 58 i386 ulimit -59 i386 oldolduname sys_olduname __ia32_sys_olduname -60 i386 umask sys_umask __ia32_sys_umask -61 i386 chroot sys_chroot __ia32_sys_chroot -62 i386 ustat sys_ustat __ia32_compat_sys_ustat -63 i386 dup2 sys_dup2 __ia32_sys_dup2 -64 i386 getppid sys_getppid __ia32_sys_getppid -65 i386 getpgrp sys_getpgrp __ia32_sys_getpgrp -66 i386 setsid sys_setsid __ia32_sys_setsid -67 i386 sigaction sys_sigaction __ia32_compat_sys_sigaction -68 i386 sgetmask sys_sgetmask __ia32_sys_sgetmask -69 i386 ssetmask sys_ssetmask __ia32_sys_ssetmask -70 i386 setreuid sys_setreuid16 __ia32_sys_setreuid16 -71 i386 setregid sys_setregid16 __ia32_sys_setregid16 -72 i386 sigsuspend sys_sigsuspend __ia32_sys_sigsuspend -73 i386 sigpending sys_sigpending __ia32_compat_sys_sigpending -74 i386 sethostname sys_sethostname __ia32_sys_sethostname -75 i386 setrlimit sys_setrlimit __ia32_compat_sys_setrlimit -76 i386 getrlimit sys_old_getrlimit __ia32_compat_sys_old_getrlimit -77 i386 getrusage sys_getrusage __ia32_compat_sys_getrusage -78 i386 gettimeofday sys_gettimeofday __ia32_compat_sys_gettimeofday -79 i386 settimeofday sys_settimeofday __ia32_compat_sys_settimeofday -80 i386 getgroups sys_getgroups16 __ia32_sys_getgroups16 -81 i386 setgroups sys_setgroups16 __ia32_sys_setgroups16 -82 i386 select sys_old_select __ia32_compat_sys_old_select -83 i386 symlink sys_symlink __ia32_sys_symlink -84 i386 oldlstat sys_lstat __ia32_sys_lstat -85 i386 readlink sys_readlink __ia32_sys_readlink -86 i386 uselib sys_uselib __ia32_sys_uselib -87 i386 swapon sys_swapon __ia32_sys_swapon -88 i386 reboot sys_reboot __ia32_sys_reboot -89 i386 readdir sys_old_readdir __ia32_compat_sys_old_readdir -90 i386 mmap sys_old_mmap __ia32_compat_sys_x86_mmap -91 i386 munmap sys_munmap __ia32_sys_munmap -92 i386 truncate sys_truncate __ia32_compat_sys_truncate -93 i386 ftruncate sys_ftruncate __ia32_compat_sys_ftruncate -94 i386 fchmod sys_fchmod __ia32_sys_fchmod -95 i386 fchown sys_fchown16 __ia32_sys_fchown16 -96 i386 getpriority sys_getpriority __ia32_sys_getpriority -97 i386 setpriority sys_setpriority __ia32_sys_setpriority +59 i386 oldolduname sys_olduname +60 i386 umask sys_umask +61 i386 chroot sys_chroot +62 i386 ustat sys_ustat compat_sys_ustat +63 i386 dup2 sys_dup2 +64 i386 getppid sys_getppid +65 i386 getpgrp sys_getpgrp +66 i386 setsid sys_setsid +67 i386 sigaction sys_sigaction compat_sys_sigaction +68 i386 sgetmask sys_sgetmask +69 i386 ssetmask sys_ssetmask +70 i386 setreuid sys_setreuid16 +71 i386 setregid sys_setregid16 +72 i386 sigsuspend sys_sigsuspend +73 i386 sigpending sys_sigpending compat_sys_sigpending +74 i386 sethostname sys_sethostname +75 i386 setrlimit sys_setrlimit compat_sys_setrlimit +76 i386 getrlimit sys_old_getrlimit compat_sys_old_getrlimit +77 i386 getrusage sys_getrusage compat_sys_getrusage +78 i386 gettimeofday sys_gettimeofday compat_sys_gettimeofday +79 i386 settimeofday sys_settimeofday compat_sys_settimeofday +80 i386 getgroups sys_getgroups16 +81 i386 setgroups sys_setgroups16 +82 i386 select sys_old_select compat_sys_old_select +83 i386 symlink sys_symlink +84 i386 oldlstat sys_lstat +85 i386 readlink sys_readlink +86 i386 uselib sys_uselib +87 i386 swapon sys_swapon +88 i386 reboot sys_reboot +89 i386 readdir sys_old_readdir compat_sys_old_readdir +90 i386 mmap sys_old_mmap compat_sys_ia32_mmap +91 i386 munmap sys_munmap +92 i386 truncate sys_truncate compat_sys_truncate +93 i386 ftruncate sys_ftruncate compat_sys_ftruncate +94 i386 fchmod sys_fchmod +95 i386 fchown sys_fchown16 +96 i386 getpriority sys_getpriority +97 i386 setpriority sys_setpriority 98 i386 profil -99 i386 statfs sys_statfs __ia32_compat_sys_statfs -100 i386 fstatfs sys_fstatfs __ia32_compat_sys_fstatfs -101 i386 ioperm sys_ioperm __ia32_sys_ioperm -102 i386 socketcall sys_socketcall __ia32_compat_sys_socketcall -103 i386 syslog sys_syslog __ia32_sys_syslog -104 i386 setitimer sys_setitimer __ia32_compat_sys_setitimer -105 i386 getitimer sys_getitimer __ia32_compat_sys_getitimer -106 i386 stat sys_newstat __ia32_compat_sys_newstat -107 i386 lstat sys_newlstat __ia32_compat_sys_newlstat -108 i386 fstat sys_newfstat __ia32_compat_sys_newfstat -109 i386 olduname sys_uname __ia32_sys_uname -110 i386 iopl sys_iopl __ia32_sys_iopl -111 i386 vhangup sys_vhangup __ia32_sys_vhangup +99 i386 statfs sys_statfs compat_sys_statfs +100 i386 fstatfs sys_fstatfs compat_sys_fstatfs +101 i386 ioperm sys_ioperm +102 i386 socketcall sys_socketcall compat_sys_socketcall +103 i386 syslog sys_syslog +104 i386 setitimer sys_setitimer compat_sys_setitimer +105 i386 getitimer sys_getitimer compat_sys_getitimer +106 i386 stat sys_newstat compat_sys_newstat +107 i386 lstat sys_newlstat compat_sys_newlstat +108 i386 fstat sys_newfstat compat_sys_newfstat +109 i386 olduname sys_uname +110 i386 iopl sys_iopl +111 i386 vhangup sys_vhangup 112 i386 idle -113 i386 vm86old sys_vm86old __ia32_sys_ni_syscall -114 i386 wait4 sys_wait4 __ia32_compat_sys_wait4 -115 i386 swapoff sys_swapoff __ia32_sys_swapoff -116 i386 sysinfo sys_sysinfo __ia32_compat_sys_sysinfo -117 i386 ipc sys_ipc __ia32_compat_sys_ipc -118 i386 fsync sys_fsync __ia32_sys_fsync -119 i386 sigreturn sys_sigreturn __ia32_compat_sys_sigreturn -120 i386 clone sys_clone __ia32_compat_sys_x86_clone -121 i386 setdomainname sys_setdomainname __ia32_sys_setdomainname -122 i386 uname sys_newuname __ia32_sys_newuname -123 i386 modify_ldt sys_modify_ldt __ia32_sys_modify_ldt -124 i386 adjtimex sys_adjtimex_time32 __ia32_sys_adjtimex_time32 -125 i386 mprotect sys_mprotect __ia32_sys_mprotect -126 i386 sigprocmask sys_sigprocmask __ia32_compat_sys_sigprocmask +113 i386 vm86old sys_vm86old sys_ni_syscall +114 i386 wait4 sys_wait4 compat_sys_wait4 +115 i386 swapoff sys_swapoff +116 i386 sysinfo sys_sysinfo compat_sys_sysinfo +117 i386 ipc sys_ipc compat_sys_ipc +118 i386 fsync sys_fsync +119 i386 sigreturn sys_sigreturn compat_sys_sigreturn +120 i386 clone sys_clone compat_sys_ia32_clone +121 i386 setdomainname sys_setdomainname +122 i386 uname sys_newuname +123 i386 modify_ldt sys_modify_ldt +124 i386 adjtimex sys_adjtimex_time32 +125 i386 mprotect sys_mprotect +126 i386 sigprocmask sys_sigprocmask compat_sys_sigprocmask 127 i386 create_module -128 i386 init_module sys_init_module __ia32_sys_init_module -129 i386 delete_module sys_delete_module __ia32_sys_delete_module +128 i386 init_module sys_init_module +129 i386 delete_module sys_delete_module 130 i386 get_kernel_syms -131 i386 quotactl sys_quotactl __ia32_compat_sys_quotactl32 -132 i386 getpgid sys_getpgid __ia32_sys_getpgid -133 i386 fchdir sys_fchdir __ia32_sys_fchdir -134 i386 bdflush sys_bdflush __ia32_sys_bdflush -135 i386 sysfs sys_sysfs __ia32_sys_sysfs -136 i386 personality sys_personality __ia32_sys_personality +131 i386 quotactl sys_quotactl compat_sys_quotactl32 +132 i386 getpgid sys_getpgid +133 i386 fchdir sys_fchdir +134 i386 bdflush sys_bdflush +135 i386 sysfs sys_sysfs +136 i386 personality sys_personality 137 i386 afs_syscall -138 i386 setfsuid sys_setfsuid16 __ia32_sys_setfsuid16 -139 i386 setfsgid sys_setfsgid16 __ia32_sys_setfsgid16 -140 i386 _llseek sys_llseek __ia32_sys_llseek -141 i386 getdents sys_getdents __ia32_compat_sys_getdents -142 i386 _newselect sys_select __ia32_compat_sys_select -143 i386 flock sys_flock __ia32_sys_flock -144 i386 msync sys_msync __ia32_sys_msync -145 i386 readv sys_readv __ia32_compat_sys_readv -146 i386 writev sys_writev __ia32_compat_sys_writev -147 i386 getsid sys_getsid __ia32_sys_getsid -148 i386 fdatasync sys_fdatasync __ia32_sys_fdatasync -149 i386 _sysctl sys_sysctl __ia32_compat_sys_sysctl -150 i386 mlock sys_mlock __ia32_sys_mlock -151 i386 munlock sys_munlock __ia32_sys_munlock -152 i386 mlockall sys_mlockall __ia32_sys_mlockall -153 i386 munlockall sys_munlockall __ia32_sys_munlockall -154 i386 sched_setparam sys_sched_setparam __ia32_sys_sched_setparam -155 i386 sched_getparam sys_sched_getparam __ia32_sys_sched_getparam -156 i386 sched_setscheduler sys_sched_setscheduler __ia32_sys_sched_setscheduler -157 i386 sched_getscheduler sys_sched_getscheduler __ia32_sys_sched_getscheduler -158 i386 sched_yield sys_sched_yield __ia32_sys_sched_yield -159 i386 sched_get_priority_max sys_sched_get_priority_max __ia32_sys_sched_get_priority_max -160 i386 sched_get_priority_min sys_sched_get_priority_min __ia32_sys_sched_get_priority_min -161 i386 sched_rr_get_interval sys_sched_rr_get_interval_time32 __ia32_sys_sched_rr_get_interval_time32 -162 i386 nanosleep sys_nanosleep_time32 __ia32_sys_nanosleep_time32 -163 i386 mremap sys_mremap __ia32_sys_mremap -164 i386 setresuid sys_setresuid16 __ia32_sys_setresuid16 -165 i386 getresuid sys_getresuid16 __ia32_sys_getresuid16 -166 i386 vm86 sys_vm86 __ia32_sys_ni_syscall +138 i386 setfsuid sys_setfsuid16 +139 i386 setfsgid sys_setfsgid16 +140 i386 _llseek sys_llseek +141 i386 getdents sys_getdents compat_sys_getdents +142 i386 _newselect sys_select compat_sys_select +143 i386 flock sys_flock +144 i386 msync sys_msync +145 i386 readv sys_readv compat_sys_readv +146 i386 writev sys_writev compat_sys_writev +147 i386 getsid sys_getsid +148 i386 fdatasync sys_fdatasync +149 i386 _sysctl sys_sysctl compat_sys_sysctl +150 i386 mlock sys_mlock +151 i386 munlock sys_munlock +152 i386 mlockall sys_mlockall +153 i386 munlockall sys_munlockall +154 i386 sched_setparam sys_sched_setparam +155 i386 sched_getparam sys_sched_getparam +156 i386 sched_setscheduler sys_sched_setscheduler +157 i386 sched_getscheduler sys_sched_getscheduler +158 i386 sched_yield sys_sched_yield +159 i386 sched_get_priority_max sys_sched_get_priority_max +160 i386 sched_get_priority_min sys_sched_get_priority_min +161 i386 sched_rr_get_interval sys_sched_rr_get_interval_time32 +162 i386 nanosleep sys_nanosleep_time32 +163 i386 mremap sys_mremap +164 i386 setresuid sys_setresuid16 +165 i386 getresuid sys_getresuid16 +166 i386 vm86 sys_vm86 sys_ni_syscall 167 i386 query_module -168 i386 poll sys_poll __ia32_sys_poll +168 i386 poll sys_poll 169 i386 nfsservctl -170 i386 setresgid sys_setresgid16 __ia32_sys_setresgid16 -171 i386 getresgid sys_getresgid16 __ia32_sys_getresgid16 -172 i386 prctl sys_prctl __ia32_sys_prctl -173 i386 rt_sigreturn sys_rt_sigreturn __ia32_compat_sys_rt_sigreturn -174 i386 rt_sigaction sys_rt_sigaction __ia32_compat_sys_rt_sigaction -175 i386 rt_sigprocmask sys_rt_sigprocmask __ia32_compat_sys_rt_sigprocmask -176 i386 rt_sigpending sys_rt_sigpending __ia32_compat_sys_rt_sigpending -177 i386 rt_sigtimedwait sys_rt_sigtimedwait_time32 __ia32_compat_sys_rt_sigtimedwait_time32 -178 i386 rt_sigqueueinfo sys_rt_sigqueueinfo __ia32_compat_sys_rt_sigqueueinfo -179 i386 rt_sigsuspend sys_rt_sigsuspend __ia32_compat_sys_rt_sigsuspend -180 i386 pread64 sys_pread64 __ia32_compat_sys_x86_pread -181 i386 pwrite64 sys_pwrite64 __ia32_compat_sys_x86_pwrite -182 i386 chown sys_chown16 __ia32_sys_chown16 -183 i386 getcwd sys_getcwd __ia32_sys_getcwd -184 i386 capget sys_capget __ia32_sys_capget -185 i386 capset sys_capset __ia32_sys_capset -186 i386 sigaltstack sys_sigaltstack __ia32_compat_sys_sigaltstack -187 i386 sendfile sys_sendfile __ia32_compat_sys_sendfile +170 i386 setresgid sys_setresgid16 +171 i386 getresgid sys_getresgid16 +172 i386 prctl sys_prctl +173 i386 rt_sigreturn sys_rt_sigreturn compat_sys_rt_sigreturn +174 i386 rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction +175 i386 rt_sigprocmask sys_rt_sigprocmask compat_sys_rt_sigprocmask +176 i386 rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending +177 i386 rt_sigtimedwait sys_rt_sigtimedwait_time32 compat_sys_rt_sigtimedwait_time32 +178 i386 rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo +179 i386 rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend +180 i386 pread64 sys_ia32_pread64 +181 i386 pwrite64 sys_ia32_pwrite64 +182 i386 chown sys_chown16 +183 i386 getcwd sys_getcwd +184 i386 capget sys_capget +185 i386 capset sys_capset +186 i386 sigaltstack sys_sigaltstack compat_sys_sigaltstack +187 i386 sendfile sys_sendfile compat_sys_sendfile 188 i386 getpmsg 189 i386 putpmsg -190 i386 vfork sys_vfork __ia32_sys_vfork -191 i386 ugetrlimit sys_getrlimit __ia32_compat_sys_getrlimit -192 i386 mmap2 sys_mmap_pgoff __ia32_sys_mmap_pgoff -193 i386 truncate64 sys_truncate64 __ia32_compat_sys_x86_truncate64 -194 i386 ftruncate64 sys_ftruncate64 __ia32_compat_sys_x86_ftruncate64 -195 i386 stat64 sys_stat64 __ia32_compat_sys_x86_stat64 -196 i386 lstat64 sys_lstat64 __ia32_compat_sys_x86_lstat64 -197 i386 fstat64 sys_fstat64 __ia32_compat_sys_x86_fstat64 -198 i386 lchown32 sys_lchown __ia32_sys_lchown -199 i386 getuid32 sys_getuid __ia32_sys_getuid -200 i386 getgid32 sys_getgid __ia32_sys_getgid -201 i386 geteuid32 sys_geteuid __ia32_sys_geteuid -202 i386 getegid32 sys_getegid __ia32_sys_getegid -203 i386 setreuid32 sys_setreuid __ia32_sys_setreuid -204 i386 setregid32 sys_setregid __ia32_sys_setregid -205 i386 getgroups32 sys_getgroups __ia32_sys_getgroups -206 i386 setgroups32 sys_setgroups __ia32_sys_setgroups -207 i386 fchown32 sys_fchown __ia32_sys_fchown -208 i386 setresuid32 sys_setresuid __ia32_sys_setresuid -209 i386 getresuid32 sys_getresuid __ia32_sys_getresuid -210 i386 setresgid32 sys_setresgid __ia32_sys_setresgid -211 i386 getresgid32 sys_getresgid __ia32_sys_getresgid -212 i386 chown32 sys_chown __ia32_sys_chown -213 i386 setuid32 sys_setuid __ia32_sys_setuid -214 i386 setgid32 sys_setgid __ia32_sys_setgid -215 i386 setfsuid32 sys_setfsuid __ia32_sys_setfsuid -216 i386 setfsgid32 sys_setfsgid __ia32_sys_setfsgid -217 i386 pivot_root sys_pivot_root __ia32_sys_pivot_root -218 i386 mincore sys_mincore __ia32_sys_mincore -219 i386 madvise sys_madvise __ia32_sys_madvise -220 i386 getdents64 sys_getdents64 __ia32_sys_getdents64 -221 i386 fcntl64 sys_fcntl64 __ia32_compat_sys_fcntl64 +190 i386 vfork sys_vfork +191 i386 ugetrlimit sys_getrlimit compat_sys_getrlimit +192 i386 mmap2 sys_mmap_pgoff +193 i386 truncate64 sys_ia32_truncate64 +194 i386 ftruncate64 sys_ia32_ftruncate64 +195 i386 stat64 sys_stat64 compat_sys_ia32_stat64 +196 i386 lstat64 sys_lstat64 compat_sys_ia32_lstat64 +197 i386 fstat64 sys_fstat64 compat_sys_ia32_fstat64 +198 i386 lchown32 sys_lchown +199 i386 getuid32 sys_getuid +200 i386 getgid32 sys_getgid +201 i386 geteuid32 sys_geteuid +202 i386 getegid32 sys_getegid +203 i386 setreuid32 sys_setreuid +204 i386 setregid32 sys_setregid +205 i386 getgroups32 sys_getgroups +206 i386 setgroups32 sys_setgroups +207 i386 fchown32 sys_fchown +208 i386 setresuid32 sys_setresuid +209 i386 getresuid32 sys_getresuid +210 i386 setresgid32 sys_setresgid +211 i386 getresgid32 sys_getresgid +212 i386 chown32 sys_chown +213 i386 setuid32 sys_setuid +214 i386 setgid32 sys_setgid +215 i386 setfsuid32 sys_setfsuid +216 i386 setfsgid32 sys_setfsgid +217 i386 pivot_root sys_pivot_root +218 i386 mincore sys_mincore +219 i386 madvise sys_madvise +220 i386 getdents64 sys_getdents64 +221 i386 fcntl64 sys_fcntl64 compat_sys_fcntl64 # 222 is unused # 223 is unused -224 i386 gettid sys_gettid __ia32_sys_gettid -225 i386 readahead sys_readahead __ia32_compat_sys_x86_readahead -226 i386 setxattr sys_setxattr __ia32_sys_setxattr -227 i386 lsetxattr sys_lsetxattr __ia32_sys_lsetxattr -228 i386 fsetxattr sys_fsetxattr __ia32_sys_fsetxattr -229 i386 getxattr sys_getxattr __ia32_sys_getxattr -230 i386 lgetxattr sys_lgetxattr __ia32_sys_lgetxattr -231 i386 fgetxattr sys_fgetxattr __ia32_sys_fgetxattr -232 i386 listxattr sys_listxattr __ia32_sys_listxattr -233 i386 llistxattr sys_llistxattr __ia32_sys_llistxattr -234 i386 flistxattr sys_flistxattr __ia32_sys_flistxattr -235 i386 removexattr sys_removexattr __ia32_sys_removexattr -236 i386 lremovexattr sys_lremovexattr __ia32_sys_lremovexattr -237 i386 fremovexattr sys_fremovexattr __ia32_sys_fremovexattr -238 i386 tkill sys_tkill __ia32_sys_tkill -239 i386 sendfile64 sys_sendfile64 __ia32_sys_sendfile64 -240 i386 futex sys_futex_time32 __ia32_sys_futex_time32 -241 i386 sched_setaffinity sys_sched_setaffinity __ia32_compat_sys_sched_setaffinity -242 i386 sched_getaffinity sys_sched_getaffinity __ia32_compat_sys_sched_getaffinity -243 i386 set_thread_area sys_set_thread_area __ia32_sys_set_thread_area -244 i386 get_thread_area sys_get_thread_area __ia32_sys_get_thread_area -245 i386 io_setup sys_io_setup __ia32_compat_sys_io_setup -246 i386 io_destroy sys_io_destroy __ia32_sys_io_destroy -247 i386 io_getevents sys_io_getevents_time32 __ia32_sys_io_getevents_time32 -248 i386 io_submit sys_io_submit __ia32_compat_sys_io_submit -249 i386 io_cancel sys_io_cancel __ia32_sys_io_cancel -250 i386 fadvise64 sys_fadvise64 __ia32_compat_sys_x86_fadvise64 +224 i386 gettid sys_gettid +225 i386 readahead sys_ia32_readahead +226 i386 setxattr sys_setxattr +227 i386 lsetxattr sys_lsetxattr +228 i386 fsetxattr sys_fsetxattr +229 i386 getxattr sys_getxattr +230 i386 lgetxattr sys_lgetxattr +231 i386 fgetxattr sys_fgetxattr +232 i386 listxattr sys_listxattr +233 i386 llistxattr sys_llistxattr +234 i386 flistxattr sys_flistxattr +235 i386 removexattr sys_removexattr +236 i386 lremovexattr sys_lremovexattr +237 i386 fremovexattr sys_fremovexattr +238 i386 tkill sys_tkill +239 i386 sendfile64 sys_sendfile64 +240 i386 futex sys_futex_time32 +241 i386 sched_setaffinity sys_sched_setaffinity compat_sys_sched_setaffinity +242 i386 sched_getaffinity sys_sched_getaffinity compat_sys_sched_getaffinity +243 i386 set_thread_area sys_set_thread_area +244 i386 get_thread_area sys_get_thread_area +245 i386 io_setup sys_io_setup compat_sys_io_setup +246 i386 io_destroy sys_io_destroy +247 i386 io_getevents sys_io_getevents_time32 +248 i386 io_submit sys_io_submit compat_sys_io_submit +249 i386 io_cancel sys_io_cancel +250 i386 fadvise64 sys_ia32_fadvise64 # 251 is available for reuse (was briefly sys_set_zone_reclaim) -252 i386 exit_group sys_exit_group __ia32_sys_exit_group -253 i386 lookup_dcookie sys_lookup_dcookie __ia32_compat_sys_lookup_dcookie -254 i386 epoll_create sys_epoll_create __ia32_sys_epoll_create -255 i386 epoll_ctl sys_epoll_ctl __ia32_sys_epoll_ctl -256 i386 epoll_wait sys_epoll_wait __ia32_sys_epoll_wait -257 i386 remap_file_pages sys_remap_file_pages __ia32_sys_remap_file_pages -258 i386 set_tid_address sys_set_tid_address __ia32_sys_set_tid_address -259 i386 timer_create sys_timer_create __ia32_compat_sys_timer_create -260 i386 timer_settime sys_timer_settime32 __ia32_sys_timer_settime32 -261 i386 timer_gettime sys_timer_gettime32 __ia32_sys_timer_gettime32 -262 i386 timer_getoverrun sys_timer_getoverrun __ia32_sys_timer_getoverrun -263 i386 timer_delete sys_timer_delete __ia32_sys_timer_delete -264 i386 clock_settime sys_clock_settime32 __ia32_sys_clock_settime32 -265 i386 clock_gettime sys_clock_gettime32 __ia32_sys_clock_gettime32 -266 i386 clock_getres sys_clock_getres_time32 __ia32_sys_clock_getres_time32 -267 i386 clock_nanosleep sys_clock_nanosleep_time32 __ia32_sys_clock_nanosleep_time32 -268 i386 statfs64 sys_statfs64 __ia32_compat_sys_statfs64 -269 i386 fstatfs64 sys_fstatfs64 __ia32_compat_sys_fstatfs64 -270 i386 tgkill sys_tgkill __ia32_sys_tgkill -271 i386 utimes sys_utimes_time32 __ia32_sys_utimes_time32 -272 i386 fadvise64_64 sys_fadvise64_64 __ia32_compat_sys_x86_fadvise64_64 +252 i386 exit_group sys_exit_group +253 i386 lookup_dcookie sys_lookup_dcookie compat_sys_lookup_dcookie +254 i386 epoll_create sys_epoll_create +255 i386 epoll_ctl sys_epoll_ctl +256 i386 epoll_wait sys_epoll_wait +257 i386 remap_file_pages sys_remap_file_pages +258 i386 set_tid_address sys_set_tid_address +259 i386 timer_create sys_timer_create compat_sys_timer_create +260 i386 timer_settime sys_timer_settime32 +261 i386 timer_gettime sys_timer_gettime32 +262 i386 timer_getoverrun sys_timer_getoverrun +263 i386 timer_delete sys_timer_delete +264 i386 clock_settime sys_clock_settime32 +265 i386 clock_gettime sys_clock_gettime32 +266 i386 clock_getres sys_clock_getres_time32 +267 i386 clock_nanosleep sys_clock_nanosleep_time32 +268 i386 statfs64 sys_statfs64 compat_sys_statfs64 +269 i386 fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 +270 i386 tgkill sys_tgkill +271 i386 utimes sys_utimes_time32 +272 i386 fadvise64_64 sys_ia32_fadvise64_64 273 i386 vserver -274 i386 mbind sys_mbind __ia32_sys_mbind -275 i386 get_mempolicy sys_get_mempolicy __ia32_compat_sys_get_mempolicy -276 i386 set_mempolicy sys_set_mempolicy __ia32_sys_set_mempolicy -277 i386 mq_open sys_mq_open __ia32_compat_sys_mq_open -278 i386 mq_unlink sys_mq_unlink __ia32_sys_mq_unlink -279 i386 mq_timedsend sys_mq_timedsend_time32 __ia32_sys_mq_timedsend_time32 -280 i386 mq_timedreceive sys_mq_timedreceive_time32 __ia32_sys_mq_timedreceive_time32 -281 i386 mq_notify sys_mq_notify __ia32_compat_sys_mq_notify -282 i386 mq_getsetattr sys_mq_getsetattr __ia32_compat_sys_mq_getsetattr -283 i386 kexec_load sys_kexec_load __ia32_compat_sys_kexec_load -284 i386 waitid sys_waitid __ia32_compat_sys_waitid +274 i386 mbind sys_mbind +275 i386 get_mempolicy sys_get_mempolicy compat_sys_get_mempolicy +276 i386 set_mempolicy sys_set_mempolicy +277 i386 mq_open sys_mq_open compat_sys_mq_open +278 i386 mq_unlink sys_mq_unlink +279 i386 mq_timedsend sys_mq_timedsend_time32 +280 i386 mq_timedreceive sys_mq_timedreceive_time32 +281 i386 mq_notify sys_mq_notify compat_sys_mq_notify +282 i386 mq_getsetattr sys_mq_getsetattr compat_sys_mq_getsetattr +283 i386 kexec_load sys_kexec_load compat_sys_kexec_load +284 i386 waitid sys_waitid compat_sys_waitid # 285 sys_setaltroot -286 i386 add_key sys_add_key __ia32_sys_add_key -287 i386 request_key sys_request_key __ia32_sys_request_key -288 i386 keyctl sys_keyctl __ia32_compat_sys_keyctl -289 i386 ioprio_set sys_ioprio_set __ia32_sys_ioprio_set -290 i386 ioprio_get sys_ioprio_get __ia32_sys_ioprio_get -291 i386 inotify_init sys_inotify_init __ia32_sys_inotify_init -292 i386 inotify_add_watch sys_inotify_add_watch __ia32_sys_inotify_add_watch -293 i386 inotify_rm_watch sys_inotify_rm_watch __ia32_sys_inotify_rm_watch -294 i386 migrate_pages sys_migrate_pages __ia32_sys_migrate_pages -295 i386 openat sys_openat __ia32_compat_sys_openat -296 i386 mkdirat sys_mkdirat __ia32_sys_mkdirat -297 i386 mknodat sys_mknodat __ia32_sys_mknodat -298 i386 fchownat sys_fchownat __ia32_sys_fchownat -299 i386 futimesat sys_futimesat_time32 __ia32_sys_futimesat_time32 -300 i386 fstatat64 sys_fstatat64 __ia32_compat_sys_x86_fstatat -301 i386 unlinkat sys_unlinkat __ia32_sys_unlinkat -302 i386 renameat sys_renameat __ia32_sys_renameat -303 i386 linkat sys_linkat __ia32_sys_linkat -304 i386 symlinkat sys_symlinkat __ia32_sys_symlinkat -305 i386 readlinkat sys_readlinkat __ia32_sys_readlinkat -306 i386 fchmodat sys_fchmodat __ia32_sys_fchmodat -307 i386 faccessat sys_faccessat __ia32_sys_faccessat -308 i386 pselect6 sys_pselect6_time32 __ia32_compat_sys_pselect6_time32 -309 i386 ppoll sys_ppoll_time32 __ia32_compat_sys_ppoll_time32 -310 i386 unshare sys_unshare __ia32_sys_unshare -311 i386 set_robust_list sys_set_robust_list __ia32_compat_sys_set_robust_list -312 i386 get_robust_list sys_get_robust_list __ia32_compat_sys_get_robust_list -313 i386 splice sys_splice __ia32_sys_splice -314 i386 sync_file_range sys_sync_file_range __ia32_compat_sys_x86_sync_file_range -315 i386 tee sys_tee __ia32_sys_tee -316 i386 vmsplice sys_vmsplice __ia32_compat_sys_vmsplice -317 i386 move_pages sys_move_pages __ia32_compat_sys_move_pages -318 i386 getcpu sys_getcpu __ia32_sys_getcpu -319 i386 epoll_pwait sys_epoll_pwait __ia32_sys_epoll_pwait -320 i386 utimensat sys_utimensat_time32 __ia32_sys_utimensat_time32 -321 i386 signalfd sys_signalfd __ia32_compat_sys_signalfd -322 i386 timerfd_create sys_timerfd_create __ia32_sys_timerfd_create -323 i386 eventfd sys_eventfd __ia32_sys_eventfd -324 i386 fallocate sys_fallocate __ia32_compat_sys_x86_fallocate -325 i386 timerfd_settime sys_timerfd_settime32 __ia32_sys_timerfd_settime32 -326 i386 timerfd_gettime sys_timerfd_gettime32 __ia32_sys_timerfd_gettime32 -327 i386 signalfd4 sys_signalfd4 __ia32_compat_sys_signalfd4 -328 i386 eventfd2 sys_eventfd2 __ia32_sys_eventfd2 -329 i386 epoll_create1 sys_epoll_create1 __ia32_sys_epoll_create1 -330 i386 dup3 sys_dup3 __ia32_sys_dup3 -331 i386 pipe2 sys_pipe2 __ia32_sys_pipe2 -332 i386 inotify_init1 sys_inotify_init1 __ia32_sys_inotify_init1 -333 i386 preadv sys_preadv __ia32_compat_sys_preadv -334 i386 pwritev sys_pwritev __ia32_compat_sys_pwritev -335 i386 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo __ia32_compat_sys_rt_tgsigqueueinfo -336 i386 perf_event_open sys_perf_event_open __ia32_sys_perf_event_open -337 i386 recvmmsg sys_recvmmsg_time32 __ia32_compat_sys_recvmmsg_time32 -338 i386 fanotify_init sys_fanotify_init __ia32_sys_fanotify_init -339 i386 fanotify_mark sys_fanotify_mark __ia32_compat_sys_fanotify_mark -340 i386 prlimit64 sys_prlimit64 __ia32_sys_prlimit64 -341 i386 name_to_handle_at sys_name_to_handle_at __ia32_sys_name_to_handle_at -342 i386 open_by_handle_at sys_open_by_handle_at __ia32_compat_sys_open_by_handle_at -343 i386 clock_adjtime sys_clock_adjtime32 __ia32_sys_clock_adjtime32 -344 i386 syncfs sys_syncfs __ia32_sys_syncfs -345 i386 sendmmsg sys_sendmmsg __ia32_compat_sys_sendmmsg -346 i386 setns sys_setns __ia32_sys_setns -347 i386 process_vm_readv sys_process_vm_readv __ia32_compat_sys_process_vm_readv -348 i386 process_vm_writev sys_process_vm_writev __ia32_compat_sys_process_vm_writev -349 i386 kcmp sys_kcmp __ia32_sys_kcmp -350 i386 finit_module sys_finit_module __ia32_sys_finit_module -351 i386 sched_setattr sys_sched_setattr __ia32_sys_sched_setattr -352 i386 sched_getattr sys_sched_getattr __ia32_sys_sched_getattr -353 i386 renameat2 sys_renameat2 __ia32_sys_renameat2 -354 i386 seccomp sys_seccomp __ia32_sys_seccomp -355 i386 getrandom sys_getrandom __ia32_sys_getrandom -356 i386 memfd_create sys_memfd_create __ia32_sys_memfd_create -357 i386 bpf sys_bpf __ia32_sys_bpf -358 i386 execveat sys_execveat __ia32_compat_sys_execveat -359 i386 socket sys_socket __ia32_sys_socket -360 i386 socketpair sys_socketpair __ia32_sys_socketpair -361 i386 bind sys_bind __ia32_sys_bind -362 i386 connect sys_connect __ia32_sys_connect -363 i386 listen sys_listen __ia32_sys_listen -364 i386 accept4 sys_accept4 __ia32_sys_accept4 -365 i386 getsockopt sys_getsockopt __ia32_compat_sys_getsockopt -366 i386 setsockopt sys_setsockopt __ia32_compat_sys_setsockopt -367 i386 getsockname sys_getsockname __ia32_sys_getsockname -368 i386 getpeername sys_getpeername __ia32_sys_getpeername -369 i386 sendto sys_sendto __ia32_sys_sendto -370 i386 sendmsg sys_sendmsg __ia32_compat_sys_sendmsg -371 i386 recvfrom sys_recvfrom __ia32_compat_sys_recvfrom -372 i386 recvmsg sys_recvmsg __ia32_compat_sys_recvmsg -373 i386 shutdown sys_shutdown __ia32_sys_shutdown -374 i386 userfaultfd sys_userfaultfd __ia32_sys_userfaultfd -375 i386 membarrier sys_membarrier __ia32_sys_membarrier -376 i386 mlock2 sys_mlock2 __ia32_sys_mlock2 -377 i386 copy_file_range sys_copy_file_range __ia32_sys_copy_file_range -378 i386 preadv2 sys_preadv2 __ia32_compat_sys_preadv2 -379 i386 pwritev2 sys_pwritev2 __ia32_compat_sys_pwritev2 -380 i386 pkey_mprotect sys_pkey_mprotect __ia32_sys_pkey_mprotect -381 i386 pkey_alloc sys_pkey_alloc __ia32_sys_pkey_alloc -382 i386 pkey_free sys_pkey_free __ia32_sys_pkey_free -383 i386 statx sys_statx __ia32_sys_statx -384 i386 arch_prctl sys_arch_prctl __ia32_compat_sys_arch_prctl -385 i386 io_pgetevents sys_io_pgetevents_time32 __ia32_compat_sys_io_pgetevents -386 i386 rseq sys_rseq __ia32_sys_rseq -393 i386 semget sys_semget __ia32_sys_semget -394 i386 semctl sys_semctl __ia32_compat_sys_semctl -395 i386 shmget sys_shmget __ia32_sys_shmget -396 i386 shmctl sys_shmctl __ia32_compat_sys_shmctl -397 i386 shmat sys_shmat __ia32_compat_sys_shmat -398 i386 shmdt sys_shmdt __ia32_sys_shmdt -399 i386 msgget sys_msgget __ia32_sys_msgget -400 i386 msgsnd sys_msgsnd __ia32_compat_sys_msgsnd -401 i386 msgrcv sys_msgrcv __ia32_compat_sys_msgrcv -402 i386 msgctl sys_msgctl __ia32_compat_sys_msgctl -403 i386 clock_gettime64 sys_clock_gettime __ia32_sys_clock_gettime -404 i386 clock_settime64 sys_clock_settime __ia32_sys_clock_settime -405 i386 clock_adjtime64 sys_clock_adjtime __ia32_sys_clock_adjtime -406 i386 clock_getres_time64 sys_clock_getres __ia32_sys_clock_getres -407 i386 clock_nanosleep_time64 sys_clock_nanosleep __ia32_sys_clock_nanosleep -408 i386 timer_gettime64 sys_timer_gettime __ia32_sys_timer_gettime -409 i386 timer_settime64 sys_timer_settime __ia32_sys_timer_settime -410 i386 timerfd_gettime64 sys_timerfd_gettime __ia32_sys_timerfd_gettime -411 i386 timerfd_settime64 sys_timerfd_settime __ia32_sys_timerfd_settime -412 i386 utimensat_time64 sys_utimensat __ia32_sys_utimensat -413 i386 pselect6_time64 sys_pselect6 __ia32_compat_sys_pselect6_time64 -414 i386 ppoll_time64 sys_ppoll __ia32_compat_sys_ppoll_time64 -416 i386 io_pgetevents_time64 sys_io_pgetevents __ia32_sys_io_pgetevents -417 i386 recvmmsg_time64 sys_recvmmsg __ia32_compat_sys_recvmmsg_time64 -418 i386 mq_timedsend_time64 sys_mq_timedsend __ia32_sys_mq_timedsend -419 i386 mq_timedreceive_time64 sys_mq_timedreceive __ia32_sys_mq_timedreceive -420 i386 semtimedop_time64 sys_semtimedop __ia32_sys_semtimedop -421 i386 rt_sigtimedwait_time64 sys_rt_sigtimedwait __ia32_compat_sys_rt_sigtimedwait_time64 -422 i386 futex_time64 sys_futex __ia32_sys_futex -423 i386 sched_rr_get_interval_time64 sys_sched_rr_get_interval __ia32_sys_sched_rr_get_interval -424 i386 pidfd_send_signal sys_pidfd_send_signal __ia32_sys_pidfd_send_signal -425 i386 io_uring_setup sys_io_uring_setup __ia32_sys_io_uring_setup -426 i386 io_uring_enter sys_io_uring_enter __ia32_sys_io_uring_enter -427 i386 io_uring_register sys_io_uring_register __ia32_sys_io_uring_register -428 i386 open_tree sys_open_tree __ia32_sys_open_tree -429 i386 move_mount sys_move_mount __ia32_sys_move_mount -430 i386 fsopen sys_fsopen __ia32_sys_fsopen -431 i386 fsconfig sys_fsconfig __ia32_sys_fsconfig -432 i386 fsmount sys_fsmount __ia32_sys_fsmount -433 i386 fspick sys_fspick __ia32_sys_fspick -434 i386 pidfd_open sys_pidfd_open __ia32_sys_pidfd_open -435 i386 clone3 sys_clone3 __ia32_sys_clone3 -437 i386 openat2 sys_openat2 __ia32_sys_openat2 -438 i386 pidfd_getfd sys_pidfd_getfd __ia32_sys_pidfd_getfd +286 i386 add_key sys_add_key +287 i386 request_key sys_request_key +288 i386 keyctl sys_keyctl compat_sys_keyctl +289 i386 ioprio_set sys_ioprio_set +290 i386 ioprio_get sys_ioprio_get +291 i386 inotify_init sys_inotify_init +292 i386 inotify_add_watch sys_inotify_add_watch +293 i386 inotify_rm_watch sys_inotify_rm_watch +294 i386 migrate_pages sys_migrate_pages +295 i386 openat sys_openat compat_sys_openat +296 i386 mkdirat sys_mkdirat +297 i386 mknodat sys_mknodat +298 i386 fchownat sys_fchownat +299 i386 futimesat sys_futimesat_time32 +300 i386 fstatat64 sys_fstatat64 compat_sys_ia32_fstatat64 +301 i386 unlinkat sys_unlinkat +302 i386 renameat sys_renameat +303 i386 linkat sys_linkat +304 i386 symlinkat sys_symlinkat +305 i386 readlinkat sys_readlinkat +306 i386 fchmodat sys_fchmodat +307 i386 faccessat sys_faccessat +308 i386 pselect6 sys_pselect6_time32 compat_sys_pselect6_time32 +309 i386 ppoll sys_ppoll_time32 compat_sys_ppoll_time32 +310 i386 unshare sys_unshare +311 i386 set_robust_list sys_set_robust_list compat_sys_set_robust_list +312 i386 get_robust_list sys_get_robust_list compat_sys_get_robust_list +313 i386 splice sys_splice +314 i386 sync_file_range sys_ia32_sync_file_range +315 i386 tee sys_tee +316 i386 vmsplice sys_vmsplice compat_sys_vmsplice +317 i386 move_pages sys_move_pages compat_sys_move_pages +318 i386 getcpu sys_getcpu +319 i386 epoll_pwait sys_epoll_pwait +320 i386 utimensat sys_utimensat_time32 +321 i386 signalfd sys_signalfd compat_sys_signalfd +322 i386 timerfd_create sys_timerfd_create +323 i386 eventfd sys_eventfd +324 i386 fallocate sys_ia32_fallocate +325 i386 timerfd_settime sys_timerfd_settime32 +326 i386 timerfd_gettime sys_timerfd_gettime32 +327 i386 signalfd4 sys_signalfd4 compat_sys_signalfd4 +328 i386 eventfd2 sys_eventfd2 +329 i386 epoll_create1 sys_epoll_create1 +330 i386 dup3 sys_dup3 +331 i386 pipe2 sys_pipe2 +332 i386 inotify_init1 sys_inotify_init1 +333 i386 preadv sys_preadv compat_sys_preadv +334 i386 pwritev sys_pwritev compat_sys_pwritev +335 i386 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +336 i386 perf_event_open sys_perf_event_open +337 i386 recvmmsg sys_recvmmsg_time32 compat_sys_recvmmsg_time32 +338 i386 fanotify_init sys_fanotify_init +339 i386 fanotify_mark sys_fanotify_mark compat_sys_fanotify_mark +340 i386 prlimit64 sys_prlimit64 +341 i386 name_to_handle_at sys_name_to_handle_at +342 i386 open_by_handle_at sys_open_by_handle_at compat_sys_open_by_handle_at +343 i386 clock_adjtime sys_clock_adjtime32 +344 i386 syncfs sys_syncfs +345 i386 sendmmsg sys_sendmmsg compat_sys_sendmmsg +346 i386 setns sys_setns +347 i386 process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv +348 i386 process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev +349 i386 kcmp sys_kcmp +350 i386 finit_module sys_finit_module +351 i386 sched_setattr sys_sched_setattr +352 i386 sched_getattr sys_sched_getattr +353 i386 renameat2 sys_renameat2 +354 i386 seccomp sys_seccomp +355 i386 getrandom sys_getrandom +356 i386 memfd_create sys_memfd_create +357 i386 bpf sys_bpf +358 i386 execveat sys_execveat compat_sys_execveat +359 i386 socket sys_socket +360 i386 socketpair sys_socketpair +361 i386 bind sys_bind +362 i386 connect sys_connect +363 i386 listen sys_listen +364 i386 accept4 sys_accept4 +365 i386 getsockopt sys_getsockopt compat_sys_getsockopt +366 i386 setsockopt sys_setsockopt compat_sys_setsockopt +367 i386 getsockname sys_getsockname +368 i386 getpeername sys_getpeername +369 i386 sendto sys_sendto +370 i386 sendmsg sys_sendmsg compat_sys_sendmsg +371 i386 recvfrom sys_recvfrom compat_sys_recvfrom +372 i386 recvmsg sys_recvmsg compat_sys_recvmsg +373 i386 shutdown sys_shutdown +374 i386 userfaultfd sys_userfaultfd +375 i386 membarrier sys_membarrier +376 i386 mlock2 sys_mlock2 +377 i386 copy_file_range sys_copy_file_range +378 i386 preadv2 sys_preadv2 compat_sys_preadv2 +379 i386 pwritev2 sys_pwritev2 compat_sys_pwritev2 +380 i386 pkey_mprotect sys_pkey_mprotect +381 i386 pkey_alloc sys_pkey_alloc +382 i386 pkey_free sys_pkey_free +383 i386 statx sys_statx +384 i386 arch_prctl sys_arch_prctl compat_sys_arch_prctl +385 i386 io_pgetevents sys_io_pgetevents_time32 compat_sys_io_pgetevents +386 i386 rseq sys_rseq +393 i386 semget sys_semget +394 i386 semctl sys_semctl compat_sys_semctl +395 i386 shmget sys_shmget +396 i386 shmctl sys_shmctl compat_sys_shmctl +397 i386 shmat sys_shmat compat_sys_shmat +398 i386 shmdt sys_shmdt +399 i386 msgget sys_msgget +400 i386 msgsnd sys_msgsnd compat_sys_msgsnd +401 i386 msgrcv sys_msgrcv compat_sys_msgrcv +402 i386 msgctl sys_msgctl compat_sys_msgctl +403 i386 clock_gettime64 sys_clock_gettime +404 i386 clock_settime64 sys_clock_settime +405 i386 clock_adjtime64 sys_clock_adjtime +406 i386 clock_getres_time64 sys_clock_getres +407 i386 clock_nanosleep_time64 sys_clock_nanosleep +408 i386 timer_gettime64 sys_timer_gettime +409 i386 timer_settime64 sys_timer_settime +410 i386 timerfd_gettime64 sys_timerfd_gettime +411 i386 timerfd_settime64 sys_timerfd_settime +412 i386 utimensat_time64 sys_utimensat +413 i386 pselect6_time64 sys_pselect6 compat_sys_pselect6_time64 +414 i386 ppoll_time64 sys_ppoll compat_sys_ppoll_time64 +416 i386 io_pgetevents_time64 sys_io_pgetevents +417 i386 recvmmsg_time64 sys_recvmmsg compat_sys_recvmmsg_time64 +418 i386 mq_timedsend_time64 sys_mq_timedsend +419 i386 mq_timedreceive_time64 sys_mq_timedreceive +420 i386 semtimedop_time64 sys_semtimedop +421 i386 rt_sigtimedwait_time64 sys_rt_sigtimedwait compat_sys_rt_sigtimedwait_time64 +422 i386 futex_time64 sys_futex +423 i386 sched_rr_get_interval_time64 sys_sched_rr_get_interval +424 i386 pidfd_send_signal sys_pidfd_send_signal +425 i386 io_uring_setup sys_io_uring_setup +426 i386 io_uring_enter sys_io_uring_enter +427 i386 io_uring_register sys_io_uring_register +428 i386 open_tree sys_open_tree +429 i386 move_mount sys_move_mount +430 i386 fsopen sys_fsopen +431 i386 fsconfig sys_fsconfig +432 i386 fsmount sys_fsmount +433 i386 fspick sys_fspick +434 i386 pidfd_open sys_pidfd_open +435 i386 clone3 sys_clone3 +437 i386 openat2 sys_openat2 +438 i386 pidfd_getfd sys_pidfd_getfd diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 44d510bc9b78..37b844f839bc 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -8,357 +8,357 @@ # # The abi is "common", "64" or "x32" for this file. # -0 common read __x64_sys_read -1 common write __x64_sys_write -2 common open __x64_sys_open -3 common close __x64_sys_close -4 common stat __x64_sys_newstat -5 common fstat __x64_sys_newfstat -6 common lstat __x64_sys_newlstat -7 common poll __x64_sys_poll -8 common lseek __x64_sys_lseek -9 common mmap __x64_sys_mmap -10 common mprotect __x64_sys_mprotect -11 common munmap __x64_sys_munmap -12 common brk __x64_sys_brk -13 64 rt_sigaction __x64_sys_rt_sigaction -14 common rt_sigprocmask __x64_sys_rt_sigprocmask -15 64 rt_sigreturn __x64_sys_rt_sigreturn/ptregs -16 64 ioctl __x64_sys_ioctl -17 common pread64 __x64_sys_pread64 -18 common pwrite64 __x64_sys_pwrite64 -19 64 readv __x64_sys_readv -20 64 writev __x64_sys_writev -21 common access __x64_sys_access -22 common pipe __x64_sys_pipe -23 common select __x64_sys_select -24 common sched_yield __x64_sys_sched_yield -25 common mremap __x64_sys_mremap -26 common msync __x64_sys_msync -27 common mincore __x64_sys_mincore -28 common madvise __x64_sys_madvise -29 common shmget __x64_sys_shmget -30 common shmat __x64_sys_shmat -31 common shmctl __x64_sys_shmctl -32 common dup __x64_sys_dup -33 common dup2 __x64_sys_dup2 -34 common pause __x64_sys_pause -35 common nanosleep __x64_sys_nanosleep -36 common getitimer __x64_sys_getitimer -37 common alarm __x64_sys_alarm -38 common setitimer __x64_sys_setitimer -39 common getpid __x64_sys_getpid -40 common sendfile __x64_sys_sendfile64 -41 common socket __x64_sys_socket -42 common connect __x64_sys_connect -43 common accept __x64_sys_accept -44 common sendto __x64_sys_sendto -45 64 recvfrom __x64_sys_recvfrom -46 64 sendmsg __x64_sys_sendmsg -47 64 recvmsg __x64_sys_recvmsg -48 common shutdown __x64_sys_shutdown -49 common bind __x64_sys_bind -50 common listen __x64_sys_listen -51 common getsockname __x64_sys_getsockname -52 common getpeername __x64_sys_getpeername -53 common socketpair __x64_sys_socketpair -54 64 setsockopt __x64_sys_setsockopt -55 64 getsockopt __x64_sys_getsockopt -56 common clone __x64_sys_clone/ptregs -57 common fork __x64_sys_fork/ptregs -58 common vfork __x64_sys_vfork/ptregs -59 64 execve __x64_sys_execve/ptregs -60 common exit __x64_sys_exit -61 common wait4 __x64_sys_wait4 -62 common kill __x64_sys_kill -63 common uname __x64_sys_newuname -64 common semget __x64_sys_semget -65 common semop __x64_sys_semop -66 common semctl __x64_sys_semctl -67 common shmdt __x64_sys_shmdt -68 common msgget __x64_sys_msgget -69 common msgsnd __x64_sys_msgsnd -70 common msgrcv __x64_sys_msgrcv -71 common msgctl __x64_sys_msgctl -72 common fcntl __x64_sys_fcntl -73 common flock __x64_sys_flock -74 common fsync __x64_sys_fsync -75 common fdatasync __x64_sys_fdatasync -76 common truncate __x64_sys_truncate -77 common ftruncate __x64_sys_ftruncate -78 common getdents __x64_sys_getdents -79 common getcwd __x64_sys_getcwd -80 common chdir __x64_sys_chdir -81 common fchdir __x64_sys_fchdir -82 common rename __x64_sys_rename -83 common mkdir __x64_sys_mkdir -84 common rmdir __x64_sys_rmdir -85 common creat __x64_sys_creat -86 common link __x64_sys_link -87 common unlink __x64_sys_unlink -88 common symlink __x64_sys_symlink -89 common readlink __x64_sys_readlink -90 common chmod __x64_sys_chmod -91 common fchmod __x64_sys_fchmod -92 common chown __x64_sys_chown -93 common fchown __x64_sys_fchown -94 common lchown __x64_sys_lchown -95 common umask __x64_sys_umask -96 common gettimeofday __x64_sys_gettimeofday -97 common getrlimit __x64_sys_getrlimit -98 common getrusage __x64_sys_getrusage -99 common sysinfo __x64_sys_sysinfo -100 common times __x64_sys_times -101 64 ptrace __x64_sys_ptrace -102 common getuid __x64_sys_getuid -103 common syslog __x64_sys_syslog -104 common getgid __x64_sys_getgid -105 common setuid __x64_sys_setuid -106 common setgid __x64_sys_setgid -107 common geteuid __x64_sys_geteuid -108 common getegid __x64_sys_getegid -109 common setpgid __x64_sys_setpgid -110 common getppid __x64_sys_getppid -111 common getpgrp __x64_sys_getpgrp -112 common setsid __x64_sys_setsid -113 common setreuid __x64_sys_setreuid -114 common setregid __x64_sys_setregid -115 common getgroups __x64_sys_getgroups -116 common setgroups __x64_sys_setgroups -117 common setresuid __x64_sys_setresuid -118 common getresuid __x64_sys_getresuid -119 common setresgid __x64_sys_setresgid -120 common getresgid __x64_sys_getresgid -121 common getpgid __x64_sys_getpgid -122 common setfsuid __x64_sys_setfsuid -123 common setfsgid __x64_sys_setfsgid -124 common getsid __x64_sys_getsid -125 common capget __x64_sys_capget -126 common capset __x64_sys_capset -127 64 rt_sigpending __x64_sys_rt_sigpending -128 64 rt_sigtimedwait __x64_sys_rt_sigtimedwait -129 64 rt_sigqueueinfo __x64_sys_rt_sigqueueinfo -130 common rt_sigsuspend __x64_sys_rt_sigsuspend -131 64 sigaltstack __x64_sys_sigaltstack -132 common utime __x64_sys_utime -133 common mknod __x64_sys_mknod +0 common read sys_read +1 common write sys_write +2 common open sys_open +3 common close sys_close +4 common stat sys_newstat +5 common fstat sys_newfstat +6 common lstat sys_newlstat +7 common poll sys_poll +8 common lseek sys_lseek +9 common mmap sys_mmap +10 common mprotect sys_mprotect +11 common munmap sys_munmap +12 common brk sys_brk +13 64 rt_sigaction sys_rt_sigaction +14 common rt_sigprocmask sys_rt_sigprocmask +15 64 rt_sigreturn sys_rt_sigreturn +16 64 ioctl sys_ioctl +17 common pread64 sys_pread64 +18 common pwrite64 sys_pwrite64 +19 64 readv sys_readv +20 64 writev sys_writev +21 common access sys_access +22 common pipe sys_pipe +23 common select sys_select +24 common sched_yield sys_sched_yield +25 common mremap sys_mremap +26 common msync sys_msync +27 common mincore sys_mincore +28 common madvise sys_madvise +29 common shmget sys_shmget +30 common shmat sys_shmat +31 common shmctl sys_shmctl +32 common dup sys_dup +33 common dup2 sys_dup2 +34 common pause sys_pause +35 common nanosleep sys_nanosleep +36 common getitimer sys_getitimer +37 common alarm sys_alarm +38 common setitimer sys_setitimer +39 common getpid sys_getpid +40 common sendfile sys_sendfile64 +41 common socket sys_socket +42 common connect sys_connect +43 common accept sys_accept +44 common sendto sys_sendto +45 64 recvfrom sys_recvfrom +46 64 sendmsg sys_sendmsg +47 64 recvmsg sys_recvmsg +48 common shutdown sys_shutdown +49 common bind sys_bind +50 common listen sys_listen +51 common getsockname sys_getsockname +52 common getpeername sys_getpeername +53 common socketpair sys_socketpair +54 64 setsockopt sys_setsockopt +55 64 getsockopt sys_getsockopt +56 common clone sys_clone +57 common fork sys_fork +58 common vfork sys_vfork +59 64 execve sys_execve +60 common exit sys_exit +61 common wait4 sys_wait4 +62 common kill sys_kill +63 common uname sys_newuname +64 common semget sys_semget +65 common semop sys_semop +66 common semctl sys_semctl +67 common shmdt sys_shmdt +68 common msgget sys_msgget +69 common msgsnd sys_msgsnd +70 common msgrcv sys_msgrcv +71 common msgctl sys_msgctl +72 common fcntl sys_fcntl +73 common flock sys_flock +74 common fsync sys_fsync +75 common fdatasync sys_fdatasync +76 common truncate sys_truncate +77 common ftruncate sys_ftruncate +78 common getdents sys_getdents +79 common getcwd sys_getcwd +80 common chdir sys_chdir +81 common fchdir sys_fchdir +82 common rename sys_rename +83 common mkdir sys_mkdir +84 common rmdir sys_rmdir +85 common creat sys_creat +86 common link sys_link +87 common unlink sys_unlink +88 common symlink sys_symlink +89 common readlink sys_readlink +90 common chmod sys_chmod +91 common fchmod sys_fchmod +92 common chown sys_chown +93 common fchown sys_fchown +94 common lchown sys_lchown +95 common umask sys_umask +96 common gettimeofday sys_gettimeofday +97 common getrlimit sys_getrlimit +98 common getrusage sys_getrusage +99 common sysinfo sys_sysinfo +100 common times sys_times +101 64 ptrace sys_ptrace +102 common getuid sys_getuid +103 common syslog sys_syslog +104 common getgid sys_getgid +105 common setuid sys_setuid +106 common setgid sys_setgid +107 common geteuid sys_geteuid +108 common getegid sys_getegid +109 common setpgid sys_setpgid +110 common getppid sys_getppid +111 common getpgrp sys_getpgrp +112 common setsid sys_setsid +113 common setreuid sys_setreuid +114 common setregid sys_setregid +115 common getgroups sys_getgroups +116 common setgroups sys_setgroups +117 common setresuid sys_setresuid +118 common getresuid sys_getresuid +119 common setresgid sys_setresgid +120 common getresgid sys_getresgid +121 common getpgid sys_getpgid +122 common setfsuid sys_setfsuid +123 common setfsgid sys_setfsgid +124 common getsid sys_getsid +125 common capget sys_capget +126 common capset sys_capset +127 64 rt_sigpending sys_rt_sigpending +128 64 rt_sigtimedwait sys_rt_sigtimedwait +129 64 rt_sigqueueinfo sys_rt_sigqueueinfo +130 common rt_sigsuspend sys_rt_sigsuspend +131 64 sigaltstack sys_sigaltstack +132 common utime sys_utime +133 common mknod sys_mknod 134 64 uselib -135 common personality __x64_sys_personality -136 common ustat __x64_sys_ustat -137 common statfs __x64_sys_statfs -138 common fstatfs __x64_sys_fstatfs -139 common sysfs __x64_sys_sysfs -140 common getpriority __x64_sys_getpriority -141 common setpriority __x64_sys_setpriority -142 common sched_setparam __x64_sys_sched_setparam -143 common sched_getparam __x64_sys_sched_getparam -144 common sched_setscheduler __x64_sys_sched_setscheduler -145 common sched_getscheduler __x64_sys_sched_getscheduler -146 common sched_get_priority_max __x64_sys_sched_get_priority_max -147 common sched_get_priority_min __x64_sys_sched_get_priority_min -148 common sched_rr_get_interval __x64_sys_sched_rr_get_interval -149 common mlock __x64_sys_mlock -150 common munlock __x64_sys_munlock -151 common mlockall __x64_sys_mlockall -152 common munlockall __x64_sys_munlockall -153 common vhangup __x64_sys_vhangup -154 common modify_ldt __x64_sys_modify_ldt -155 common pivot_root __x64_sys_pivot_root -156 64 _sysctl __x64_sys_sysctl -157 common prctl __x64_sys_prctl -158 common arch_prctl __x64_sys_arch_prctl -159 common adjtimex __x64_sys_adjtimex -160 common setrlimit __x64_sys_setrlimit -161 common chroot __x64_sys_chroot -162 common sync __x64_sys_sync -163 common acct __x64_sys_acct -164 common settimeofday __x64_sys_settimeofday -165 common mount __x64_sys_mount -166 common umount2 __x64_sys_umount -167 common swapon __x64_sys_swapon -168 common swapoff __x64_sys_swapoff -169 common reboot __x64_sys_reboot -170 common sethostname __x64_sys_sethostname -171 common setdomainname __x64_sys_setdomainname -172 common iopl __x64_sys_iopl/ptregs -173 common ioperm __x64_sys_ioperm +135 common personality sys_personality +136 common ustat sys_ustat +137 common statfs sys_statfs +138 common fstatfs sys_fstatfs +139 common sysfs sys_sysfs +140 common getpriority sys_getpriority +141 common setpriority sys_setpriority +142 common sched_setparam sys_sched_setparam +143 common sched_getparam sys_sched_getparam +144 common sched_setscheduler sys_sched_setscheduler +145 common sched_getscheduler sys_sched_getscheduler +146 common sched_get_priority_max sys_sched_get_priority_max +147 common sched_get_priority_min sys_sched_get_priority_min +148 common sched_rr_get_interval sys_sched_rr_get_interval +149 common mlock sys_mlock +150 common munlock sys_munlock +151 common mlockall sys_mlockall +152 common munlockall sys_munlockall +153 common vhangup sys_vhangup +154 common modify_ldt sys_modify_ldt +155 common pivot_root sys_pivot_root +156 64 _sysctl sys_sysctl +157 common prctl sys_prctl +158 common arch_prctl sys_arch_prctl +159 common adjtimex sys_adjtimex +160 common setrlimit sys_setrlimit +161 common chroot sys_chroot +162 common sync sys_sync +163 common acct sys_acct +164 common settimeofday sys_settimeofday +165 common mount sys_mount +166 common umount2 sys_umount +167 common swapon sys_swapon +168 common swapoff sys_swapoff +169 common reboot sys_reboot +170 common sethostname sys_sethostname +171 common setdomainname sys_setdomainname +172 common iopl sys_iopl +173 common ioperm sys_ioperm 174 64 create_module -175 common init_module __x64_sys_init_module -176 common delete_module __x64_sys_delete_module +175 common init_module sys_init_module +176 common delete_module sys_delete_module 177 64 get_kernel_syms 178 64 query_module -179 common quotactl __x64_sys_quotactl +179 common quotactl sys_quotactl 180 64 nfsservctl 181 common getpmsg 182 common putpmsg 183 common afs_syscall 184 common tuxcall 185 common security -186 common gettid __x64_sys_gettid -187 common readahead __x64_sys_readahead -188 common setxattr __x64_sys_setxattr -189 common lsetxattr __x64_sys_lsetxattr -190 common fsetxattr __x64_sys_fsetxattr -191 common getxattr __x64_sys_getxattr -192 common lgetxattr __x64_sys_lgetxattr -193 common fgetxattr __x64_sys_fgetxattr -194 common listxattr __x64_sys_listxattr -195 common llistxattr __x64_sys_llistxattr -196 common flistxattr __x64_sys_flistxattr -197 common removexattr __x64_sys_removexattr -198 common lremovexattr __x64_sys_lremovexattr -199 common fremovexattr __x64_sys_fremovexattr -200 common tkill __x64_sys_tkill -201 common time __x64_sys_time -202 common futex __x64_sys_futex -203 common sched_setaffinity __x64_sys_sched_setaffinity -204 common sched_getaffinity __x64_sys_sched_getaffinity +186 common gettid sys_gettid +187 common readahead sys_readahead +188 common setxattr sys_setxattr +189 common lsetxattr sys_lsetxattr +190 common fsetxattr sys_fsetxattr +191 common getxattr sys_getxattr +192 common lgetxattr sys_lgetxattr +193 common fgetxattr sys_fgetxattr +194 common listxattr sys_listxattr +195 common llistxattr sys_llistxattr +196 common flistxattr sys_flistxattr +197 common removexattr sys_removexattr +198 common lremovexattr sys_lremovexattr +199 common fremovexattr sys_fremovexattr +200 common tkill sys_tkill +201 common time sys_time +202 common futex sys_futex +203 common sched_setaffinity sys_sched_setaffinity +204 common sched_getaffinity sys_sched_getaffinity 205 64 set_thread_area -206 64 io_setup __x64_sys_io_setup -207 common io_destroy __x64_sys_io_destroy -208 common io_getevents __x64_sys_io_getevents -209 64 io_submit __x64_sys_io_submit -210 common io_cancel __x64_sys_io_cancel +206 64 io_setup sys_io_setup +207 common io_destroy sys_io_destroy +208 common io_getevents sys_io_getevents +209 64 io_submit sys_io_submit +210 common io_cancel sys_io_cancel 211 64 get_thread_area -212 common lookup_dcookie __x64_sys_lookup_dcookie -213 common epoll_create __x64_sys_epoll_create +212 common lookup_dcookie sys_lookup_dcookie +213 common epoll_create sys_epoll_create 214 64 epoll_ctl_old 215 64 epoll_wait_old -216 common remap_file_pages __x64_sys_remap_file_pages -217 common getdents64 __x64_sys_getdents64 -218 common set_tid_address __x64_sys_set_tid_address -219 common restart_syscall __x64_sys_restart_syscall -220 common semtimedop __x64_sys_semtimedop -221 common fadvise64 __x64_sys_fadvise64 -222 64 timer_create __x64_sys_timer_create -223 common timer_settime __x64_sys_timer_settime -224 common timer_gettime __x64_sys_timer_gettime -225 common timer_getoverrun __x64_sys_timer_getoverrun -226 common timer_delete __x64_sys_timer_delete -227 common clock_settime __x64_sys_clock_settime -228 common clock_gettime __x64_sys_clock_gettime -229 common clock_getres __x64_sys_clock_getres -230 common clock_nanosleep __x64_sys_clock_nanosleep -231 common exit_group __x64_sys_exit_group -232 common epoll_wait __x64_sys_epoll_wait -233 common epoll_ctl __x64_sys_epoll_ctl -234 common tgkill __x64_sys_tgkill -235 common utimes __x64_sys_utimes +216 common remap_file_pages sys_remap_file_pages +217 common getdents64 sys_getdents64 +218 common set_tid_address sys_set_tid_address +219 common restart_syscall sys_restart_syscall +220 common semtimedop sys_semtimedop +221 common fadvise64 sys_fadvise64 +222 64 timer_create sys_timer_create +223 common timer_settime sys_timer_settime +224 common timer_gettime sys_timer_gettime +225 common timer_getoverrun sys_timer_getoverrun +226 common timer_delete sys_timer_delete +227 common clock_settime sys_clock_settime +228 common clock_gettime sys_clock_gettime +229 common clock_getres sys_clock_getres +230 common clock_nanosleep sys_clock_nanosleep +231 common exit_group sys_exit_group +232 common epoll_wait sys_epoll_wait +233 common epoll_ctl sys_epoll_ctl +234 common tgkill sys_tgkill +235 common utimes sys_utimes 236 64 vserver -237 common mbind __x64_sys_mbind -238 common set_mempolicy __x64_sys_set_mempolicy -239 common get_mempolicy __x64_sys_get_mempolicy -240 common mq_open __x64_sys_mq_open -241 common mq_unlink __x64_sys_mq_unlink -242 common mq_timedsend __x64_sys_mq_timedsend -243 common mq_timedreceive __x64_sys_mq_timedreceive -244 64 mq_notify __x64_sys_mq_notify -245 common mq_getsetattr __x64_sys_mq_getsetattr -246 64 kexec_load __x64_sys_kexec_load -247 64 waitid __x64_sys_waitid -248 common add_key __x64_sys_add_key -249 common request_key __x64_sys_request_key -250 common keyctl __x64_sys_keyctl -251 common ioprio_set __x64_sys_ioprio_set -252 common ioprio_get __x64_sys_ioprio_get -253 common inotify_init __x64_sys_inotify_init -254 common inotify_add_watch __x64_sys_inotify_add_watch -255 common inotify_rm_watch __x64_sys_inotify_rm_watch -256 common migrate_pages __x64_sys_migrate_pages -257 common openat __x64_sys_openat -258 common mkdirat __x64_sys_mkdirat -259 common mknodat __x64_sys_mknodat -260 common fchownat __x64_sys_fchownat -261 common futimesat __x64_sys_futimesat -262 common newfstatat __x64_sys_newfstatat -263 common unlinkat __x64_sys_unlinkat -264 common renameat __x64_sys_renameat -265 common linkat __x64_sys_linkat -266 common symlinkat __x64_sys_symlinkat -267 common readlinkat __x64_sys_readlinkat -268 common fchmodat __x64_sys_fchmodat -269 common faccessat __x64_sys_faccessat -270 common pselect6 __x64_sys_pselect6 -271 common ppoll __x64_sys_ppoll -272 common unshare __x64_sys_unshare -273 64 set_robust_list __x64_sys_set_robust_list -274 64 get_robust_list __x64_sys_get_robust_list -275 common splice __x64_sys_splice -276 common tee __x64_sys_tee -277 common sync_file_range __x64_sys_sync_file_range -278 64 vmsplice __x64_sys_vmsplice -279 64 move_pages __x64_sys_move_pages -280 common utimensat __x64_sys_utimensat -281 common epoll_pwait __x64_sys_epoll_pwait -282 common signalfd __x64_sys_signalfd -283 common timerfd_create __x64_sys_timerfd_create -284 common eventfd __x64_sys_eventfd -285 common fallocate __x64_sys_fallocate -286 common timerfd_settime __x64_sys_timerfd_settime -287 common timerfd_gettime __x64_sys_timerfd_gettime -288 common accept4 __x64_sys_accept4 -289 common signalfd4 __x64_sys_signalfd4 -290 common eventfd2 __x64_sys_eventfd2 -291 common epoll_create1 __x64_sys_epoll_create1 -292 common dup3 __x64_sys_dup3 -293 common pipe2 __x64_sys_pipe2 -294 common inotify_init1 __x64_sys_inotify_init1 -295 64 preadv __x64_sys_preadv -296 64 pwritev __x64_sys_pwritev -297 64 rt_tgsigqueueinfo __x64_sys_rt_tgsigqueueinfo -298 common perf_event_open __x64_sys_perf_event_open -299 64 recvmmsg __x64_sys_recvmmsg -300 common fanotify_init __x64_sys_fanotify_init -301 common fanotify_mark __x64_sys_fanotify_mark -302 common prlimit64 __x64_sys_prlimit64 -303 common name_to_handle_at __x64_sys_name_to_handle_at -304 common open_by_handle_at __x64_sys_open_by_handle_at -305 common clock_adjtime __x64_sys_clock_adjtime -306 common syncfs __x64_sys_syncfs -307 64 sendmmsg __x64_sys_sendmmsg -308 common setns __x64_sys_setns -309 common getcpu __x64_sys_getcpu -310 64 process_vm_readv __x64_sys_process_vm_readv -311 64 process_vm_writev __x64_sys_process_vm_writev -312 common kcmp __x64_sys_kcmp -313 common finit_module __x64_sys_finit_module -314 common sched_setattr __x64_sys_sched_setattr -315 common sched_getattr __x64_sys_sched_getattr -316 common renameat2 __x64_sys_renameat2 -317 common seccomp __x64_sys_seccomp -318 common getrandom __x64_sys_getrandom -319 common memfd_create __x64_sys_memfd_create -320 common kexec_file_load __x64_sys_kexec_file_load -321 common bpf __x64_sys_bpf -322 64 execveat __x64_sys_execveat/ptregs -323 common userfaultfd __x64_sys_userfaultfd -324 common membarrier __x64_sys_membarrier -325 common mlock2 __x64_sys_mlock2 -326 common copy_file_range __x64_sys_copy_file_range -327 64 preadv2 __x64_sys_preadv2 -328 64 pwritev2 __x64_sys_pwritev2 -329 common pkey_mprotect __x64_sys_pkey_mprotect -330 common pkey_alloc __x64_sys_pkey_alloc -331 common pkey_free __x64_sys_pkey_free -332 common statx __x64_sys_statx -333 common io_pgetevents __x64_sys_io_pgetevents -334 common rseq __x64_sys_rseq +237 common mbind sys_mbind +238 common set_mempolicy sys_set_mempolicy +239 common get_mempolicy sys_get_mempolicy +240 common mq_open sys_mq_open +241 common mq_unlink sys_mq_unlink +242 common mq_timedsend sys_mq_timedsend +243 common mq_timedreceive sys_mq_timedreceive +244 64 mq_notify sys_mq_notify +245 common mq_getsetattr sys_mq_getsetattr +246 64 kexec_load sys_kexec_load +247 64 waitid sys_waitid +248 common add_key sys_add_key +249 common request_key sys_request_key +250 common keyctl sys_keyctl +251 common ioprio_set sys_ioprio_set +252 common ioprio_get sys_ioprio_get +253 common inotify_init sys_inotify_init +254 common inotify_add_watch sys_inotify_add_watch +255 common inotify_rm_watch sys_inotify_rm_watch +256 common migrate_pages sys_migrate_pages +257 common openat sys_openat +258 common mkdirat sys_mkdirat +259 common mknodat sys_mknodat +260 common fchownat sys_fchownat +261 common futimesat sys_futimesat +262 common newfstatat sys_newfstatat +263 common unlinkat sys_unlinkat +264 common renameat sys_renameat +265 common linkat sys_linkat +266 common symlinkat sys_symlinkat +267 common readlinkat sys_readlinkat +268 common fchmodat sys_fchmodat +269 common faccessat sys_faccessat +270 common pselect6 sys_pselect6 +271 common ppoll sys_ppoll +272 common unshare sys_unshare +273 64 set_robust_list sys_set_robust_list +274 64 get_robust_list sys_get_robust_list +275 common splice sys_splice +276 common tee sys_tee +277 common sync_file_range sys_sync_file_range +278 64 vmsplice sys_vmsplice +279 64 move_pages sys_move_pages +280 common utimensat sys_utimensat +281 common epoll_pwait sys_epoll_pwait +282 common signalfd sys_signalfd +283 common timerfd_create sys_timerfd_create +284 common eventfd sys_eventfd +285 common fallocate sys_fallocate +286 common timerfd_settime sys_timerfd_settime +287 common timerfd_gettime sys_timerfd_gettime +288 common accept4 sys_accept4 +289 common signalfd4 sys_signalfd4 +290 common eventfd2 sys_eventfd2 +291 common epoll_create1 sys_epoll_create1 +292 common dup3 sys_dup3 +293 common pipe2 sys_pipe2 +294 common inotify_init1 sys_inotify_init1 +295 64 preadv sys_preadv +296 64 pwritev sys_pwritev +297 64 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo +298 common perf_event_open sys_perf_event_open +299 64 recvmmsg sys_recvmmsg +300 common fanotify_init sys_fanotify_init +301 common fanotify_mark sys_fanotify_mark +302 common prlimit64 sys_prlimit64 +303 common name_to_handle_at sys_name_to_handle_at +304 common open_by_handle_at sys_open_by_handle_at +305 common clock_adjtime sys_clock_adjtime +306 common syncfs sys_syncfs +307 64 sendmmsg sys_sendmmsg +308 common setns sys_setns +309 common getcpu sys_getcpu +310 64 process_vm_readv sys_process_vm_readv +311 64 process_vm_writev sys_process_vm_writev +312 common kcmp sys_kcmp +313 common finit_module sys_finit_module +314 common sched_setattr sys_sched_setattr +315 common sched_getattr sys_sched_getattr +316 common renameat2 sys_renameat2 +317 common seccomp sys_seccomp +318 common getrandom sys_getrandom +319 common memfd_create sys_memfd_create +320 common kexec_file_load sys_kexec_file_load +321 common bpf sys_bpf +322 64 execveat sys_execveat +323 common userfaultfd sys_userfaultfd +324 common membarrier sys_membarrier +325 common mlock2 sys_mlock2 +326 common copy_file_range sys_copy_file_range +327 64 preadv2 sys_preadv2 +328 64 pwritev2 sys_pwritev2 +329 common pkey_mprotect sys_pkey_mprotect +330 common pkey_alloc sys_pkey_alloc +331 common pkey_free sys_pkey_free +332 common statx sys_statx +333 common io_pgetevents sys_io_pgetevents +334 common rseq sys_rseq # don't use numbers 387 through 423, add new calls after the last # 'common' entry -424 common pidfd_send_signal __x64_sys_pidfd_send_signal -425 common io_uring_setup __x64_sys_io_uring_setup -426 common io_uring_enter __x64_sys_io_uring_enter -427 common io_uring_register __x64_sys_io_uring_register -428 common open_tree __x64_sys_open_tree -429 common move_mount __x64_sys_move_mount -430 common fsopen __x64_sys_fsopen -431 common fsconfig __x64_sys_fsconfig -432 common fsmount __x64_sys_fsmount -433 common fspick __x64_sys_fspick -434 common pidfd_open __x64_sys_pidfd_open -435 common clone3 __x64_sys_clone3/ptregs -437 common openat2 __x64_sys_openat2 -438 common pidfd_getfd __x64_sys_pidfd_getfd +424 common pidfd_send_signal sys_pidfd_send_signal +425 common io_uring_setup sys_io_uring_setup +426 common io_uring_enter sys_io_uring_enter +427 common io_uring_register sys_io_uring_register +428 common open_tree sys_open_tree +429 common move_mount sys_move_mount +430 common fsopen sys_fsopen +431 common fsconfig sys_fsconfig +432 common fsmount sys_fsmount +433 common fspick sys_fspick +434 common pidfd_open sys_pidfd_open +435 common clone3 sys_clone3 +437 common openat2 sys_openat2 +438 common pidfd_getfd sys_pidfd_getfd # # x32-specific system call numbers start at 512 to avoid cache impact @@ -366,39 +366,39 @@ # on-the-fly for compat_sys_*() compatibility system calls if X86_X32 # is defined. # -512 x32 rt_sigaction __x32_compat_sys_rt_sigaction -513 x32 rt_sigreturn sys32_x32_rt_sigreturn -514 x32 ioctl __x32_compat_sys_ioctl -515 x32 readv __x32_compat_sys_readv -516 x32 writev __x32_compat_sys_writev -517 x32 recvfrom __x32_compat_sys_recvfrom -518 x32 sendmsg __x32_compat_sys_sendmsg -519 x32 recvmsg __x32_compat_sys_recvmsg -520 x32 execve __x32_compat_sys_execve/ptregs -521 x32 ptrace __x32_compat_sys_ptrace -522 x32 rt_sigpending __x32_compat_sys_rt_sigpending -523 x32 rt_sigtimedwait __x32_compat_sys_rt_sigtimedwait_time64 -524 x32 rt_sigqueueinfo __x32_compat_sys_rt_sigqueueinfo -525 x32 sigaltstack __x32_compat_sys_sigaltstack -526 x32 timer_create __x32_compat_sys_timer_create -527 x32 mq_notify __x32_compat_sys_mq_notify -528 x32 kexec_load __x32_compat_sys_kexec_load -529 x32 waitid __x32_compat_sys_waitid -530 x32 set_robust_list __x32_compat_sys_set_robust_list -531 x32 get_robust_list __x32_compat_sys_get_robust_list -532 x32 vmsplice __x32_compat_sys_vmsplice -533 x32 move_pages __x32_compat_sys_move_pages -534 x32 preadv __x32_compat_sys_preadv64 -535 x32 pwritev __x32_compat_sys_pwritev64 -536 x32 rt_tgsigqueueinfo __x32_compat_sys_rt_tgsigqueueinfo -537 x32 recvmmsg __x32_compat_sys_recvmmsg_time64 -538 x32 sendmmsg __x32_compat_sys_sendmmsg -539 x32 process_vm_readv __x32_compat_sys_process_vm_readv -540 x32 process_vm_writev __x32_compat_sys_process_vm_writev -541 x32 setsockopt __x32_compat_sys_setsockopt -542 x32 getsockopt __x32_compat_sys_getsockopt -543 x32 io_setup __x32_compat_sys_io_setup -544 x32 io_submit __x32_compat_sys_io_submit -545 x32 execveat __x32_compat_sys_execveat/ptregs -546 x32 preadv2 __x32_compat_sys_preadv64v2 -547 x32 pwritev2 __x32_compat_sys_pwritev64v2 +512 x32 rt_sigaction compat_sys_rt_sigaction +513 x32 rt_sigreturn compat_sys_x32_rt_sigreturn +514 x32 ioctl compat_sys_ioctl +515 x32 readv compat_sys_readv +516 x32 writev compat_sys_writev +517 x32 recvfrom compat_sys_recvfrom +518 x32 sendmsg compat_sys_sendmsg +519 x32 recvmsg compat_sys_recvmsg +520 x32 execve compat_sys_execve +521 x32 ptrace compat_sys_ptrace +522 x32 rt_sigpending compat_sys_rt_sigpending +523 x32 rt_sigtimedwait compat_sys_rt_sigtimedwait_time64 +524 x32 rt_sigqueueinfo compat_sys_rt_sigqueueinfo +525 x32 sigaltstack compat_sys_sigaltstack +526 x32 timer_create compat_sys_timer_create +527 x32 mq_notify compat_sys_mq_notify +528 x32 kexec_load compat_sys_kexec_load +529 x32 waitid compat_sys_waitid +530 x32 set_robust_list compat_sys_set_robust_list +531 x32 get_robust_list compat_sys_get_robust_list +532 x32 vmsplice compat_sys_vmsplice +533 x32 move_pages compat_sys_move_pages +534 x32 preadv compat_sys_preadv64 +535 x32 pwritev compat_sys_pwritev64 +536 x32 rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +537 x32 recvmmsg compat_sys_recvmmsg_time64 +538 x32 sendmmsg compat_sys_sendmmsg +539 x32 process_vm_readv compat_sys_process_vm_readv +540 x32 process_vm_writev compat_sys_process_vm_writev +541 x32 setsockopt compat_sys_setsockopt +542 x32 getsockopt compat_sys_getsockopt +543 x32 io_setup compat_sys_io_setup +544 x32 io_submit compat_sys_io_submit +545 x32 execveat compat_sys_execveat +546 x32 preadv2 compat_sys_preadv64v2 +547 x32 pwritev2 compat_sys_pwritev64v2 diff --git a/arch/x86/entry/syscalls/syscallhdr.sh b/arch/x86/entry/syscalls/syscallhdr.sh index 12fbbcfe7ef3..cc1e63857427 100644 --- a/arch/x86/entry/syscalls/syscallhdr.sh +++ b/arch/x86/entry/syscalls/syscallhdr.sh @@ -15,14 +15,21 @@ grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( echo "#define ${fileguard} 1" echo "" + max=0 while read nr abi name entry ; do if [ -z "$offset" ]; then echo "#define __NR_${prefix}${name} $nr" else echo "#define __NR_${prefix}${name} ($offset + $nr)" fi + + max=$nr done echo "" + echo "#ifdef __KERNEL__" + echo "#define __NR_${prefix}syscall_max $max" + echo "#endif" + echo "" echo "#endif /* ${fileguard} */" ) > "$out" diff --git a/arch/x86/entry/syscalls/syscalltbl.sh b/arch/x86/entry/syscalls/syscalltbl.sh index 1af2be39e7d9..929bde120d6b 100644 --- a/arch/x86/entry/syscalls/syscalltbl.sh +++ b/arch/x86/entry/syscalls/syscalltbl.sh @@ -9,15 +9,7 @@ syscall_macro() { local nr="$2" local entry="$3" - # Entry can be either just a function name or "function/qualifier" - real_entry="${entry%%/*}" - if [ "$entry" = "$real_entry" ]; then - qualifier= - else - qualifier=${entry#*/} - fi - - echo "__SYSCALL_${abi}($nr, $real_entry, $qualifier)" + echo "__SYSCALL_${abi}($nr, $entry)" } emit() { @@ -25,27 +17,15 @@ emit() { local nr="$2" local entry="$3" local compat="$4" - local umlentry="" if [ "$abi" != "I386" -a -n "$compat" ]; then echo "a compat entry ($abi: $compat) for a 64-bit syscall makes no sense" >&2 exit 1 fi - # For CONFIG_UML, we need to strip the __x64_sys prefix - if [ "$abi" = "64" -a "${entry}" != "${entry#__x64_sys}" ]; then - umlentry="sys${entry#__x64_sys}" - fi - if [ -z "$compat" ]; then - if [ -n "$entry" -a -z "$umlentry" ]; then - syscall_macro "$abi" "$nr" "$entry" - elif [ -n "$umlentry" ]; then # implies -n "$entry" - echo "#ifdef CONFIG_X86" + if [ -n "$entry" ]; then syscall_macro "$abi" "$nr" "$entry" - echo "#else /* CONFIG_UML */" - syscall_macro "$abi" "$nr" "$umlentry" - echo "#endif" fi else echo "#ifdef CONFIG_X86_32" @@ -61,24 +41,6 @@ emit() { grep '^[0-9]' "$in" | sort -n | ( while read nr abi name entry compat; do abi=`echo "$abi" | tr '[a-z]' '[A-Z]'` - if [ "$abi" = "COMMON" -o "$abi" = "64" ]; then - emit 64 "$nr" "$entry" "$compat" - if [ "$abi" = "COMMON" ]; then - # COMMON means that this syscall exists in the same form for - # 64-bit and X32. - echo "#ifdef CONFIG_X86_X32_ABI" - emit X32 "$nr" "$entry" "$compat" - echo "#endif" - fi - elif [ "$abi" = "X32" ]; then - echo "#ifdef CONFIG_X86_X32_ABI" - emit X32 "$nr" "$entry" "$compat" - echo "#endif" - elif [ "$abi" = "I386" ]; then - emit "$abi" "$nr" "$entry" "$compat" - else - echo "Unknown abi $abi" >&2 - exit 1 - fi + emit "$abi" "$nr" "$entry" "$compat" done ) > "$out" diff --git a/arch/x86/entry/thunk_32.S b/arch/x86/entry/thunk_32.S index e010d4ae11f1..3a07ce3ec70b 100644 --- a/arch/x86/entry/thunk_32.S +++ b/arch/x86/entry/thunk_32.S @@ -35,9 +35,9 @@ SYM_CODE_END(\name) #endif #ifdef CONFIG_PREEMPTION - THUNK ___preempt_schedule, preempt_schedule - THUNK ___preempt_schedule_notrace, preempt_schedule_notrace - EXPORT_SYMBOL(___preempt_schedule) - EXPORT_SYMBOL(___preempt_schedule_notrace) + THUNK preempt_schedule_thunk, preempt_schedule + THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace + EXPORT_SYMBOL(preempt_schedule_thunk) + EXPORT_SYMBOL(preempt_schedule_notrace_thunk) #endif diff --git a/arch/x86/entry/thunk_64.S b/arch/x86/entry/thunk_64.S index c5c3b6e86e62..dbe4493b534e 100644 --- a/arch/x86/entry/thunk_64.S +++ b/arch/x86/entry/thunk_64.S @@ -47,10 +47,10 @@ SYM_FUNC_END(\name) #endif #ifdef CONFIG_PREEMPTION - THUNK ___preempt_schedule, preempt_schedule - THUNK ___preempt_schedule_notrace, preempt_schedule_notrace - EXPORT_SYMBOL(___preempt_schedule) - EXPORT_SYMBOL(___preempt_schedule_notrace) + THUNK preempt_schedule_thunk, preempt_schedule + THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace + EXPORT_SYMBOL(preempt_schedule_thunk) + EXPORT_SYMBOL(preempt_schedule_notrace_thunk) #endif #if defined(CONFIG_TRACE_IRQFLAGS) \ diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S index ea7e0155c604..4d152933547d 100644 --- a/arch/x86/entry/vdso/vdso-layout.lds.S +++ b/arch/x86/entry/vdso/vdso-layout.lds.S @@ -57,6 +57,13 @@ SECTIONS *(.gnu.linkonce.b.*) } :text + /* + * Discard .note.gnu.property sections which are unused and have + * different alignment requirement from vDSO note sections. + */ + /DISCARD/ : { + *(.note.gnu.property) + } .note : { *(.note.*) } :text :note .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr diff --git a/arch/x86/entry/vdso/vdso32/vclock_gettime.c b/arch/x86/entry/vdso/vdso32/vclock_gettime.c index 9242b28418d5..1e82bd43286c 100644 --- a/arch/x86/entry/vdso/vdso32/vclock_gettime.c +++ b/arch/x86/entry/vdso/vdso32/vclock_gettime.c @@ -13,6 +13,7 @@ */ #undef CONFIG_64BIT #undef CONFIG_X86_64 +#undef CONFIG_COMPAT #undef CONFIG_PGTABLE_LEVELS #undef CONFIG_ILLEGAL_POINTER_VALUE #undef CONFIG_SPARSEMEM_VMEMMAP diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index c1b8496b5606..43428cc514c8 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -38,6 +38,8 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page) } #undef EMIT_VVAR +unsigned int vclocks_used __read_mostly; + #if defined(CONFIG_X86_64) unsigned int __read_mostly vdso64_enabled = 1; #endif @@ -219,7 +221,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, } else if (sym_offset == image->sym_pvclock_page) { struct pvclock_vsyscall_time_info *pvti = pvclock_get_pvti_cpu0_va(); - if (pvti && vclock_was_used(VCLOCK_PVCLOCK)) { + if (pvti && vclock_was_used(VDSO_CLOCKMODE_PVCLOCK)) { return vmf_insert_pfn_prot(vma, vmf->address, __pa(pvti) >> PAGE_SHIFT, pgprot_decrypted(vma->vm_page_prot)); @@ -227,7 +229,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, } else if (sym_offset == image->sym_hvclock_page) { struct ms_hyperv_tsc_page *tsc_pg = hv_get_tsc_page(); - if (tsc_pg && vclock_was_used(VCLOCK_HVCLOCK)) + if (tsc_pg && vclock_was_used(VDSO_CLOCKMODE_HVCLOCK)) return vmf_insert_pfn(vma, vmf->address, virt_to_phys(tsc_pg) >> PAGE_SHIFT); } else if (sym_offset == image->sym_timens_page) { @@ -445,6 +447,8 @@ __setup("vdso=", vdso_setup); static int __init init_vdso(void) { + BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); + init_vdso_image(&vdso_image_64); #ifdef CONFIG_X86_X32_ABI diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c index abef51320e3a..43b09e9c93a2 100644 --- a/arch/x86/events/amd/power.c +++ b/arch/x86/events/amd/power.c @@ -259,7 +259,7 @@ static int power_cpu_init(unsigned int cpu) } static const struct x86_cpu_id cpu_match[] = { - { .vendor = X86_VENDOR_AMD, .family = 0x15 }, + X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL), {}, }; diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 4d867a752f0e..76400c052b0e 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -180,6 +180,31 @@ static void amd_uncore_del(struct perf_event *event, int flags) hwc->idx = -1; } +/* + * Convert logical CPU number to L3 PMC Config ThreadMask format + */ +static u64 l3_thread_slice_mask(int cpu) +{ + u64 thread_mask, core = topology_core_id(cpu); + unsigned int shift, thread = 0; + + if (topology_smt_supported() && !topology_is_primary_thread(cpu)) + thread = 1; + + if (boot_cpu_data.x86 <= 0x18) { + shift = AMD64_L3_THREAD_SHIFT + 2 * (core % 4) + thread; + thread_mask = BIT_ULL(shift); + + return AMD64_L3_SLICE_MASK | thread_mask; + } + + core = (core << AMD64_L3_COREID_SHIFT) & AMD64_L3_COREID_MASK; + shift = AMD64_L3_THREAD_SHIFT + thread; + thread_mask = BIT_ULL(shift); + + return AMD64_L3_EN_ALL_SLICES | core | thread_mask; +} + static int amd_uncore_event_init(struct perf_event *event) { struct amd_uncore *uncore; @@ -203,18 +228,11 @@ static int amd_uncore_event_init(struct perf_event *event) return -EINVAL; /* - * SliceMask and ThreadMask need to be set for certain L3 events in - * Family 17h. For other events, the two fields do not affect the count. + * SliceMask and ThreadMask need to be set for certain L3 events. + * For other events, the two fields do not affect the count. */ - if (l3_mask && is_llc_event(event)) { - int thread = 2 * (cpu_data(event->cpu).cpu_core_id % 4); - - if (smp_num_siblings > 1) - thread += cpu_data(event->cpu).apicid & 1; - - hwc->config |= (1ULL << (AMD64_L3_THREAD_SHIFT + thread) & - AMD64_L3_THREAD_MASK) | AMD64_L3_SLICE_MASK; - } + if (l3_mask && is_llc_event(event)) + hwc->config |= l3_thread_slice_mask(event->cpu); uncore = event_to_amd_uncore(event); if (!uncore) @@ -520,9 +538,9 @@ static int __init amd_uncore_init(void) if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) return -ENODEV; - if (boot_cpu_data.x86 == 0x17 || boot_cpu_data.x86 == 0x18) { + if (boot_cpu_data.x86 >= 0x17) { /* - * For F17h or F18h, the Northbridge counters are + * For F17h and above, the Northbridge counters are * repurposed as Data Fabric counters. Also, L3 * counters are supported too. The PMUs are exported * based on family as either L2 or L3 and NB or DF. diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 3bb738f5a472..a619763e96e1 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2490,7 +2490,7 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent /* 32-bit process in 64-bit kernel. */ unsigned long ss_base, cs_base; struct stack_frame_ia32 frame; - const void __user *fp; + const struct stack_frame_ia32 __user *fp; if (!test_thread_flag(TIF_IA32)) return 0; @@ -2501,18 +2501,12 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent fp = compat_ptr(ss_base + regs->bp); pagefault_disable(); while (entry->nr < entry->max_stack) { - unsigned long bytes; - frame.next_frame = 0; - frame.return_address = 0; - if (!valid_user_frame(fp, sizeof(frame))) break; - bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); - if (bytes != 0) + if (__get_user(frame.next_frame, &fp->next_frame)) break; - bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); - if (bytes != 0) + if (__get_user(frame.return_address, &fp->return_address)) break; perf_callchain_store(entry, cs_base + frame.return_address); @@ -2533,7 +2527,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { struct stack_frame frame; - const unsigned long __user *fp; + const struct stack_frame __user *fp; if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { /* TODO: We don't support guest os callchain now */ @@ -2546,7 +2540,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) return; - fp = (unsigned long __user *)regs->bp; + fp = (void __user *)regs->bp; perf_callchain_store(entry, regs->ip); @@ -2558,19 +2552,12 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs pagefault_disable(); while (entry->nr < entry->max_stack) { - unsigned long bytes; - - frame.next_frame = NULL; - frame.return_address = 0; - if (!valid_user_frame(fp, sizeof(frame))) break; - bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); - if (bytes != 0) + if (__get_user(frame.next_frame, &fp->next_frame)) break; - bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); - if (bytes != 0) + if (__get_user(frame.return_address, &fp->return_address)) break; perf_callchain_store(entry, frame.return_address); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index dff6623804c2..332954cccece 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -1945,6 +1945,14 @@ static __initconst const u64 knl_hw_cache_extra_regs * intel_bts events don't coexist with intel PMU's BTS events because of * x86_add_exclusive(x86_lbr_exclusive_lbr); there's no need to keep them * disabled around intel PMU's event batching etc, only inside the PMI handler. + * + * Avoid PEBS_ENABLE MSR access in PMIs. + * The GLOBAL_CTRL has been disabled. All the counters do not count anymore. + * It doesn't matter if the PEBS is enabled or not. + * Usually, the PEBS status are not changed in PMIs. It's unnecessary to + * access PEBS_ENABLE MSR in disable_all()/enable_all(). + * However, there are some cases which may change PEBS status, e.g. PMI + * throttle. The PEBS_ENABLE should be updated where the status changes. */ static void __intel_pmu_disable_all(void) { @@ -1954,13 +1962,12 @@ static void __intel_pmu_disable_all(void) if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) intel_pmu_disable_bts(); - - intel_pmu_pebs_disable_all(); } static void intel_pmu_disable_all(void) { __intel_pmu_disable_all(); + intel_pmu_pebs_disable_all(); intel_pmu_lbr_disable_all(); } @@ -1968,7 +1975,6 @@ static void __intel_pmu_enable_all(int added, bool pmi) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - intel_pmu_pebs_enable_all(); intel_pmu_lbr_enable_all(pmi); wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask); @@ -1986,6 +1992,7 @@ static void __intel_pmu_enable_all(int added, bool pmi) static void intel_pmu_enable_all(int added) { + intel_pmu_pebs_enable_all(); __intel_pmu_enable_all(added, false); } @@ -2374,9 +2381,21 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) * PEBS overflow sets bit 62 in the global status register */ if (__test_and_clear_bit(62, (unsigned long *)&status)) { + u64 pebs_enabled = cpuc->pebs_enabled; + handled++; x86_pmu.drain_pebs(regs); status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI; + + /* + * PMI throttle may be triggered, which stops the PEBS event. + * Although cpuc->pebs_enabled is updated accordingly, the + * MSR_IA32_PEBS_ENABLE is not updated. Because the + * cpuc->enabled has been forced to 0 in PMI. + * Update the MSR if pebs_enabled is changed. + */ + if (pebs_enabled != cpuc->pebs_enabled) + wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); } /* diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 4814c964692c..e4aa20c0426f 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -594,63 +594,60 @@ static const struct cstate_model glm_cstates __initconst = { }; -#define X86_CSTATES_MODEL(model, states) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long) &(states) } - static const struct x86_cpu_id intel_cstates_match[] __initconst = { - X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM, nhm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM_EP, nhm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM_EX, nhm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE, nhm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE_EP, nhm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE_EX, nhm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_SANDYBRIDGE, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_SANDYBRIDGE_X, snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_IVYBRIDGE, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_IVYBRIDGE_X, snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_HASWELL, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_X, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_G, snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_L, hswult_cstates), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hswult_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_SILVERMONT, slm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_SILVERMONT_D, slm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_AIRMONT, slm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &slm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &slm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &slm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_D, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_G, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_X, snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_L, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE, snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_X, snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &snb_cstates), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &snb_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_KABYLAKE_L, hswult_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_KABYLAKE, hswult_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_COMETLAKE_L, hswult_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_COMETLAKE, hswult_cstates), + X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &hswult_cstates), + X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &hswult_cstates), + X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &hswult_cstates), + X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &hswult_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_CANNONLAKE_L, cnl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(CANNONLAKE_L, &cnl_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_XEON_PHI_KNL, knl_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_XEON_PHI_KNM, knl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_GOLDMONT, glm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_GOLDMONT_D, glm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_GOLDMONT_PLUS, glm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_TREMONT_D, glm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ATOM_TREMONT, glm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &glm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &glm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &glm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &glm_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, &glm_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ICELAKE_L, icl_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_ICELAKE, icl_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_TIGERLAKE_L, icl_cstates), - X86_CSTATES_MODEL(INTEL_FAM6_TIGERLAKE, icl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &icl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &icl_cstates), { }, }; MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 534c76606049..65113b16804a 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -585,6 +585,7 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) cpuc->lbr_entries[i].reserved = 0; } cpuc->lbr_stack.nr = i; + cpuc->lbr_stack.hw_idx = tos; } /* @@ -680,6 +681,7 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) out++; } cpuc->lbr_stack.nr = out; + cpuc->lbr_stack.hw_idx = tos; } void intel_pmu_lbr_read(void) @@ -1120,6 +1122,13 @@ void intel_pmu_store_pebs_lbrs(struct pebs_lbr *lbr) int i; cpuc->lbr_stack.nr = x86_pmu.lbr_nr; + + /* Cannot get TOS for large PEBS */ + if (cpuc->n_pebs == cpuc->n_large_pebs) + cpuc->lbr_stack.hw_idx = -1ULL; + else + cpuc->lbr_stack.hw_idx = intel_pmu_lbr_tos(); + for (i = 0; i < x86_pmu.lbr_nr; i++) { u64 info = lbr->lbr[i].info; struct perf_branch_entry *e = &cpuc->lbr_entries[i]; diff --git a/arch/x86/events/intel/rapl.c b/arch/x86/events/intel/rapl.c index 09913121e726..a5dbd25852cb 100644 --- a/arch/x86/events/intel/rapl.c +++ b/arch/x86/events/intel/rapl.c @@ -668,9 +668,6 @@ static int __init init_rapl_pmus(void) return 0; } -#define X86_RAPL_MODEL_MATCH(model, init) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&init } - static struct rapl_model model_snb = { .events = BIT(PERF_RAPL_PP0) | BIT(PERF_RAPL_PKG) | @@ -716,36 +713,35 @@ static struct rapl_model model_skl = { }; static const struct x86_cpu_id rapl_model_match[] __initconst = { - X86_RAPL_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE, model_snb), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X, model_snbep), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE, model_snb), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X, model_snbep), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_X, model_hsx), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_L, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_G, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_G, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_X, model_hsx), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_D, model_hsx), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL, model_knl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNM, model_knl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_L, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_X, model_hsx), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_KABYLAKE_L, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_KABYLAKE, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_CANNONLAKE_L, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_ATOM_GOLDMONT, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_ATOM_GOLDMONT_D, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_ATOM_GOLDMONT_PLUS, model_hsw), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_ICELAKE_L, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_ICELAKE, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_COMETLAKE_L, model_skl), - X86_RAPL_MODEL_MATCH(INTEL_FAM6_COMETLAKE, model_skl), + X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &model_snb), + X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &model_snbep), + X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &model_snb), + X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &model_snbep), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &model_hsx), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &model_hsx), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &model_hsx), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &model_knl), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &model_knl), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &model_hsx), + X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(CANNONLAKE_L, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &model_hsw), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &model_skl), {}, }; - MODULE_DEVICE_TABLE(x86cpu, rapl_model_match); static int __init rapl_pmu_init(void) diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 86467f85c383..1ba72c563313 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1392,10 +1392,6 @@ err: return ret; } - -#define X86_UNCORE_MODEL_MATCH(model, init) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&init } - struct intel_uncore_init_fun { void (*cpu_init)(void); int (*pci_init)(void); @@ -1470,6 +1466,16 @@ static const struct intel_uncore_init_fun icl_uncore_init __initconst = { .pci_init = skl_uncore_pci_init, }; +static const struct intel_uncore_init_fun tgl_uncore_init __initconst = { + .cpu_init = icl_uncore_cpu_init, + .mmio_init = tgl_uncore_mmio_init, +}; + +static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = { + .cpu_init = icl_uncore_cpu_init, + .mmio_init = tgl_l_uncore_mmio_init, +}; + static const struct intel_uncore_init_fun snr_uncore_init __initconst = { .cpu_init = snr_uncore_cpu_init, .pci_init = snr_uncore_pci_init, @@ -1477,38 +1483,39 @@ static const struct intel_uncore_init_fun snr_uncore_init __initconst = { }; static const struct x86_cpu_id intel_uncore_match[] __initconst = { - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EP, nhm_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM, nhm_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE, nhm_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EP, nhm_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE, snb_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE, ivb_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL, hsw_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_L, hsw_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_G, hsw_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL, bdw_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_G, bdw_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X, snbep_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EX, nhmex_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EX, nhmex_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X, ivbep_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_X, hswep_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_X, bdx_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_D, bdx_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL, knl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNM, knl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE, skl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_L, skl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_X, skx_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_KABYLAKE_L, skl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_KABYLAKE, skl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE_L, icl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE_NNPI, icl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE, icl_uncore_init), - X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ATOM_TREMONT_D, snr_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &ivb_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &hsw_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hsw_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &hsw_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &bdw_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &bdw_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snbep_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhmex_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhmex_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ivbep_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &hswep_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &bdx_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &bdx_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &skl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &skl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &skx_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &skl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &skl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_NNPI, &icl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &tgl_l_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &tgl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init), {}, }; - MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match); static int __init intel_uncore_init(void) diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index bbfdaa720b45..b30429f8a53a 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -154,6 +154,7 @@ struct freerunning_counters { unsigned int box_offset; unsigned int num_counters; unsigned int bits; + unsigned *box_offsets; }; struct pci2phy_map { @@ -310,7 +311,9 @@ unsigned int uncore_freerunning_counter(struct intel_uncore_box *box, return pmu->type->freerunning[type].counter_base + pmu->type->freerunning[type].counter_offset * idx + - pmu->type->freerunning[type].box_offset * pmu->pmu_idx; + (pmu->type->freerunning[type].box_offsets ? + pmu->type->freerunning[type].box_offsets[pmu->pmu_idx] : + pmu->type->freerunning[type].box_offset * pmu->pmu_idx); } static inline @@ -527,6 +530,8 @@ void snb_uncore_cpu_init(void); void nhm_uncore_cpu_init(void); void skl_uncore_cpu_init(void); void icl_uncore_cpu_init(void); +void tgl_uncore_mmio_init(void); +void tgl_l_uncore_mmio_init(void); int snb_pci2phy_map_init(int devid); /* uncore_snbep.c */ diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index c37cb12d0ef6..3de1065eefc4 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -44,6 +44,11 @@ #define PCI_DEVICE_ID_INTEL_WHL_UD_IMC 0x3e35 #define PCI_DEVICE_ID_INTEL_ICL_U_IMC 0x8a02 #define PCI_DEVICE_ID_INTEL_ICL_U2_IMC 0x8a12 +#define PCI_DEVICE_ID_INTEL_TGL_U1_IMC 0x9a02 +#define PCI_DEVICE_ID_INTEL_TGL_U2_IMC 0x9a04 +#define PCI_DEVICE_ID_INTEL_TGL_U3_IMC 0x9a12 +#define PCI_DEVICE_ID_INTEL_TGL_U4_IMC 0x9a14 +#define PCI_DEVICE_ID_INTEL_TGL_H_IMC 0x9a36 /* SNB event control */ @@ -1002,3 +1007,157 @@ void nhm_uncore_cpu_init(void) } /* end of Nehalem uncore support */ + +/* Tiger Lake MMIO uncore support */ + +static const struct pci_device_id tgl_uncore_pci_ids[] = { + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U1_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U2_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U3_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U4_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_H_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* end: all zeroes */ } +}; + +enum perf_tgl_uncore_imc_freerunning_types { + TGL_MMIO_UNCORE_IMC_DATA_TOTAL, + TGL_MMIO_UNCORE_IMC_DATA_READ, + TGL_MMIO_UNCORE_IMC_DATA_WRITE, + TGL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX +}; + +static struct freerunning_counters tgl_l_uncore_imc_freerunning[] = { + [TGL_MMIO_UNCORE_IMC_DATA_TOTAL] = { 0x5040, 0x0, 0x0, 1, 64 }, + [TGL_MMIO_UNCORE_IMC_DATA_READ] = { 0x5058, 0x0, 0x0, 1, 64 }, + [TGL_MMIO_UNCORE_IMC_DATA_WRITE] = { 0x50A0, 0x0, 0x0, 1, 64 }, +}; + +static struct freerunning_counters tgl_uncore_imc_freerunning[] = { + [TGL_MMIO_UNCORE_IMC_DATA_TOTAL] = { 0xd840, 0x0, 0x0, 1, 64 }, + [TGL_MMIO_UNCORE_IMC_DATA_READ] = { 0xd858, 0x0, 0x0, 1, 64 }, + [TGL_MMIO_UNCORE_IMC_DATA_WRITE] = { 0xd8A0, 0x0, 0x0, 1, 64 }, +}; + +static struct uncore_event_desc tgl_uncore_imc_events[] = { + INTEL_UNCORE_EVENT_DESC(data_total, "event=0xff,umask=0x10"), + INTEL_UNCORE_EVENT_DESC(data_total.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(data_total.unit, "MiB"), + + INTEL_UNCORE_EVENT_DESC(data_read, "event=0xff,umask=0x20"), + INTEL_UNCORE_EVENT_DESC(data_read.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(data_read.unit, "MiB"), + + INTEL_UNCORE_EVENT_DESC(data_write, "event=0xff,umask=0x30"), + INTEL_UNCORE_EVENT_DESC(data_write.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(data_write.unit, "MiB"), + + { /* end: all zeroes */ } +}; + +static struct pci_dev *tgl_uncore_get_mc_dev(void) +{ + const struct pci_device_id *ids = tgl_uncore_pci_ids; + struct pci_dev *mc_dev = NULL; + + while (ids && ids->vendor) { + mc_dev = pci_get_device(PCI_VENDOR_ID_INTEL, ids->device, NULL); + if (mc_dev) + return mc_dev; + ids++; + } + + return mc_dev; +} + +#define TGL_UNCORE_MMIO_IMC_MEM_OFFSET 0x10000 + +static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +{ + struct pci_dev *pdev = tgl_uncore_get_mc_dev(); + struct intel_uncore_pmu *pmu = box->pmu; + resource_size_t addr; + u32 mch_bar; + + if (!pdev) { + pr_warn("perf uncore: Cannot find matched IMC device.\n"); + return; + } + + pci_read_config_dword(pdev, SNB_UNCORE_PCI_IMC_BAR_OFFSET, &mch_bar); + /* MCHBAR is disabled */ + if (!(mch_bar & BIT(0))) { + pr_warn("perf uncore: MCHBAR is disabled. Failed to map IMC free-running counters.\n"); + return; + } + mch_bar &= ~BIT(0); + addr = (resource_size_t)(mch_bar + TGL_UNCORE_MMIO_IMC_MEM_OFFSET * pmu->pmu_idx); + +#ifdef CONFIG_PHYS_ADDR_T_64BIT + pci_read_config_dword(pdev, SNB_UNCORE_PCI_IMC_BAR_OFFSET + 4, &mch_bar); + addr |= ((resource_size_t)mch_bar << 32); +#endif + + box->io_addr = ioremap(addr, SNB_UNCORE_PCI_IMC_MAP_SIZE); +} + +static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = { + .init_box = tgl_uncore_imc_freerunning_init_box, + .exit_box = uncore_mmio_exit_box, + .read_counter = uncore_mmio_read_counter, + .hw_config = uncore_freerunning_hw_config, +}; + +static struct attribute *tgl_uncore_imc_formats_attr[] = { + &format_attr_event.attr, + &format_attr_umask.attr, + NULL +}; + +static const struct attribute_group tgl_uncore_imc_format_group = { + .name = "format", + .attrs = tgl_uncore_imc_formats_attr, +}; + +static struct intel_uncore_type tgl_uncore_imc_free_running = { + .name = "imc_free_running", + .num_counters = 3, + .num_boxes = 2, + .num_freerunning_types = TGL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX, + .freerunning = tgl_uncore_imc_freerunning, + .ops = &tgl_uncore_imc_freerunning_ops, + .event_descs = tgl_uncore_imc_events, + .format_group = &tgl_uncore_imc_format_group, +}; + +static struct intel_uncore_type *tgl_mmio_uncores[] = { + &tgl_uncore_imc_free_running, + NULL +}; + +void tgl_l_uncore_mmio_init(void) +{ + tgl_uncore_imc_free_running.freerunning = tgl_l_uncore_imc_freerunning; + uncore_mmio_uncores = tgl_mmio_uncores; +} + +void tgl_uncore_mmio_init(void) +{ + uncore_mmio_uncores = tgl_mmio_uncores; +} + +/* end of Tiger Lake MMIO uncore support */ diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index ad20220af303..01023f0d935b 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -4380,10 +4380,10 @@ static struct pci_dev *snr_uncore_get_mc_dev(int id) return mc_dev; } -static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) +static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, + unsigned int box_ctl, int mem_offset) { struct pci_dev *pdev = snr_uncore_get_mc_dev(box->dieid); - unsigned int box_ctl = uncore_mmio_box_ctl(box); resource_size_t addr; u32 pci_dword; @@ -4393,7 +4393,7 @@ static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) pci_read_config_dword(pdev, SNR_IMC_MMIO_BASE_OFFSET, &pci_dword); addr = (pci_dword & SNR_IMC_MMIO_BASE_MASK) << 23; - pci_read_config_dword(pdev, SNR_IMC_MMIO_MEM0_OFFSET, &pci_dword); + pci_read_config_dword(pdev, mem_offset, &pci_dword); addr |= (pci_dword & SNR_IMC_MMIO_MEM0_MASK) << 12; addr += box_ctl; @@ -4405,6 +4405,12 @@ static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr); } +static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) +{ + __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), + SNR_IMC_MMIO_MEM0_OFFSET); +} + static void snr_uncore_mmio_disable_box(struct intel_uncore_box *box) { u32 config; diff --git a/arch/x86/ia32/Makefile b/arch/x86/ia32/Makefile index d13b352b2aa7..8e4d0391ff6c 100644 --- a/arch/x86/ia32/Makefile +++ b/arch/x86/ia32/Makefile @@ -3,7 +3,7 @@ # Makefile for the ia32 kernel emulation subsystem. # -obj-$(CONFIG_IA32_EMULATION) := sys_ia32.o ia32_signal.o +obj-$(CONFIG_IA32_EMULATION) := ia32_signal.o obj-$(CONFIG_IA32_AOUT) += ia32_aout.o diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index a3aefe9b9401..f9d8804144d0 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -36,70 +36,56 @@ #include <asm/sighandling.h> #include <asm/smap.h> +static inline void reload_segments(struct sigcontext_32 *sc) +{ + unsigned int cur; + + savesegment(gs, cur); + if ((sc->gs | 0x03) != cur) + load_gs_index(sc->gs | 0x03); + savesegment(fs, cur); + if ((sc->fs | 0x03) != cur) + loadsegment(fs, sc->fs | 0x03); + savesegment(ds, cur); + if ((sc->ds | 0x03) != cur) + loadsegment(ds, sc->ds | 0x03); + savesegment(es, cur); + if ((sc->es | 0x03) != cur) + loadsegment(es, sc->es | 0x03); +} + /* * Do a signal return; undo the signal stack. */ -#define loadsegment_gs(v) load_gs_index(v) -#define loadsegment_fs(v) loadsegment(fs, v) -#define loadsegment_ds(v) loadsegment(ds, v) -#define loadsegment_es(v) loadsegment(es, v) - -#define get_user_seg(seg) ({ unsigned int v; savesegment(seg, v); v; }) -#define set_user_seg(seg, v) loadsegment_##seg(v) - -#define COPY(x) { \ - get_user_ex(regs->x, &sc->x); \ -} - -#define GET_SEG(seg) ({ \ - unsigned short tmp; \ - get_user_ex(tmp, &sc->seg); \ - tmp; \ -}) - -#define COPY_SEG_CPL3(seg) do { \ - regs->seg = GET_SEG(seg) | 3; \ -} while (0) - -#define RELOAD_SEG(seg) { \ - unsigned int pre = (seg) | 3; \ - unsigned int cur = get_user_seg(seg); \ - if (pre != cur) \ - set_user_seg(seg, pre); \ -} - static int ia32_restore_sigcontext(struct pt_regs *regs, - struct sigcontext_32 __user *sc) + struct sigcontext_32 __user *usc) { - unsigned int tmpflags, err = 0; - u16 gs, fs, es, ds; - void __user *buf; - u32 tmp; + struct sigcontext_32 sc; /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - get_user_try { - gs = GET_SEG(gs); - fs = GET_SEG(fs); - ds = GET_SEG(ds); - es = GET_SEG(es); - - COPY(di); COPY(si); COPY(bp); COPY(sp); COPY(bx); - COPY(dx); COPY(cx); COPY(ip); COPY(ax); - /* Don't touch extended registers */ - - COPY_SEG_CPL3(cs); - COPY_SEG_CPL3(ss); - - get_user_ex(tmpflags, &sc->flags); - regs->flags = (regs->flags & ~FIX_EFLAGS) | (tmpflags & FIX_EFLAGS); - /* disable syscall checks */ - regs->orig_ax = -1; + if (unlikely(copy_from_user(&sc, usc, sizeof(sc)))) + return -EFAULT; - get_user_ex(tmp, &sc->fpstate); - buf = compat_ptr(tmp); - } get_user_catch(err); + /* Get only the ia32 registers. */ + regs->bx = sc.bx; + regs->cx = sc.cx; + regs->dx = sc.dx; + regs->si = sc.si; + regs->di = sc.di; + regs->bp = sc.bp; + regs->ax = sc.ax; + regs->sp = sc.sp; + regs->ip = sc.ip; + + /* Get CS/SS and force CPL3 */ + regs->cs = sc.cs | 0x03; + regs->ss = sc.ss | 0x03; + + regs->flags = (regs->flags & ~FIX_EFLAGS) | (sc.flags & FIX_EFLAGS); + /* disable syscall checks */ + regs->orig_ax = -1; /* * Reload fs and gs if they have changed in the signal @@ -107,14 +93,8 @@ static int ia32_restore_sigcontext(struct pt_regs *regs, * the handler, but does not clobber them at least in the * normal case. */ - RELOAD_SEG(gs); - RELOAD_SEG(fs); - RELOAD_SEG(ds); - RELOAD_SEG(es); - - err |= fpu__restore_sig(buf, 1); - - return err; + reload_segments(&sc); + return fpu__restore_sig(compat_ptr(sc.fpstate), 1); } COMPAT_SYSCALL_DEFINE0(sigreturn) @@ -126,10 +106,7 @@ COMPAT_SYSCALL_DEFINE0(sigreturn) if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__get_user(set.sig[0], &frame->sc.oldmask) - || (_COMPAT_NSIG_WORDS > 1 - && __copy_from_user((((char *) &set.sig) + 4), - &frame->extramask, - sizeof(frame->extramask)))) + || __get_user(((__u32 *)&set)[1], &frame->extramask[0])) goto badframe; set_current_blocked(&set); @@ -153,7 +130,7 @@ COMPAT_SYSCALL_DEFINE0(rt_sigreturn) if (!access_ok(frame, sizeof(*frame))) goto badframe; - if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) + if (__get_user(set.sig[0], (__u64 __user *)&frame->uc.uc_sigmask)) goto badframe; set_current_blocked(&set); @@ -175,44 +152,51 @@ badframe: * Set up a signal frame. */ -static int ia32_setup_sigcontext(struct sigcontext_32 __user *sc, - void __user *fpstate, - struct pt_regs *regs, unsigned int mask) +#define get_user_seg(seg) ({ unsigned int v; savesegment(seg, v); v; }) + +static __always_inline int +__unsafe_setup_sigcontext32(struct sigcontext_32 __user *sc, + void __user *fpstate, + struct pt_regs *regs, unsigned int mask) { - int err = 0; - - put_user_try { - put_user_ex(get_user_seg(gs), (unsigned int __user *)&sc->gs); - put_user_ex(get_user_seg(fs), (unsigned int __user *)&sc->fs); - put_user_ex(get_user_seg(ds), (unsigned int __user *)&sc->ds); - put_user_ex(get_user_seg(es), (unsigned int __user *)&sc->es); - - put_user_ex(regs->di, &sc->di); - put_user_ex(regs->si, &sc->si); - put_user_ex(regs->bp, &sc->bp); - put_user_ex(regs->sp, &sc->sp); - put_user_ex(regs->bx, &sc->bx); - put_user_ex(regs->dx, &sc->dx); - put_user_ex(regs->cx, &sc->cx); - put_user_ex(regs->ax, &sc->ax); - put_user_ex(current->thread.trap_nr, &sc->trapno); - put_user_ex(current->thread.error_code, &sc->err); - put_user_ex(regs->ip, &sc->ip); - put_user_ex(regs->cs, (unsigned int __user *)&sc->cs); - put_user_ex(regs->flags, &sc->flags); - put_user_ex(regs->sp, &sc->sp_at_signal); - put_user_ex(regs->ss, (unsigned int __user *)&sc->ss); - - put_user_ex(ptr_to_compat(fpstate), &sc->fpstate); - - /* non-iBCS2 extensions.. */ - put_user_ex(mask, &sc->oldmask); - put_user_ex(current->thread.cr2, &sc->cr2); - } put_user_catch(err); - - return err; + unsafe_put_user(get_user_seg(gs), (unsigned int __user *)&sc->gs, Efault); + unsafe_put_user(get_user_seg(fs), (unsigned int __user *)&sc->fs, Efault); + unsafe_put_user(get_user_seg(ds), (unsigned int __user *)&sc->ds, Efault); + unsafe_put_user(get_user_seg(es), (unsigned int __user *)&sc->es, Efault); + + unsafe_put_user(regs->di, &sc->di, Efault); + unsafe_put_user(regs->si, &sc->si, Efault); + unsafe_put_user(regs->bp, &sc->bp, Efault); + unsafe_put_user(regs->sp, &sc->sp, Efault); + unsafe_put_user(regs->bx, &sc->bx, Efault); + unsafe_put_user(regs->dx, &sc->dx, Efault); + unsafe_put_user(regs->cx, &sc->cx, Efault); + unsafe_put_user(regs->ax, &sc->ax, Efault); + unsafe_put_user(current->thread.trap_nr, &sc->trapno, Efault); + unsafe_put_user(current->thread.error_code, &sc->err, Efault); + unsafe_put_user(regs->ip, &sc->ip, Efault); + unsafe_put_user(regs->cs, (unsigned int __user *)&sc->cs, Efault); + unsafe_put_user(regs->flags, &sc->flags, Efault); + unsafe_put_user(regs->sp, &sc->sp_at_signal, Efault); + unsafe_put_user(regs->ss, (unsigned int __user *)&sc->ss, Efault); + + unsafe_put_user(ptr_to_compat(fpstate), &sc->fpstate, Efault); + + /* non-iBCS2 extensions.. */ + unsafe_put_user(mask, &sc->oldmask, Efault); + unsafe_put_user(current->thread.cr2, &sc->cr2, Efault); + return 0; + +Efault: + return -EFAULT; } +#define unsafe_put_sigcontext32(sc, fp, regs, set, label) \ +do { \ + if (__unsafe_setup_sigcontext32(sc, fp, regs, set->sig[0])) \ + goto label; \ +} while(0) + /* * Determine which stack to use.. */ @@ -252,8 +236,7 @@ int ia32_setup_frame(int sig, struct ksignal *ksig, { struct sigframe_ia32 __user *frame; void __user *restorer; - int err = 0; - void __user *fpstate = NULL; + void __user *fp = NULL; /* copy_to_user optimizes that into a single 8 byte store */ static const struct { @@ -266,22 +249,7 @@ int ia32_setup_frame(int sig, struct ksignal *ksig, 0x80cd, /* int $0x80 */ }; - frame = get_sigframe(ksig, regs, sizeof(*frame), &fpstate); - - if (!access_ok(frame, sizeof(*frame))) - return -EFAULT; - - if (__put_user(sig, &frame->sig)) - return -EFAULT; - - if (ia32_setup_sigcontext(&frame->sc, fpstate, regs, set->sig[0])) - return -EFAULT; - - if (_COMPAT_NSIG_WORDS > 1) { - if (__copy_to_user(frame->extramask, &set->sig[1], - sizeof(frame->extramask))) - return -EFAULT; - } + frame = get_sigframe(ksig, regs, sizeof(*frame), &fp); if (ksig->ka.sa.sa_flags & SA_RESTORER) { restorer = ksig->ka.sa.sa_restorer; @@ -294,19 +262,20 @@ int ia32_setup_frame(int sig, struct ksignal *ksig, restorer = &frame->retcode; } - put_user_try { - put_user_ex(ptr_to_compat(restorer), &frame->pretcode); - - /* - * These are actually not used anymore, but left because some - * gdb versions depend on them as a marker. - */ - put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode); - } put_user_catch(err); - - if (err) + if (!user_access_begin(frame, sizeof(*frame))) return -EFAULT; + unsafe_put_user(sig, &frame->sig, Efault); + unsafe_put_sigcontext32(&frame->sc, fp, regs, set, Efault); + unsafe_put_user(set->sig[1], &frame->extramask[0], Efault); + unsafe_put_user(ptr_to_compat(restorer), &frame->pretcode, Efault); + /* + * These are actually not used anymore, but left because some + * gdb versions depend on them as a marker. + */ + unsafe_put_user(*((u64 *)&code), (u64 __user *)frame->retcode, Efault); + user_access_end(); + /* Set up registers for signal handler */ regs->sp = (unsigned long) frame; regs->ip = (unsigned long) ksig->ka.sa.sa_handler; @@ -323,6 +292,9 @@ int ia32_setup_frame(int sig, struct ksignal *ksig, regs->ss = __USER32_DS; return 0; +Efault: + user_access_end(); + return -EFAULT; } int ia32_setup_rt_frame(int sig, struct ksignal *ksig, @@ -330,10 +302,9 @@ int ia32_setup_rt_frame(int sig, struct ksignal *ksig, { struct rt_sigframe_ia32 __user *frame; void __user *restorer; - int err = 0; - void __user *fpstate = NULL; + void __user *fp = NULL; - /* __copy_to_user optimizes that into a single 8 byte store */ + /* unsafe_put_user optimizes that into a single 8 byte store */ static const struct { u8 movl; u32 val; @@ -346,44 +317,40 @@ int ia32_setup_rt_frame(int sig, struct ksignal *ksig, 0, }; - frame = get_sigframe(ksig, regs, sizeof(*frame), &fpstate); + frame = get_sigframe(ksig, regs, sizeof(*frame), &fp); - if (!access_ok(frame, sizeof(*frame))) + if (!user_access_begin(frame, sizeof(*frame))) return -EFAULT; - put_user_try { - put_user_ex(sig, &frame->sig); - put_user_ex(ptr_to_compat(&frame->info), &frame->pinfo); - put_user_ex(ptr_to_compat(&frame->uc), &frame->puc); + unsafe_put_user(sig, &frame->sig, Efault); + unsafe_put_user(ptr_to_compat(&frame->info), &frame->pinfo, Efault); + unsafe_put_user(ptr_to_compat(&frame->uc), &frame->puc, Efault); - /* Create the ucontext. */ - if (static_cpu_has(X86_FEATURE_XSAVE)) - put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags); - else - put_user_ex(0, &frame->uc.uc_flags); - put_user_ex(0, &frame->uc.uc_link); - compat_save_altstack_ex(&frame->uc.uc_stack, regs->sp); + /* Create the ucontext. */ + if (static_cpu_has(X86_FEATURE_XSAVE)) + unsafe_put_user(UC_FP_XSTATE, &frame->uc.uc_flags, Efault); + else + unsafe_put_user(0, &frame->uc.uc_flags, Efault); + unsafe_put_user(0, &frame->uc.uc_link, Efault); + unsafe_compat_save_altstack(&frame->uc.uc_stack, regs->sp, Efault); - if (ksig->ka.sa.sa_flags & SA_RESTORER) - restorer = ksig->ka.sa.sa_restorer; - else - restorer = current->mm->context.vdso + - vdso_image_32.sym___kernel_rt_sigreturn; - put_user_ex(ptr_to_compat(restorer), &frame->pretcode); - - /* - * Not actually used anymore, but left because some gdb - * versions need it. - */ - put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode); - } put_user_catch(err); - - err |= __copy_siginfo_to_user32(&frame->info, &ksig->info, false); - err |= ia32_setup_sigcontext(&frame->uc.uc_mcontext, fpstate, - regs, set->sig[0]); - err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); - - if (err) + if (ksig->ka.sa.sa_flags & SA_RESTORER) + restorer = ksig->ka.sa.sa_restorer; + else + restorer = current->mm->context.vdso + + vdso_image_32.sym___kernel_rt_sigreturn; + unsafe_put_user(ptr_to_compat(restorer), &frame->pretcode, Efault); + + /* + * Not actually used anymore, but left because some gdb + * versions need it. + */ + unsafe_put_user(*((u64 *)&code), (u64 __user *)frame->retcode, Efault); + unsafe_put_sigcontext32(&frame->uc.uc_mcontext, fp, regs, set, Efault); + unsafe_put_user(*(__u64 *)set, (__u64 *)&frame->uc.uc_sigmask, Efault); + user_access_end(); + + if (__copy_siginfo_to_user32(&frame->info, &ksig->info, false)) return -EFAULT; /* Set up registers for signal handler */ @@ -402,4 +369,7 @@ int ia32_setup_rt_frame(int sig, struct ksignal *ksig, regs->ss = __USER32_DS; return 0; +Efault: + user_access_end(); + return -EFAULT; } diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h index 1ae4e5791afa..c7df20e78b09 100644 --- a/arch/x86/include/asm/amd_nb.h +++ b/arch/x86/include/asm/amd_nb.h @@ -12,7 +12,6 @@ struct amd_nb_bus_dev_range { u8 dev_limit; }; -extern const struct pci_device_id amd_nb_misc_ids[]; extern const struct amd_nb_bus_dev_range amd_nb_bus_dev_ranges[]; extern bool early_is_amd_nb(u32 value); diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index cd339b88d5d4..0f63585edf5f 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -138,9 +138,6 @@ # define _ASM_EXTABLE_FAULT(from, to) \ _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) -# define _ASM_EXTABLE_EX(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_ext) - # define _ASM_NOKPROBE(entry) \ .pushsection "_kprobe_blacklist","aw" ; \ _ASM_ALIGN ; \ @@ -166,9 +163,6 @@ # define _ASM_EXTABLE_FAULT(from, to) \ _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) -# define _ASM_EXTABLE_EX(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_ext) - /* For C file, we already have NOKPROBE_SYMBOL macro */ #endif diff --git a/arch/x86/include/asm/clocksource.h b/arch/x86/include/asm/clocksource.h index dc4cfc888d6d..dc9dc7b3911a 100644 --- a/arch/x86/include/asm/clocksource.h +++ b/arch/x86/include/asm/clocksource.h @@ -4,14 +4,18 @@ #ifndef _ASM_X86_CLOCKSOURCE_H #define _ASM_X86_CLOCKSOURCE_H -#define VCLOCK_NONE 0 /* No vDSO clock available. */ -#define VCLOCK_TSC 1 /* vDSO should use vread_tsc. */ -#define VCLOCK_PVCLOCK 2 /* vDSO should use vread_pvclock. */ -#define VCLOCK_HVCLOCK 3 /* vDSO should use vread_hvclock. */ -#define VCLOCK_MAX 3 +#include <asm/vdso/clocksource.h> -struct arch_clocksource_data { - int vclock_mode; -}; +extern unsigned int vclocks_used; + +static inline bool vclock_was_used(int vclock) +{ + return READ_ONCE(vclocks_used) & (1U << vclock); +} + +static inline void vclocks_set_used(unsigned int which) +{ + WRITE_ONCE(vclocks_used, READ_ONCE(vclocks_used) | (1 << which)); +} #endif /* _ASM_X86_CLOCKSOURCE_H */ diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h index adc6cc86b062..ff6f3ca649b3 100644 --- a/arch/x86/include/asm/cpu.h +++ b/arch/x86/include/asm/cpu.h @@ -40,4 +40,16 @@ int mwait_usable(const struct cpuinfo_x86 *); unsigned int x86_family(unsigned int sig); unsigned int x86_model(unsigned int sig); unsigned int x86_stepping(unsigned int sig); +#ifdef CONFIG_CPU_SUP_INTEL +extern void __init cpu_set_core_cap_bits(struct cpuinfo_x86 *c); +extern void switch_to_sld(unsigned long tifn); +extern bool handle_user_split_lock(struct pt_regs *regs, long error_code); +#else +static inline void __init cpu_set_core_cap_bits(struct cpuinfo_x86 *c) {} +static inline void switch_to_sld(unsigned long tifn) {} +static inline bool handle_user_split_lock(struct pt_regs *regs, long error_code) +{ + return false; +} +#endif #endif /* _ASM_X86_CPU_H */ diff --git a/arch/x86/include/asm/cpu_device_id.h b/arch/x86/include/asm/cpu_device_id.h index 31c379c1da41..cf3d621c6892 100644 --- a/arch/x86/include/asm/cpu_device_id.h +++ b/arch/x86/include/asm/cpu_device_id.h @@ -5,9 +5,139 @@ /* * Declare drivers belonging to specific x86 CPUs * Similar in spirit to pci_device_id and related PCI functions + * + * The wildcard initializers are in mod_devicetable.h because + * file2alias needs them. Sigh. */ - #include <linux/mod_devicetable.h> +/* Get the INTEL_FAM* model defines */ +#include <asm/intel-family.h> +/* And the X86_VENDOR_* ones */ +#include <asm/processor.h> + +/* Centaur FAM6 models */ +#define X86_CENTAUR_FAM6_C7_A 0xa +#define X86_CENTAUR_FAM6_C7_D 0xd +#define X86_CENTAUR_FAM6_NANO 0xf + +/** + * X86_MATCH_VENDOR_FAM_MODEL_FEATURE - Base macro for CPU matching + * @_vendor: The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY + * The name is expanded to X86_VENDOR_@_vendor + * @_family: The family number or X86_FAMILY_ANY + * @_model: The model number, model constant or X86_MODEL_ANY + * @_feature: A X86_FEATURE bit or X86_FEATURE_ANY + * @_data: Driver specific data or NULL. The internal storage + * format is unsigned long. The supplied value, pointer + * etc. is casted to unsigned long internally. + * + * Use only if you need all selectors. Otherwise use one of the shorter + * macros of the X86_MATCH_* family. If there is no matching shorthand + * macro, consider to add one. If you really need to wrap one of the macros + * into another macro at the usage site for good reasons, then please + * start this local macro with X86_MATCH to allow easy grepping. + */ +#define X86_MATCH_VENDOR_FAM_MODEL_FEATURE(_vendor, _family, _model, \ + _feature, _data) { \ + .vendor = X86_VENDOR_##_vendor, \ + .family = _family, \ + .model = _model, \ + .feature = _feature, \ + .driver_data = (unsigned long) _data \ +} + +/** + * X86_MATCH_VENDOR_FAM_FEATURE - Macro for matching vendor, family and CPU feature + * @vendor: The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY + * The name is expanded to X86_VENDOR_@vendor + * @family: The family number or X86_FAMILY_ANY + * @feature: A X86_FEATURE bit + * @data: Driver specific data or NULL. The internal storage + * format is unsigned long. The supplied value, pointer + * etc. is casted to unsigned long internally. + * + * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are + * set to wildcards. + */ +#define X86_MATCH_VENDOR_FAM_FEATURE(vendor, family, feature, data) \ + X86_MATCH_VENDOR_FAM_MODEL_FEATURE(vendor, family, \ + X86_MODEL_ANY, feature, data) + +/** + * X86_MATCH_VENDOR_FEATURE - Macro for matching vendor and CPU feature + * @vendor: The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY + * The name is expanded to X86_VENDOR_@vendor + * @feature: A X86_FEATURE bit + * @data: Driver specific data or NULL. The internal storage + * format is unsigned long. The supplied value, pointer + * etc. is casted to unsigned long internally. + * + * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are + * set to wildcards. + */ +#define X86_MATCH_VENDOR_FEATURE(vendor, feature, data) \ + X86_MATCH_VENDOR_FAM_FEATURE(vendor, X86_FAMILY_ANY, feature, data) + +/** + * X86_MATCH_FEATURE - Macro for matching a CPU feature + * @feature: A X86_FEATURE bit + * @data: Driver specific data or NULL. The internal storage + * format is unsigned long. The supplied value, pointer + * etc. is casted to unsigned long internally. + * + * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are + * set to wildcards. + */ +#define X86_MATCH_FEATURE(feature, data) \ + X86_MATCH_VENDOR_FEATURE(ANY, feature, data) + +/** + * X86_MATCH_VENDOR_FAM_MODEL - Match vendor, family and model + * @vendor: The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY + * The name is expanded to X86_VENDOR_@vendor + * @family: The family number or X86_FAMILY_ANY + * @model: The model number, model constant or X86_MODEL_ANY + * @data: Driver specific data or NULL. The internal storage + * format is unsigned long. The supplied value, pointer + * etc. is casted to unsigned long internally. + * + * All other missing arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are + * set to wildcards. + */ +#define X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, data) \ + X86_MATCH_VENDOR_FAM_MODEL_FEATURE(vendor, family, model, \ + X86_FEATURE_ANY, data) + +/** + * X86_MATCH_VENDOR_FAM - Match vendor and family + * @vendor: The vendor name, e.g. INTEL, AMD, HYGON, ..., ANY + * The name is expanded to X86_VENDOR_@vendor + * @family: The family number or X86_FAMILY_ANY + * @data: Driver specific data or NULL. The internal storage + * format is unsigned long. The supplied value, pointer + * etc. is casted to unsigned long internally. + * + * All other missing arguments to X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are + * set of wildcards. + */ +#define X86_MATCH_VENDOR_FAM(vendor, family, data) \ + X86_MATCH_VENDOR_FAM_MODEL(vendor, family, X86_MODEL_ANY, data) + +/** + * X86_MATCH_INTEL_FAM6_MODEL - Match vendor INTEL, family 6 and model + * @model: The model name without the INTEL_FAM6_ prefix or ANY + * The model name is expanded to INTEL_FAM6_@model internally + * @data: Driver specific data or NULL. The internal storage + * format is unsigned long. The supplied value, pointer + * etc. is casted to unsigned long internally. + * + * The vendor is set to INTEL, the family to 6 and all other missing + * arguments of X86_MATCH_VENDOR_FAM_MODEL_FEATURE() are set to wildcards. + * + * See X86_MATCH_VENDOR_FAM_MODEL_FEATURE() for further information. + */ +#define X86_MATCH_INTEL_FAM6_MODEL(model, data) \ + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, INTEL_FAM6_##model, data) /* * Match specific microcode revisions. diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index f3327cb56edf..db189945e9b0 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -217,7 +217,7 @@ #define X86_FEATURE_IBRS ( 7*32+25) /* Indirect Branch Restricted Speculation */ #define X86_FEATURE_IBPB ( 7*32+26) /* Indirect Branch Prediction Barrier */ #define X86_FEATURE_STIBP ( 7*32+27) /* Single Thread Indirect Branch Predictors */ -#define X86_FEATURE_ZEN ( 7*32+28) /* "" CPU is AMD family 0x17 (Zen) */ +#define X86_FEATURE_ZEN ( 7*32+28) /* "" CPU is AMD family 0x17 or above (Zen) */ #define X86_FEATURE_L1TF_PTEINV ( 7*32+29) /* "" L1TF workaround PTE inversion */ #define X86_FEATURE_IBRS_ENHANCED ( 7*32+30) /* Enhanced IBRS */ #define X86_FEATURE_MSR_IA32_FEAT_CTL ( 7*32+31) /* "" MSR IA32_FEAT_CTL configured */ @@ -285,6 +285,7 @@ #define X86_FEATURE_CQM_MBM_LOCAL (11*32+ 3) /* LLC Local MBM monitoring */ #define X86_FEATURE_FENCE_SWAPGS_USER (11*32+ 4) /* "" LFENCE in user entry SWAPGS path */ #define X86_FEATURE_FENCE_SWAPGS_KERNEL (11*32+ 5) /* "" LFENCE in kernel entry SWAPGS path */ +#define X86_FEATURE_SPLIT_LOCK_DETECT (11*32+ 6) /* #AC for split lock */ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX512_BF16 (12*32+ 5) /* AVX512 BFLOAT16 instructions */ @@ -299,6 +300,7 @@ #define X86_FEATURE_AMD_IBRS (13*32+14) /* "" Indirect Branch Restricted Speculation */ #define X86_FEATURE_AMD_STIBP (13*32+15) /* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_AMD_STIBP_ALWAYS_ON (13*32+17) /* "" Single Thread Indirect Branch Predictors always-on preferred */ +#define X86_FEATURE_AMD_PPIN (13*32+23) /* Protected Processor Inventory Number */ #define X86_FEATURE_AMD_SSBD (13*32+24) /* "" Speculative Store Bypass Disable */ #define X86_FEATURE_VIRT_SSBD (13*32+25) /* Virtualized Speculative Store Bypass Disable */ #define X86_FEATURE_AMD_SSB_NO (13*32+26) /* "" Speculative Store Bypass is fixed in hardware. */ @@ -367,6 +369,7 @@ #define X86_FEATURE_INTEL_STIBP (18*32+27) /* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28) /* Flush L1D cache */ #define X86_FEATURE_ARCH_CAPABILITIES (18*32+29) /* IA32_ARCH_CAPABILITIES MSR (Intel) */ +#define X86_FEATURE_CORE_CAPABILITIES (18*32+30) /* "" IA32_CORE_CAPABILITIES MSR */ #define X86_FEATURE_SPEC_CTRL_SSBD (18*32+31) /* "" Speculative Store Bypass Disable */ /* diff --git a/arch/x86/include/asm/dwarf2.h b/arch/x86/include/asm/dwarf2.h index ae391f609840..f71a0cce9373 100644 --- a/arch/x86/include/asm/dwarf2.h +++ b/arch/x86/include/asm/dwarf2.h @@ -42,8 +42,8 @@ * Emit CFI data in .debug_frame sections, not .eh_frame sections. * The latter we currently just discard since we don't do DWARF * unwinding at runtime. So only the offline DWARF information is - * useful to anyone. Note we should not use this directive if - * vmlinux.lds.S gets changed so it doesn't discard .eh_frame. + * useful to anyone. Note we should not use this directive if we + * ever decide to enable DWARF unwinding at runtime. */ .cfi_sections .debug_frame #else diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index 86169a24b0d8..cdcf48d52a12 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -10,6 +10,8 @@ #include <asm/mmu_context.h> #include <linux/build_bug.h> +extern unsigned long efi_fw_vendor, efi_config_table; + /* * We map the EFI regions needed for runtime services non-contiguously, * with preserved alignment on virtual addresses starting from -4G down @@ -34,8 +36,6 @@ static inline bool efi_have_uv1_memmap(void) #define EFI32_LOADER_SIGNATURE "EL32" #define EFI64_LOADER_SIGNATURE "EL64" -#define MAX_CMDLINE_ADDRESS UINT_MAX - #define ARCH_EFI_IRQ_FLAGS_MASK X86_EFLAGS_IF /* @@ -180,7 +180,6 @@ extern void __init efi_uv1_memmap_phys_epilog(pgd_t *save_pgd); struct efi_setup_data { u64 fw_vendor; - u64 runtime; u64 tables; u64 smbios; u64 reserved[8]; @@ -219,7 +218,8 @@ extern void efi_thunk_runtime_setup(void); efi_status_t efi_set_virtual_address_map(unsigned long memory_map_size, unsigned long descriptor_size, u32 descriptor_version, - efi_memory_desc_t *virtual_map); + efi_memory_desc_t *virtual_map, + unsigned long systab_phys); /* arch specific definitions used by the stub code */ @@ -270,6 +270,11 @@ static inline void *efi64_zero_upper(void *p) return p; } +static inline u32 efi64_convert_status(efi_status_t status) +{ + return (u32)(status | (u64)status >> 32); +} + #define __efi64_argmap_free_pages(addr, size) \ ((addr), 0, (size)) @@ -285,11 +290,21 @@ static inline void *efi64_zero_upper(void *p) #define __efi64_argmap_locate_protocol(protocol, reg, interface) \ ((protocol), (reg), efi64_zero_upper(interface)) +#define __efi64_argmap_locate_device_path(protocol, path, handle) \ + ((protocol), (path), efi64_zero_upper(handle)) + +#define __efi64_argmap_exit(handle, status, size, data) \ + ((handle), efi64_convert_status(status), (size), (data)) + /* PCI I/O */ #define __efi64_argmap_get_location(protocol, seg, bus, dev, func) \ ((protocol), efi64_zero_upper(seg), efi64_zero_upper(bus), \ efi64_zero_upper(dev), efi64_zero_upper(func)) +/* LoadFile */ +#define __efi64_argmap_load_file(protocol, path, policy, bufsize, buf) \ + ((protocol), (path), (policy), efi64_zero_upper(bufsize), (buf)) + /* * The macros below handle the plumbing for the argument mapping. To add a * mapping for a specific EFI method, simply define a macro diff --git a/arch/x86/include/asm/futex.h b/arch/x86/include/asm/futex.h index 13c83fe97988..f9c00110a69a 100644 --- a/arch/x86/include/asm/futex.h +++ b/arch/x86/include/asm/futex.h @@ -12,76 +12,103 @@ #include <asm/processor.h> #include <asm/smap.h> -#define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \ - asm volatile("\t" ASM_STAC "\n" \ - "1:\t" insn "\n" \ - "2:\t" ASM_CLAC "\n" \ +#define unsafe_atomic_op1(insn, oval, uaddr, oparg, label) \ +do { \ + int oldval = 0, ret; \ + asm volatile("1:\t" insn "\n" \ + "2:\n" \ "\t.section .fixup,\"ax\"\n" \ "3:\tmov\t%3, %1\n" \ "\tjmp\t2b\n" \ "\t.previous\n" \ _ASM_EXTABLE_UA(1b, 3b) \ : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ - : "i" (-EFAULT), "0" (oparg), "1" (0)) + : "i" (-EFAULT), "0" (oparg), "1" (0)); \ + if (ret) \ + goto label; \ + *oval = oldval; \ +} while(0) -#define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \ - asm volatile("\t" ASM_STAC "\n" \ - "1:\tmovl %2, %0\n" \ - "\tmovl\t%0, %3\n" \ + +#define unsafe_atomic_op2(insn, oval, uaddr, oparg, label) \ +do { \ + int oldval = 0, ret, tem; \ + asm volatile("1:\tmovl %2, %0\n" \ + "2:\tmovl\t%0, %3\n" \ "\t" insn "\n" \ - "2:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ - "\tjnz\t1b\n" \ - "3:\t" ASM_CLAC "\n" \ + "3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ + "\tjnz\t2b\n" \ + "4:\n" \ "\t.section .fixup,\"ax\"\n" \ - "4:\tmov\t%5, %1\n" \ - "\tjmp\t3b\n" \ + "5:\tmov\t%5, %1\n" \ + "\tjmp\t4b\n" \ "\t.previous\n" \ - _ASM_EXTABLE_UA(1b, 4b) \ - _ASM_EXTABLE_UA(2b, 4b) \ + _ASM_EXTABLE_UA(1b, 5b) \ + _ASM_EXTABLE_UA(3b, 5b) \ : "=&a" (oldval), "=&r" (ret), \ "+m" (*uaddr), "=&r" (tem) \ - : "r" (oparg), "i" (-EFAULT), "1" (0)) + : "r" (oparg), "i" (-EFAULT), "1" (0)); \ + if (ret) \ + goto label; \ + *oval = oldval; \ +} while(0) -static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, +static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { - int oldval = 0, ret, tem; - - pagefault_disable(); + if (!user_access_begin(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: - __futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg); + unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault); break; case FUTEX_OP_ADD: - __futex_atomic_op1(LOCK_PREFIX "xaddl %0, %2", ret, oldval, - uaddr, oparg); + unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval, + uaddr, oparg, Efault); break; case FUTEX_OP_OR: - __futex_atomic_op2("orl %4, %3", ret, oldval, uaddr, oparg); + unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault); break; case FUTEX_OP_ANDN: - __futex_atomic_op2("andl %4, %3", ret, oldval, uaddr, ~oparg); + unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault); break; case FUTEX_OP_XOR: - __futex_atomic_op2("xorl %4, %3", ret, oldval, uaddr, oparg); + unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault); break; default: - ret = -ENOSYS; + user_access_end(); + return -ENOSYS; } - - pagefault_enable(); - - if (!ret) - *oval = oldval; - - return ret; + user_access_end(); + return 0; +Efault: + user_access_end(); + return -EFAULT; } static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { - return user_atomic_cmpxchg_inatomic(uval, uaddr, oldval, newval); + int ret = 0; + + if (!user_access_begin(uaddr, sizeof(u32))) + return -EFAULT; + asm volatile("\n" + "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" + "2:\n" + "\t.section .fixup, \"ax\"\n" + "3:\tmov %3, %0\n" + "\tjmp 2b\n" + "\t.previous\n" + _ASM_EXTABLE_UA(1b, 3b) + : "+r" (ret), "=a" (oldval), "+m" (*uaddr) + : "i" (-EFAULT), "r" (newval), "1" (oldval) + : "memory" + ); + user_access_end(); + *uval = oldval; + return ret; } #endif diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index 4981c293f926..8f1e94f29a16 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -35,6 +35,9 @@ * The #define line may optionally include a comment including platform names. */ +/* Wildcard match for FAM6 so X86_MATCH_INTEL_FAM6_MODEL(ANY) works */ +#define INTEL_FAM6_ANY X86_MODEL_ANY + #define INTEL_FAM6_CORE_YONAH 0x0E #define INTEL_FAM6_CORE2_MEROM 0x0F @@ -118,17 +121,7 @@ #define INTEL_FAM6_XEON_PHI_KNL 0x57 /* Knights Landing */ #define INTEL_FAM6_XEON_PHI_KNM 0x85 /* Knights Mill */ -/* Useful macros */ -#define INTEL_CPU_FAM_ANY(_family, _model, _driver_data) \ -{ \ - .vendor = X86_VENDOR_INTEL, \ - .family = _family, \ - .model = _model, \ - .feature = X86_FEATURE_ANY, \ - .driver_data = (kernel_ulong_t)&_driver_data \ -} - -#define INTEL_CPU_FAM6(_model, _driver_data) \ - INTEL_CPU_FAM_ANY(6, INTEL_FAM6_##_model, _driver_data) +/* Family 5 */ +#define INTEL_FAM5_QUARK_X1000 0x09 /* Quark X1000 SoC */ #endif /* _ASM_X86_INTEL_FAMILY_H */ diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index a176f6165d85..72fba0eeeb30 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -36,7 +36,7 @@ extern void native_init_IRQ(void); extern void handle_irq(struct irq_desc *desc, struct pt_regs *regs); -extern __visible unsigned int do_IRQ(struct pt_regs *regs); +extern __visible void do_IRQ(struct pt_regs *regs); extern void init_ISA_irqs(void); diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h index 95b1f053bd96..073eb7ad2f56 100644 --- a/arch/x86/include/asm/kprobes.h +++ b/arch/x86/include/asm/kprobes.h @@ -36,6 +36,7 @@ typedef u8 kprobe_opcode_t; /* optinsn template addresses */ extern __visible kprobe_opcode_t optprobe_template_entry[]; +extern __visible kprobe_opcode_t optprobe_template_clac[]; extern __visible kprobe_opcode_t optprobe_template_val[]; extern __visible kprobe_opcode_t optprobe_template_call[]; extern __visible kprobe_opcode_t optprobe_template_end[]; diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index 4359b955e0b7..f9cea081c05b 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -102,7 +102,7 @@ #define MCE_OVERFLOW 0 /* bit 0 in flags means overflow */ -#define MCE_LOG_LEN 32 +#define MCE_LOG_MIN_LEN 32U #define MCE_LOG_SIGNATURE "MACHINECHECK" /* AMD Scalable MCA */ @@ -135,11 +135,11 @@ */ struct mce_log_buffer { char signature[12]; /* "MACHINECHECK" */ - unsigned len; /* = MCE_LOG_LEN */ + unsigned len; /* = elements in .mce_entry[] */ unsigned next; unsigned flags; unsigned recordlen; /* length of struct mce */ - struct mce entry[MCE_LOG_LEN]; + struct mce entry[]; }; enum mce_notifier_prios { @@ -238,9 +238,6 @@ extern void mce_disable_bank(int bank); /* * Exception handler */ - -/* Call the installed machine check handler for this CPU setup. */ -extern void (*machine_check_vector)(struct pt_regs *, long error_code); void do_machine_check(struct pt_regs *, long); /* diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index 6b79515abb82..edc2c581704a 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -46,7 +46,9 @@ typedef int (*hyperv_fill_flush_list_func)( #define hv_set_reference_tsc(val) \ wrmsrl(HV_X64_MSR_REFERENCE_TSC, val) #define hv_set_clocksource_vdso(val) \ - ((val).archdata.vclock_mode = VCLOCK_HVCLOCK) + ((val).vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK) +#define hv_enable_vdso_clocksource() \ + vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK); #define hv_get_raw_timer() rdtsc_ordered() void hyperv_callback_vector(void); diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index d5e517d1c3dd..12c9684d59ba 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -41,6 +41,10 @@ /* Intel MSRs. Some also available on other CPUs */ +#define MSR_TEST_CTRL 0x00000033 +#define MSR_TEST_CTRL_SPLIT_LOCK_DETECT_BIT 29 +#define MSR_TEST_CTRL_SPLIT_LOCK_DETECT BIT(MSR_TEST_CTRL_SPLIT_LOCK_DETECT_BIT) + #define MSR_IA32_SPEC_CTRL 0x00000048 /* Speculation Control */ #define SPEC_CTRL_IBRS BIT(0) /* Indirect Branch Restricted Speculation */ #define SPEC_CTRL_STIBP_SHIFT 1 /* Single Thread Indirect Branch Predictor (STIBP) bit */ @@ -70,6 +74,11 @@ */ #define MSR_IA32_UMWAIT_CONTROL_TIME_MASK (~0x03U) +/* Abbreviated from Intel SDM name IA32_CORE_CAPABILITIES */ +#define MSR_IA32_CORE_CAPS 0x000000cf +#define MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT_BIT 5 +#define MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT BIT(MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT_BIT) + #define MSR_PKG_CST_CONFIG_CONTROL 0x000000e2 #define NHM_C3_AUTO_DEMOTE (1UL << 25) #define NHM_C1_AUTO_DEMOTE (1UL << 26) diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index 9d5252c9685c..b809f117f3f4 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -23,6 +23,8 @@ #define MWAITX_MAX_LOOPS ((u32)-1) #define MWAITX_DISABLE_CSTATES 0xf0 +u32 get_umwait_control_msr(void); + static inline void __monitor(const void *eax, unsigned long ecx, unsigned long edx) { diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 29964b0e1075..e855e9cf2c37 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -50,11 +50,22 @@ #define AMD64_L3_SLICE_SHIFT 48 #define AMD64_L3_SLICE_MASK \ - ((0xFULL) << AMD64_L3_SLICE_SHIFT) + (0xFULL << AMD64_L3_SLICE_SHIFT) +#define AMD64_L3_SLICEID_MASK \ + (0x7ULL << AMD64_L3_SLICE_SHIFT) #define AMD64_L3_THREAD_SHIFT 56 #define AMD64_L3_THREAD_MASK \ - ((0xFFULL) << AMD64_L3_THREAD_SHIFT) + (0xFFULL << AMD64_L3_THREAD_SHIFT) +#define AMD64_L3_F19H_THREAD_MASK \ + (0x3ULL << AMD64_L3_THREAD_SHIFT) + +#define AMD64_L3_EN_ALL_CORES BIT_ULL(47) +#define AMD64_L3_EN_ALL_SLICES BIT_ULL(46) + +#define AMD64_L3_COREID_SHIFT 42 +#define AMD64_L3_COREID_MASK \ + (0x7ULL << AMD64_L3_COREID_SHIFT) #define X86_RAW_EVENT_MASK \ (ARCH_PERFMON_EVENTSEL_EVENT | \ diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 7e118660bbd9..afda66a6d325 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -595,12 +595,6 @@ static inline pmd_t pmd_mknotpresent(pmd_t pmd) __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE))); } -static inline pud_t pud_mknotpresent(pud_t pud) -{ - return pfn_pud(pud_pfn(pud), - __pgprot(pud_flags(pud) & ~(_PAGE_PRESENT|_PAGE_PROTNONE))); -} - static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask); static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) @@ -627,12 +621,15 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) return __pmd(val); } -/* mprotect needs to preserve PAT bits when updating vm_page_prot */ +/* + * mprotect needs to preserve PAT and encryption bits when updating + * vm_page_prot + */ #define pgprot_modify pgprot_modify static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) { pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK; - pgprotval_t addbits = pgprot_val(newprot); + pgprotval_t addbits = pgprot_val(newprot) & ~_PAGE_CHG_MASK; return __pgprot(preservebits | addbits); } diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 0239998d8cdc..65c2ecd730c5 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -118,7 +118,7 @@ */ #define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \ - _PAGE_SOFT_DIRTY | _PAGE_DEVMAP) + _PAGE_SOFT_DIRTY | _PAGE_DEVMAP | _PAGE_ENC) #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) /* diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index 19b137f1b3be..2ff9b98812b7 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -4,6 +4,11 @@ #define ARCH_DEFAULT_PKEY 0 +/* + * If more than 16 keys are ever supported, a thorough audit + * will be necessary to ensure that the types that store key + * numbers and masks have sufficient capacity. + */ #define arch_max_pkey() (boot_cpu_has(X86_FEATURE_OSPKE) ? 16 : 1) extern int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 3d4cb83a8828..69485ca13665 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -103,14 +103,14 @@ static __always_inline bool should_resched(int preempt_offset) } #ifdef CONFIG_PREEMPTION - extern asmlinkage void ___preempt_schedule(void); + extern asmlinkage void preempt_schedule_thunk(void); # define __preempt_schedule() \ - asm volatile ("call ___preempt_schedule" : ASM_CALL_CONSTRAINT) + asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT) extern asmlinkage void preempt_schedule(void); - extern asmlinkage void ___preempt_schedule_notrace(void); + extern asmlinkage void preempt_schedule_notrace_thunk(void); # define __preempt_schedule_notrace() \ - asm volatile ("call ___preempt_schedule_notrace" : ASM_CALL_CONSTRAINT) + asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT) extern asmlinkage void preempt_schedule_notrace(void); #endif diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 09705ccc393c..3bcf27caf6c9 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -26,6 +26,7 @@ struct vm86; #include <asm/fpu/types.h> #include <asm/unwind_hints.h> #include <asm/vmxfeatures.h> +#include <asm/vdso/processor.h> #include <linux/personality.h> #include <linux/cache.h> @@ -541,7 +542,6 @@ struct thread_struct { mm_segment_t addr_limit; unsigned int sig_on_uaccess_err:1; - unsigned int uaccess_err:1; /* uaccess failed */ /* Floating point and extended processor state */ struct fpu fpu; @@ -677,17 +677,6 @@ static inline unsigned int cpuid_edx(unsigned int op) return edx; } -/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ -static __always_inline void rep_nop(void) -{ - asm volatile("rep; nop" ::: "memory"); -} - -static __always_inline void cpu_relax(void) -{ - rep_nop(); -} - /* * This function forces the icache and prefetched instruction stream to * catch up with reality in two very specific cases: diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h index 036c360910c5..a6e8373a5170 100644 --- a/arch/x86/include/asm/sections.h +++ b/arch/x86/include/asm/sections.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_SECTIONS_H #define _ASM_X86_SECTIONS_H +#define arch_is_kernel_initmem_freed arch_is_kernel_initmem_freed + #include <asm-generic/sections.h> #include <asm/extable.h> @@ -14,4 +16,22 @@ extern char __end_rodata_hpage_align[]; extern char __end_of_kernel_reserve[]; +extern unsigned long _brk_start, _brk_end; + +static inline bool arch_is_kernel_initmem_freed(unsigned long addr) +{ + /* + * If _brk_start has not been cleared, brk allocation is incomplete, + * and we can not make assumptions about its use. + */ + if (_brk_start) + return 0; + + /* + * After brk allocation is complete, space between _brk_end and _end + * is available for allocation. + */ + return addr >= _brk_end && addr < (unsigned long)&_end; +} + #endif /* _ASM_X86_SECTIONS_H */ diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 64c3dce374e5..950532ccbc4a 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -46,6 +46,8 @@ int set_memory_4k(unsigned long addr, int numpages); int set_memory_encrypted(unsigned long addr, int numpages); int set_memory_decrypted(unsigned long addr, int numpages); int set_memory_np_noalias(unsigned long addr, int numpages); +int set_memory_nonglobal(unsigned long addr, int numpages); +int set_memory_global(unsigned long addr, int numpages); int set_pages_array_uc(struct page **pages, int addrinarray); int set_pages_array_wc(struct page **pages, int addrinarray); diff --git a/arch/x86/include/asm/sigframe.h b/arch/x86/include/asm/sigframe.h index f176114c04d4..84eab2724875 100644 --- a/arch/x86/include/asm/sigframe.h +++ b/arch/x86/include/asm/sigframe.h @@ -33,11 +33,7 @@ struct sigframe_ia32 { * legacy application accessing/modifying it. */ struct _fpstate_32 fpstate_unused; -#ifdef CONFIG_IA32_EMULATION - unsigned int extramask[_COMPAT_NSIG_WORDS-1]; -#else /* !CONFIG_IA32_EMULATION */ - unsigned long extramask[_NSIG_WORDS-1]; -#endif /* CONFIG_IA32_EMULATION */ + unsigned int extramask[1]; char retcode[8]; /* fp state follows here */ }; diff --git a/arch/x86/include/asm/sighandling.h b/arch/x86/include/asm/sighandling.h index 2fcbd6f33ef7..65e667279e0f 100644 --- a/arch/x86/include/asm/sighandling.h +++ b/arch/x86/include/asm/sighandling.h @@ -14,12 +14,5 @@ X86_EFLAGS_CF | X86_EFLAGS_RF) void signal_fault(struct pt_regs *regs, void __user *frame, char *where); -int setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate, - struct pt_regs *regs, unsigned long mask); - - -#ifdef CONFIG_X86_X32_ABI -asmlinkage long sys32_x32_rt_sigreturn(void); -#endif #endif /* _ASM_X86_SIGHANDLING_H */ diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h index 8db3fdb6102e..7cbf733d11af 100644 --- a/arch/x86/include/asm/syscall.h +++ b/arch/x86/include/asm/syscall.h @@ -13,23 +13,14 @@ #include <uapi/linux/audit.h> #include <linux/sched.h> #include <linux/err.h> -#include <asm/asm-offsets.h> /* For NR_syscalls */ #include <asm/thread_info.h> /* for TS_COMPAT */ #include <asm/unistd.h> -#ifdef CONFIG_X86_64 -typedef asmlinkage long (*sys_call_ptr_t)(const struct pt_regs *); -#else -typedef asmlinkage long (*sys_call_ptr_t)(unsigned long, unsigned long, - unsigned long, unsigned long, - unsigned long, unsigned long); -#endif /* CONFIG_X86_64 */ +typedef long (*sys_call_ptr_t)(const struct pt_regs *); extern const sys_call_ptr_t sys_call_table[]; #if defined(CONFIG_X86_32) #define ia32_sys_call_table sys_call_table -#define __NR_syscall_compat_max __NR_syscall_max -#define IA32_NR_syscalls NR_syscalls #endif #if defined(CONFIG_IA32_EMULATION) @@ -168,6 +159,11 @@ static inline int syscall_get_arch(struct task_struct *task) task->thread_info.status & TS_COMPAT) ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64; } + +void do_syscall_64(unsigned long nr, struct pt_regs *regs); +void do_int80_syscall_32(struct pt_regs *regs); +long do_fast_syscall_32(struct pt_regs *regs); + #endif /* CONFIG_X86_32 */ #endif /* _ASM_X86_SYSCALL_H */ diff --git a/arch/x86/include/asm/syscall_wrapper.h b/arch/x86/include/asm/syscall_wrapper.h index e2389ce9bf58..a84333adeef2 100644 --- a/arch/x86/include/asm/syscall_wrapper.h +++ b/arch/x86/include/asm/syscall_wrapper.h @@ -8,6 +8,50 @@ struct pt_regs; +extern long __x64_sys_ni_syscall(const struct pt_regs *regs); +extern long __ia32_sys_ni_syscall(const struct pt_regs *regs); + +/* + * Instead of the generic __SYSCALL_DEFINEx() definition, the x86 version takes + * struct pt_regs *regs as the only argument of the syscall stub(s) named as: + * __x64_sys_*() - 64-bit native syscall + * __ia32_sys_*() - 32-bit native syscall or common compat syscall + * __ia32_compat_sys_*() - 32-bit compat syscall + * __x32_compat_sys_*() - 64-bit X32 compat syscall + * + * The registers are decoded according to the ABI: + * 64-bit: RDI, RSI, RDX, R10, R8, R9 + * 32-bit: EBX, ECX, EDX, ESI, EDI, EBP + * + * The stub then passes the decoded arguments to the __se_sys_*() wrapper to + * perform sign-extension (omitted for zero-argument syscalls). Finally the + * arguments are passed to the __do_sys_*() function which is the actual + * syscall. These wrappers are marked as inline so the compiler can optimize + * the functions where appropriate. + * + * Example assembly (slightly re-ordered for better readability): + * + * <__x64_sys_recv>: <-- syscall with 4 parameters + * callq <__fentry__> + * + * mov 0x70(%rdi),%rdi <-- decode regs->di + * mov 0x68(%rdi),%rsi <-- decode regs->si + * mov 0x60(%rdi),%rdx <-- decode regs->dx + * mov 0x38(%rdi),%rcx <-- decode regs->r10 + * + * xor %r9d,%r9d <-- clear %r9 + * xor %r8d,%r8d <-- clear %r8 + * + * callq __sys_recvfrom <-- do the actual work in __sys_recvfrom() + * which takes 6 arguments + * + * cltq <-- extend return value to 64-bit + * retq <-- return + * + * This approach avoids leaking random user-provided register content down + * the call chain. + */ + /* Mapping of registers to parameters for syscalls on x86-64 and x32 */ #define SC_X86_64_REGS_TO_ARGS(x, ...) \ __MAP(x,__SC_ARGS \ @@ -21,68 +65,96 @@ struct pt_regs; ,,(unsigned int)regs->dx,,(unsigned int)regs->si \ ,,(unsigned int)regs->di,,(unsigned int)regs->bp) -#ifdef CONFIG_IA32_EMULATION -/* - * For IA32 emulation, we need to handle "compat" syscalls *and* create - * additional wrappers (aptly named __ia32_sys_xyzzy) which decode the - * ia32 regs in the proper order for shared or "common" syscalls. As some - * syscalls may not be implemented, we need to expand COND_SYSCALL in - * kernel/sys_ni.c and SYS_NI in kernel/time/posix-stubs.c to cover this - * case as well. - */ -#define __IA32_COMPAT_SYS_STUB0(x, name) \ - asmlinkage long __ia32_compat_sys_##name(const struct pt_regs *regs);\ - ALLOW_ERROR_INJECTION(__ia32_compat_sys_##name, ERRNO); \ - asmlinkage long __ia32_compat_sys_##name(const struct pt_regs *regs)\ +#define __SYS_STUB0(abi, name) \ + long __##abi##_##name(const struct pt_regs *regs); \ + ALLOW_ERROR_INJECTION(__##abi##_##name, ERRNO); \ + long __##abi##_##name(const struct pt_regs *regs) \ + __alias(__do_##name); + +#define __SYS_STUBx(abi, name, ...) \ + long __##abi##_##name(const struct pt_regs *regs); \ + ALLOW_ERROR_INJECTION(__##abi##_##name, ERRNO); \ + long __##abi##_##name(const struct pt_regs *regs) \ { \ - return __se_compat_sys_##name(); \ + return __se_##name(__VA_ARGS__); \ } -#define __IA32_COMPAT_SYS_STUBx(x, name, ...) \ - asmlinkage long __ia32_compat_sys##name(const struct pt_regs *regs);\ - ALLOW_ERROR_INJECTION(__ia32_compat_sys##name, ERRNO); \ - asmlinkage long __ia32_compat_sys##name(const struct pt_regs *regs)\ +#define __COND_SYSCALL(abi, name) \ + __weak long __##abi##_##name(const struct pt_regs *__unused) \ { \ - return __se_compat_sys##name(SC_IA32_REGS_TO_ARGS(x,__VA_ARGS__));\ + return sys_ni_syscall(); \ } +#define __SYS_NI(abi, name) \ + SYSCALL_ALIAS(__##abi##_##name, sys_ni_posix_timers); + +#ifdef CONFIG_X86_64 +#define __X64_SYS_STUB0(name) \ + __SYS_STUB0(x64, sys_##name) + +#define __X64_SYS_STUBx(x, name, ...) \ + __SYS_STUBx(x64, sys##name, \ + SC_X86_64_REGS_TO_ARGS(x, __VA_ARGS__)) + +#define __X64_COND_SYSCALL(name) \ + __COND_SYSCALL(x64, sys_##name) + +#define __X64_SYS_NI(name) \ + __SYS_NI(x64, sys_##name) +#else /* CONFIG_X86_64 */ +#define __X64_SYS_STUB0(name) +#define __X64_SYS_STUBx(x, name, ...) +#define __X64_COND_SYSCALL(name) +#define __X64_SYS_NI(name) +#endif /* CONFIG_X86_64 */ + +#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION) +#define __IA32_SYS_STUB0(name) \ + __SYS_STUB0(ia32, sys_##name) + #define __IA32_SYS_STUBx(x, name, ...) \ - asmlinkage long __ia32_sys##name(const struct pt_regs *regs); \ - ALLOW_ERROR_INJECTION(__ia32_sys##name, ERRNO); \ - asmlinkage long __ia32_sys##name(const struct pt_regs *regs) \ - { \ - return __se_sys##name(SC_IA32_REGS_TO_ARGS(x,__VA_ARGS__));\ - } + __SYS_STUBx(ia32, sys##name, \ + SC_IA32_REGS_TO_ARGS(x, __VA_ARGS__)) + +#define __IA32_COND_SYSCALL(name) \ + __COND_SYSCALL(ia32, sys_##name) +#define __IA32_SYS_NI(name) \ + __SYS_NI(ia32, sys_##name) +#else /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */ +#define __IA32_SYS_STUB0(name) +#define __IA32_SYS_STUBx(x, name, ...) +#define __IA32_COND_SYSCALL(name) +#define __IA32_SYS_NI(name) +#endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */ + +#ifdef CONFIG_IA32_EMULATION /* - * To keep the naming coherent, re-define SYSCALL_DEFINE0 to create an alias - * named __ia32_sys_*() + * For IA32 emulation, we need to handle "compat" syscalls *and* create + * additional wrappers (aptly named __ia32_sys_xyzzy) which decode the + * ia32 regs in the proper order for shared or "common" syscalls. As some + * syscalls may not be implemented, we need to expand COND_SYSCALL in + * kernel/sys_ni.c and SYS_NI in kernel/time/posix-stubs.c to cover this + * case as well. */ +#define __IA32_COMPAT_SYS_STUB0(name) \ + __SYS_STUB0(ia32, compat_sys_##name) -#define SYSCALL_DEFINE0(sname) \ - SYSCALL_METADATA(_##sname, 0); \ - asmlinkage long __x64_sys_##sname(const struct pt_regs *__unused);\ - ALLOW_ERROR_INJECTION(__x64_sys_##sname, ERRNO); \ - SYSCALL_ALIAS(__ia32_sys_##sname, __x64_sys_##sname); \ - asmlinkage long __x64_sys_##sname(const struct pt_regs *__unused) +#define __IA32_COMPAT_SYS_STUBx(x, name, ...) \ + __SYS_STUBx(ia32, compat_sys##name, \ + SC_IA32_REGS_TO_ARGS(x, __VA_ARGS__)) -#define COND_SYSCALL(name) \ - asmlinkage __weak long __x64_sys_##name(const struct pt_regs *__unused) \ - { \ - return sys_ni_syscall(); \ - } \ - asmlinkage __weak long __ia32_sys_##name(const struct pt_regs *__unused)\ - { \ - return sys_ni_syscall(); \ - } +#define __IA32_COMPAT_COND_SYSCALL(name) \ + __COND_SYSCALL(ia32, compat_sys_##name) -#define SYS_NI(name) \ - SYSCALL_ALIAS(__x64_sys_##name, sys_ni_posix_timers); \ - SYSCALL_ALIAS(__ia32_sys_##name, sys_ni_posix_timers) +#define __IA32_COMPAT_SYS_NI(name) \ + __SYS_NI(ia32, compat_sys_##name) #else /* CONFIG_IA32_EMULATION */ +#define __IA32_COMPAT_SYS_STUB0(name) #define __IA32_COMPAT_SYS_STUBx(x, name, ...) -#define __IA32_SYS_STUBx(x, fullname, name, ...) +#define __IA32_COMPAT_COND_SYSCALL(name) +#define __IA32_COMPAT_SYS_NI(name) #endif /* CONFIG_IA32_EMULATION */ @@ -92,25 +164,23 @@ struct pt_regs; * of the x86-64-style parameter ordering of x32 syscalls. The syscalls common * with x86_64 obviously do not need such care. */ -#define __X32_COMPAT_SYS_STUB0(x, name, ...) \ - asmlinkage long __x32_compat_sys_##name(const struct pt_regs *regs);\ - ALLOW_ERROR_INJECTION(__x32_compat_sys_##name, ERRNO); \ - asmlinkage long __x32_compat_sys_##name(const struct pt_regs *regs)\ - { \ - return __se_compat_sys_##name();\ - } +#define __X32_COMPAT_SYS_STUB0(name) \ + __SYS_STUB0(x32, compat_sys_##name) #define __X32_COMPAT_SYS_STUBx(x, name, ...) \ - asmlinkage long __x32_compat_sys##name(const struct pt_regs *regs);\ - ALLOW_ERROR_INJECTION(__x32_compat_sys##name, ERRNO); \ - asmlinkage long __x32_compat_sys##name(const struct pt_regs *regs)\ - { \ - return __se_compat_sys##name(SC_X86_64_REGS_TO_ARGS(x,__VA_ARGS__));\ - } + __SYS_STUBx(x32, compat_sys##name, \ + SC_X86_64_REGS_TO_ARGS(x, __VA_ARGS__)) + +#define __X32_COMPAT_COND_SYSCALL(name) \ + __COND_SYSCALL(x32, compat_sys_##name) +#define __X32_COMPAT_SYS_NI(name) \ + __SYS_NI(x32, compat_sys_##name) #else /* CONFIG_X86_X32 */ -#define __X32_COMPAT_SYS_STUB0(x, name) +#define __X32_COMPAT_SYS_STUB0(name) #define __X32_COMPAT_SYS_STUBx(x, name, ...) +#define __X32_COMPAT_COND_SYSCALL(name) +#define __X32_COMPAT_SYS_NI(name) #endif /* CONFIG_X86_X32 */ @@ -121,15 +191,12 @@ struct pt_regs; * of them. */ #define COMPAT_SYSCALL_DEFINE0(name) \ - static long __se_compat_sys_##name(void); \ - static inline long __do_compat_sys_##name(void); \ - __IA32_COMPAT_SYS_STUB0(x, name) \ - __X32_COMPAT_SYS_STUB0(x, name) \ - static long __se_compat_sys_##name(void) \ - { \ - return __do_compat_sys_##name(); \ - } \ - static inline long __do_compat_sys_##name(void) + static long \ + __do_compat_sys_##name(const struct pt_regs *__unused); \ + __IA32_COMPAT_SYS_STUB0(name) \ + __X32_COMPAT_SYS_STUB0(name) \ + static long \ + __do_compat_sys_##name(const struct pt_regs *__unused) #define COMPAT_SYSCALL_DEFINEx(x, name, ...) \ static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ @@ -148,58 +215,19 @@ struct pt_regs; * kernel/time/posix-stubs.c to cover this case as well. */ #define COND_SYSCALL_COMPAT(name) \ - cond_syscall(__ia32_compat_sys_##name); \ - cond_syscall(__x32_compat_sys_##name) + __IA32_COMPAT_COND_SYSCALL(name) \ + __X32_COMPAT_COND_SYSCALL(name) #define COMPAT_SYS_NI(name) \ - SYSCALL_ALIAS(__ia32_compat_sys_##name, sys_ni_posix_timers); \ - SYSCALL_ALIAS(__x32_compat_sys_##name, sys_ni_posix_timers) + __IA32_COMPAT_SYS_NI(name) \ + __X32_COMPAT_SYS_NI(name) #endif /* CONFIG_COMPAT */ - -/* - * Instead of the generic __SYSCALL_DEFINEx() definition, this macro takes - * struct pt_regs *regs as the only argument of the syscall stub named - * __x64_sys_*(). It decodes just the registers it needs and passes them on to - * the __se_sys_*() wrapper performing sign extension and then to the - * __do_sys_*() function doing the actual job. These wrappers and functions - * are inlined (at least in very most cases), meaning that the assembly looks - * as follows (slightly re-ordered for better readability): - * - * <__x64_sys_recv>: <-- syscall with 4 parameters - * callq <__fentry__> - * - * mov 0x70(%rdi),%rdi <-- decode regs->di - * mov 0x68(%rdi),%rsi <-- decode regs->si - * mov 0x60(%rdi),%rdx <-- decode regs->dx - * mov 0x38(%rdi),%rcx <-- decode regs->r10 - * - * xor %r9d,%r9d <-- clear %r9 - * xor %r8d,%r8d <-- clear %r8 - * - * callq __sys_recvfrom <-- do the actual work in __sys_recvfrom() - * which takes 6 arguments - * - * cltq <-- extend return value to 64-bit - * retq <-- return - * - * This approach avoids leaking random user-provided register content down - * the call chain. - * - * If IA32_EMULATION is enabled, this macro generates an additional wrapper - * named __ia32_sys_*() which decodes the struct pt_regs *regs according - * to the i386 calling convention (bx, cx, dx, si, di, bp). - */ #define __SYSCALL_DEFINEx(x, name, ...) \ - asmlinkage long __x64_sys##name(const struct pt_regs *regs); \ - ALLOW_ERROR_INJECTION(__x64_sys##name, ERRNO); \ static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));\ - asmlinkage long __x64_sys##name(const struct pt_regs *regs) \ - { \ - return __se_sys##name(SC_X86_64_REGS_TO_ARGS(x,__VA_ARGS__));\ - } \ + __X64_SYS_STUBx(x, name, __VA_ARGS__) \ __IA32_SYS_STUBx(x, name, __VA_ARGS__) \ static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ { \ @@ -217,33 +245,28 @@ struct pt_regs; * SYSCALL_DEFINEx() -- which is essential for the COND_SYSCALL() and SYS_NI() * macros to work correctly. */ -#ifndef SYSCALL_DEFINE0 #define SYSCALL_DEFINE0(sname) \ SYSCALL_METADATA(_##sname, 0); \ - asmlinkage long __x64_sys_##sname(const struct pt_regs *__unused);\ - ALLOW_ERROR_INJECTION(__x64_sys_##sname, ERRNO); \ - asmlinkage long __x64_sys_##sname(const struct pt_regs *__unused) -#endif - -#ifndef COND_SYSCALL -#define COND_SYSCALL(name) \ - asmlinkage __weak long __x64_sys_##name(const struct pt_regs *__unused) \ - { \ - return sys_ni_syscall(); \ - } -#endif + static long __do_sys_##sname(const struct pt_regs *__unused); \ + __X64_SYS_STUB0(sname) \ + __IA32_SYS_STUB0(sname) \ + static long __do_sys_##sname(const struct pt_regs *__unused) + +#define COND_SYSCALL(name) \ + __X64_COND_SYSCALL(name) \ + __IA32_COND_SYSCALL(name) -#ifndef SYS_NI -#define SYS_NI(name) SYSCALL_ALIAS(__x64_sys_##name, sys_ni_posix_timers); -#endif +#define SYS_NI(name) \ + __X64_SYS_NI(name) \ + __IA32_SYS_NI(name) /* * For VSYSCALLS, we need to declare these three syscalls with the new * pt_regs-based calling convention for in-kernel use. */ -asmlinkage long __x64_sys_getcpu(const struct pt_regs *regs); -asmlinkage long __x64_sys_gettimeofday(const struct pt_regs *regs); -asmlinkage long __x64_sys_time(const struct pt_regs *regs); +long __x64_sys_getcpu(const struct pt_regs *regs); +long __x64_sys_gettimeofday(const struct pt_regs *regs); +long __x64_sys_time(const struct pt_regs *regs); #endif /* _ASM_X86_SYSCALL_WRAPPER_H */ diff --git a/arch/x86/include/asm/syscalls.h b/arch/x86/include/asm/syscalls.h index 91b7b6e1a115..6714a358235d 100644 --- a/arch/x86/include/asm/syscalls.h +++ b/arch/x86/include/asm/syscalls.h @@ -8,42 +8,8 @@ #ifndef _ASM_X86_SYSCALLS_H #define _ASM_X86_SYSCALLS_H -#include <linux/compiler.h> -#include <linux/linkage.h> -#include <linux/signal.h> -#include <linux/types.h> - /* Common in X86_32 and X86_64 */ /* kernel/ioport.c */ long ksys_ioperm(unsigned long from, unsigned long num, int turn_on); -#ifdef CONFIG_X86_32 -/* - * These definitions are only valid on pure 32-bit systems; x86-64 uses a - * different syscall calling convention - */ -asmlinkage long sys_ioperm(unsigned long, unsigned long, int); -asmlinkage long sys_iopl(unsigned int); - -/* kernel/ldt.c */ -asmlinkage long sys_modify_ldt(int, void __user *, unsigned long); - -/* kernel/signal.c */ -asmlinkage long sys_rt_sigreturn(void); - -/* kernel/tls.c */ -asmlinkage long sys_set_thread_area(struct user_desc __user *); -asmlinkage long sys_get_thread_area(struct user_desc __user *); - -/* X86_32 only */ - -/* kernel/signal.c */ -asmlinkage long sys_sigreturn(void); - -/* kernel/vm86_32.c */ -struct vm86_struct; -asmlinkage long sys_vm86old(struct vm86_struct __user *); -asmlinkage long sys_vm86(unsigned long, unsigned long); - -#endif /* CONFIG_X86_32 */ #endif /* _ASM_X86_SYSCALLS_H */ diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index cf4327986e98..8de8ceccb8bc 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -92,7 +92,7 @@ struct thread_info { #define TIF_NOCPUID 15 /* CPUID is not accessible in userland */ #define TIF_NOTSC 16 /* TSC is not accessible in userland */ #define TIF_IA32 17 /* IA32 compatibility process */ -#define TIF_NOHZ 19 /* in adaptive nohz mode */ +#define TIF_SLD 18 /* Restore split lock detection on context switch */ #define TIF_MEMDIE 20 /* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21 /* idle is polling for TIF_NEED_RESCHED */ #define TIF_IO_BITMAP 22 /* uses I/O bitmap */ @@ -122,7 +122,7 @@ struct thread_info { #define _TIF_NOCPUID (1 << TIF_NOCPUID) #define _TIF_NOTSC (1 << TIF_NOTSC) #define _TIF_IA32 (1 << TIF_IA32) -#define _TIF_NOHZ (1 << TIF_NOHZ) +#define _TIF_SLD (1 << TIF_SLD) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) #define _TIF_FORCED_TF (1 << TIF_FORCED_TF) @@ -133,19 +133,15 @@ struct thread_info { #define _TIF_X32 (1 << TIF_X32) #define _TIF_FSCHECK (1 << TIF_FSCHECK) -/* - * work to do in syscall_trace_enter(). Also includes TIF_NOHZ for - * enter_from_user_mode() - */ +/* Work to do before invoking the actual syscall. */ #define _TIF_WORK_SYSCALL_ENTRY \ (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_AUDIT | \ - _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \ - _TIF_NOHZ) + _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT) /* flags to check in __switch_to() */ #define _TIF_WORK_CTXSW_BASE \ (_TIF_NOCPUID | _TIF_NOTSC | _TIF_BLOCKSTEP | \ - _TIF_SSBD | _TIF_SPEC_FORCE_UPDATE) + _TIF_SSBD | _TIF_SPEC_FORCE_UPDATE | _TIF_SLD) /* * Avoid calls to __switch_to_xtra() on UP as STIBP is not evaluated. diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 4b14d2318251..79d8d5496330 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -193,4 +193,29 @@ static inline void sched_clear_itmt_support(void) } #endif /* CONFIG_SCHED_MC_PRIO */ +#ifdef CONFIG_SMP +#include <asm/cpufeature.h> + +DECLARE_STATIC_KEY_FALSE(arch_scale_freq_key); + +#define arch_scale_freq_invariant() static_branch_likely(&arch_scale_freq_key) + +DECLARE_PER_CPU(unsigned long, arch_freq_scale); + +static inline long arch_scale_freq_capacity(int cpu) +{ + return per_cpu(arch_freq_scale, cpu); +} +#define arch_scale_freq_capacity arch_scale_freq_capacity + +extern void arch_scale_freq_tick(void); +#define arch_scale_freq_tick arch_scale_freq_tick + +extern void arch_set_max_freq_ratio(bool turbo_disabled); +#else +static inline void arch_set_max_freq_ratio(bool turbo_disabled) +{ +} +#endif + #endif /* _ASM_X86_TOPOLOGY_H */ diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index ffa0dc8a535e..c26a7e1d8a2c 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -76,27 +76,24 @@ dotraplinkage void do_coprocessor_segment_overrun(struct pt_regs *regs, long err dotraplinkage void do_invalid_TSS(struct pt_regs *regs, long error_code); dotraplinkage void do_segment_not_present(struct pt_regs *regs, long error_code); dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code); -#ifdef CONFIG_X86_64 -dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long address); -asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs); -asmlinkage __visible notrace -struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s); -void __init trap_init(void); -#endif dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code); dotraplinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address); dotraplinkage void do_spurious_interrupt_bug(struct pt_regs *regs, long error_code); dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code); dotraplinkage void do_alignment_check(struct pt_regs *regs, long error_code); -#ifdef CONFIG_X86_MCE -dotraplinkage void do_machine_check(struct pt_regs *regs, long error_code); -#endif dotraplinkage void do_simd_coprocessor_error(struct pt_regs *regs, long error_code); #ifdef CONFIG_X86_32 dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code); #endif dotraplinkage void do_mce(struct pt_regs *regs, long error_code); +#ifdef CONFIG_X86_64 +asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs); +asmlinkage __visible notrace +struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s); +void __init trap_init(void); +#endif + static inline int get_si_code(unsigned long condition) { if (condition & DR_STEP) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 61d93f062a36..c8247a84244b 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -193,23 +193,12 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) : : "A" (x), "r" (addr) \ : : label) -#define __put_user_asm_ex_u64(x, addr) \ - asm volatile("\n" \ - "1: movl %%eax,0(%1)\n" \ - "2: movl %%edx,4(%1)\n" \ - "3:" \ - _ASM_EXTABLE_EX(1b, 2b) \ - _ASM_EXTABLE_EX(2b, 3b) \ - : : "A" (x), "r" (addr)) - #define __put_user_x8(x, ptr, __ret_pu) \ asm volatile("call __put_user_8" : "=a" (__ret_pu) \ : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") #else #define __put_user_goto_u64(x, ptr, label) \ __put_user_goto(x, ptr, "q", "", "er", label) -#define __put_user_asm_ex_u64(x, addr) \ - __put_user_asm_ex(x, addr, "q", "", "er") #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu) #endif @@ -289,31 +278,6 @@ do { \ } \ } while (0) -/* - * This doesn't do __uaccess_begin/end - the exception handling - * around it must do that. - */ -#define __put_user_size_ex(x, ptr, size) \ -do { \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __put_user_asm_ex(x, ptr, "b", "b", "iq"); \ - break; \ - case 2: \ - __put_user_asm_ex(x, ptr, "w", "w", "ir"); \ - break; \ - case 4: \ - __put_user_asm_ex(x, ptr, "l", "k", "ir"); \ - break; \ - case 8: \ - __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \ - break; \ - default: \ - __put_user_bad(); \ - } \ -} while (0) - #ifdef CONFIG_X86_32 #define __get_user_asm_u64(x, ptr, retval, errret) \ ({ \ @@ -335,12 +299,9 @@ do { \ "i" (errret), "0" (retval)); \ }) -#define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad() #else #define __get_user_asm_u64(x, ptr, retval, errret) \ __get_user_asm(x, ptr, retval, "q", "", "=r", errret) -#define __get_user_asm_ex_u64(x, ptr) \ - __get_user_asm_ex(x, ptr, "q", "", "=r") #endif #define __get_user_size(x, ptr, size, retval, errret) \ @@ -378,53 +339,6 @@ do { \ : "=r" (err), ltype(x) \ : "m" (__m(addr)), "i" (errret), "0" (err)) -#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("\n" \ - "1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_UA(1b, 3b) \ - : "=r" (err), ltype(x) \ - : "m" (__m(addr)), "i" (errret), "0" (err)) - -/* - * This doesn't do __uaccess_begin/end - the exception handling - * around it must do that. - */ -#define __get_user_size_ex(x, ptr, size) \ -do { \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __get_user_asm_ex(x, ptr, "b", "b", "=q"); \ - break; \ - case 2: \ - __get_user_asm_ex(x, ptr, "w", "w", "=r"); \ - break; \ - case 4: \ - __get_user_asm_ex(x, ptr, "l", "k", "=r"); \ - break; \ - case 8: \ - __get_user_asm_ex_u64(x, ptr); \ - break; \ - default: \ - (x) = __get_user_bad(); \ - } \ -} while (0) - -#define __get_user_asm_ex(x, addr, itype, rtype, ltype) \ - asm volatile("1: mov"itype" %1,%"rtype"0\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3:xor"itype" %"rtype"0,%"rtype"0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_EX(1b, 3b) \ - : ltype(x) : "m" (__m(addr))) - #define __put_user_nocheck(x, ptr, size) \ ({ \ __label__ __pu_label; \ @@ -480,29 +394,6 @@ struct __large_struct { unsigned long buf[100]; }; retval = __put_user_failed(x, addr, itype, rtype, ltype, errret); \ } while (0) -#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ - asm volatile("1: mov"itype" %"rtype"0,%1\n" \ - "2:\n" \ - _ASM_EXTABLE_EX(1b, 2b) \ - : : ltype(x), "m" (__m(addr))) - -/* - * uaccess_try and catch - */ -#define uaccess_try do { \ - current->thread.uaccess_err = 0; \ - __uaccess_begin(); \ - barrier(); - -#define uaccess_try_nospec do { \ - current->thread.uaccess_err = 0; \ - __uaccess_begin_nospec(); \ - -#define uaccess_catch(err) \ - __uaccess_end(); \ - (err) |= (current->thread.uaccess_err ? -EFAULT : 0); \ -} while (0) - /** * __get_user - Get a simple variable from user space, with less checking. * @x: Variable to store result. @@ -552,28 +443,6 @@ struct __large_struct { unsigned long buf[100]; }; #define __put_user(x, ptr) \ __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) -/* - * {get|put}_user_try and catch - * - * get_user_try { - * get_user_ex(...); - * } get_user_catch(err) - */ -#define get_user_try uaccess_try_nospec -#define get_user_catch(err) uaccess_catch(err) - -#define get_user_ex(x, ptr) do { \ - unsigned long __gue_val; \ - __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \ - (x) = (__force __typeof__(*(ptr)))__gue_val; \ -} while (0) - -#define put_user_try uaccess_try -#define put_user_catch(err) uaccess_catch(err) - -#define put_user_ex(x, ptr) \ - __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - extern unsigned long copy_from_user_nmi(void *to, const void __user *from, unsigned long n); extern __must_check long @@ -584,99 +453,6 @@ extern __must_check long strnlen_user(const char __user *str, long n); unsigned long __must_check clear_user(void __user *mem, unsigned long len); unsigned long __must_check __clear_user(void __user *mem, unsigned long len); -extern void __cmpxchg_wrong_size(void) - __compiletime_error("Bad argument size for cmpxchg"); - -#define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \ -({ \ - int __ret = 0; \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __uaccess_begin_nospec(); \ - switch (size) { \ - case 1: \ - { \ - asm volatile("\n" \ - "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ - "2:\n" \ - "\t.section .fixup, \"ax\"\n" \ - "3:\tmov %3, %0\n" \ - "\tjmp 2b\n" \ - "\t.previous\n" \ - _ASM_EXTABLE_UA(1b, 3b) \ - : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ - : "i" (-EFAULT), "q" (__new), "1" (__old) \ - : "memory" \ - ); \ - break; \ - } \ - case 2: \ - { \ - asm volatile("\n" \ - "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ - "2:\n" \ - "\t.section .fixup, \"ax\"\n" \ - "3:\tmov %3, %0\n" \ - "\tjmp 2b\n" \ - "\t.previous\n" \ - _ASM_EXTABLE_UA(1b, 3b) \ - : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ - : "i" (-EFAULT), "r" (__new), "1" (__old) \ - : "memory" \ - ); \ - break; \ - } \ - case 4: \ - { \ - asm volatile("\n" \ - "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ - "2:\n" \ - "\t.section .fixup, \"ax\"\n" \ - "3:\tmov %3, %0\n" \ - "\tjmp 2b\n" \ - "\t.previous\n" \ - _ASM_EXTABLE_UA(1b, 3b) \ - : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ - : "i" (-EFAULT), "r" (__new), "1" (__old) \ - : "memory" \ - ); \ - break; \ - } \ - case 8: \ - { \ - if (!IS_ENABLED(CONFIG_X86_64)) \ - __cmpxchg_wrong_size(); \ - \ - asm volatile("\n" \ - "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ - "2:\n" \ - "\t.section .fixup, \"ax\"\n" \ - "3:\tmov %3, %0\n" \ - "\tjmp 2b\n" \ - "\t.previous\n" \ - _ASM_EXTABLE_UA(1b, 3b) \ - : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ - : "i" (-EFAULT), "r" (__new), "1" (__old) \ - : "memory" \ - ); \ - break; \ - } \ - default: \ - __cmpxchg_wrong_size(); \ - } \ - __uaccess_end(); \ - *(uval) = __old; \ - __ret; \ -}) - -#define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \ -({ \ - access_ok((ptr), sizeof(*(ptr))) ? \ - __user_atomic_cmpxchg_inatomic((uval), (ptr), \ - (old), (new), sizeof(*(ptr))) : \ - -EFAULT; \ -}) - /* * movsl can be slow when source and dest are not both 8-byte aligned */ @@ -695,15 +471,6 @@ extern struct movsl_mask { #endif /* - * We rely on the nested NMI work to allow atomic faults from the NMI path; the - * nested NMI paths are careful to preserve CR2. - * - * Caller must use pagefault_enable/disable, or run in interrupt context, - * and also do a uaccess_ok() check - */ -#define __copy_from_user_nmi __copy_from_user_inatomic - -/* * The "unsafe" user accesses aren't really "unsafe", but the naming * is a big fat warning: you have to not only do the access_ok() * checking before using them, but you have to surround them with the diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index ba2dc1930630..388a40660c7b 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h @@ -23,33 +23,6 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) static __always_inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { - if (__builtin_constant_p(n)) { - unsigned long ret; - - switch (n) { - case 1: - ret = 0; - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u8 *)to, from, ret, - "b", "b", "=q", 1); - __uaccess_end(); - return ret; - case 2: - ret = 0; - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u16 *)to, from, ret, - "w", "w", "=r", 2); - __uaccess_end(); - return ret; - case 4: - ret = 0; - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u32 *)to, from, ret, - "l", "k", "=r", 4); - __uaccess_end(); - return ret; - } - } return __copy_user_ll(to, (__force const void *)from, n); } diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 5cd1caa8bc65..bc10e3dc64fe 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -65,117 +65,13 @@ copy_to_user_mcsafe(void *to, const void *from, unsigned len) static __always_inline __must_check unsigned long raw_copy_from_user(void *dst, const void __user *src, unsigned long size) { - int ret = 0; - - if (!__builtin_constant_p(size)) - return copy_user_generic(dst, (__force void *)src, size); - switch (size) { - case 1: - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src, - ret, "b", "b", "=q", 1); - __uaccess_end(); - return ret; - case 2: - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src, - ret, "w", "w", "=r", 2); - __uaccess_end(); - return ret; - case 4: - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, - ret, "l", "k", "=r", 4); - __uaccess_end(); - return ret; - case 8: - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, - ret, "q", "", "=r", 8); - __uaccess_end(); - return ret; - case 10: - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, - ret, "q", "", "=r", 10); - if (likely(!ret)) - __get_user_asm_nozero(*(u16 *)(8 + (char *)dst), - (u16 __user *)(8 + (char __user *)src), - ret, "w", "w", "=r", 2); - __uaccess_end(); - return ret; - case 16: - __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, - ret, "q", "", "=r", 16); - if (likely(!ret)) - __get_user_asm_nozero(*(u64 *)(8 + (char *)dst), - (u64 __user *)(8 + (char __user *)src), - ret, "q", "", "=r", 8); - __uaccess_end(); - return ret; - default: - return copy_user_generic(dst, (__force void *)src, size); - } + return copy_user_generic(dst, (__force void *)src, size); } static __always_inline __must_check unsigned long raw_copy_to_user(void __user *dst, const void *src, unsigned long size) { - int ret = 0; - - if (!__builtin_constant_p(size)) - return copy_user_generic((__force void *)dst, src, size); - switch (size) { - case 1: - __uaccess_begin(); - __put_user_asm(*(u8 *)src, (u8 __user *)dst, - ret, "b", "b", "iq", 1); - __uaccess_end(); - return ret; - case 2: - __uaccess_begin(); - __put_user_asm(*(u16 *)src, (u16 __user *)dst, - ret, "w", "w", "ir", 2); - __uaccess_end(); - return ret; - case 4: - __uaccess_begin(); - __put_user_asm(*(u32 *)src, (u32 __user *)dst, - ret, "l", "k", "ir", 4); - __uaccess_end(); - return ret; - case 8: - __uaccess_begin(); - __put_user_asm(*(u64 *)src, (u64 __user *)dst, - ret, "q", "", "er", 8); - __uaccess_end(); - return ret; - case 10: - __uaccess_begin(); - __put_user_asm(*(u64 *)src, (u64 __user *)dst, - ret, "q", "", "er", 10); - if (likely(!ret)) { - asm("":::"memory"); - __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, - ret, "w", "w", "ir", 2); - } - __uaccess_end(); - return ret; - case 16: - __uaccess_begin(); - __put_user_asm(*(u64 *)src, (u64 __user *)dst, - ret, "q", "", "er", 16); - if (likely(!ret)) { - asm("":::"memory"); - __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, - ret, "q", "", "er", 8); - } - __uaccess_end(); - return ret; - default: - return copy_user_generic((__force void *)dst, src, size); - } + return copy_user_generic((__force void *)dst, src, size); } static __always_inline __must_check diff --git a/arch/x86/include/asm/unistd.h b/arch/x86/include/asm/unistd.h index a7dd080749ce..c1c3d31b15c0 100644 --- a/arch/x86/include/asm/unistd.h +++ b/arch/x86/include/asm/unistd.h @@ -13,10 +13,13 @@ # define __ARCH_WANT_SYS_OLD_MMAP # define __ARCH_WANT_SYS_OLD_SELECT +# define __NR_ia32_syscall_max __NR_syscall_max + # else # include <asm/unistd_64.h> # include <asm/unistd_64_x32.h> +# include <asm/unistd_32_ia32.h> # define __ARCH_WANT_SYS_TIME # define __ARCH_WANT_SYS_UTIME # define __ARCH_WANT_COMPAT_SYS_PREADV64 @@ -26,6 +29,10 @@ # endif +# define NR_syscalls (__NR_syscall_max + 1) +# define X32_NR_syscalls (__NR_x32_syscall_max + 1) +# define IA32_NR_syscalls (__NR_ia32_syscall_max + 1) + # define __ARCH_WANT_NEW_STAT # define __ARCH_WANT_OLD_READDIR # define __ARCH_WANT_OLD_STAT diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h index 7803114aa140..13687bf0e0a9 100644 --- a/arch/x86/include/asm/uv/uv_bau.h +++ b/arch/x86/include/asm/uv/uv_bau.h @@ -858,4 +858,6 @@ static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u) return 1; } +void uv_bau_message_interrupt(struct pt_regs *regs); + #endif /* _ASM_X86_UV_UV_BAU_H */ diff --git a/arch/x86/include/asm/vdso/clocksource.h b/arch/x86/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..119ac8612d89 --- /dev/null +++ b/arch/x86/include/asm/vdso/clocksource.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_CLOCKSOURCE_H +#define __ASM_VDSO_CLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_TSC, \ + VDSO_CLOCKMODE_PVCLOCK, \ + VDSO_CLOCKMODE_HVCLOCK + +#endif /* __ASM_VDSO_CLOCKSOURCE_H */ diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index 6ee1f7dba34b..9a6dc9b4ec99 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -243,7 +243,7 @@ static u64 vread_hvclock(void) static inline u64 __arch_get_hw_counter(s32 clock_mode) { - if (clock_mode == VCLOCK_TSC) + if (likely(clock_mode == VDSO_CLOCKMODE_TSC)) return (u64)rdtsc_ordered(); /* * For any memory-mapped vclock type, we need to make sure that gcc @@ -252,13 +252,13 @@ static inline u64 __arch_get_hw_counter(s32 clock_mode) * question isn't enabled, which will segfault. Hence the barriers. */ #ifdef CONFIG_PARAVIRT_CLOCK - if (clock_mode == VCLOCK_PVCLOCK) { + if (clock_mode == VDSO_CLOCKMODE_PVCLOCK) { barrier(); return vread_pvclock(); } #endif #ifdef CONFIG_HYPERV_TIMER - if (clock_mode == VCLOCK_HVCLOCK) { + if (clock_mode == VDSO_CLOCKMODE_HVCLOCK) { barrier(); return vread_hvclock(); } diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h new file mode 100644 index 000000000000..57b1a7034c64 --- /dev/null +++ b/arch/x86/include/asm/vdso/processor.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ +static __always_inline void rep_nop(void) +{ + asm volatile("rep; nop" ::: "memory"); +} + +static __always_inline void cpu_relax(void) +{ + rep_nop(); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/x86/include/asm/vdso/vsyscall.h b/arch/x86/include/asm/vdso/vsyscall.h index 0026ab2123ce..be199a9b2676 100644 --- a/arch/x86/include/asm/vdso/vsyscall.h +++ b/arch/x86/include/asm/vdso/vsyscall.h @@ -10,8 +10,6 @@ #include <asm/vgtod.h> #include <asm/vvar.h> -int vclocks_used __read_mostly; - DEFINE_VVAR(struct vdso_data, _vdso_data); /* * Update the vDSO data page to keep in sync with kernel timekeeping. @@ -23,19 +21,6 @@ struct vdso_data *__x86_get_k_vdso_data(void) } #define __arch_get_k_vdso_data __x86_get_k_vdso_data -static __always_inline -int __x86_get_clock_mode(struct timekeeper *tk) -{ - int vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; - - /* Mark the new vclock used. */ - BUILD_BUG_ON(VCLOCK_MAX >= 32); - WRITE_ONCE(vclocks_used, READ_ONCE(vclocks_used) | (1 << vclock_mode)); - - return vclock_mode; -} -#define __arch_get_clock_mode __x86_get_clock_mode - /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h index a2638c6124ed..7aa38b2ad8a9 100644 --- a/arch/x86/include/asm/vgtod.h +++ b/arch/x86/include/asm/vgtod.h @@ -2,6 +2,11 @@ #ifndef _ASM_X86_VGTOD_H #define _ASM_X86_VGTOD_H +/* + * This check is required to prevent ARCH=um to include + * unwanted headers. + */ +#ifdef CONFIG_GENERIC_GETTIMEOFDAY #include <linux/compiler.h> #include <asm/clocksource.h> #include <vdso/datapage.h> @@ -14,11 +19,6 @@ typedef u64 gtod_long_t; #else typedef unsigned long gtod_long_t; #endif - -extern int vclocks_used; -static inline bool vclock_was_used(int vclock) -{ - return READ_ONCE(vclocks_used) & (1 << vclock); -} +#endif /* CONFIG_GENERIC_GETTIMEOFDAY */ #endif /* _ASM_X86_VGTOD_H */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 9b294c13809a..bb5abfef0256 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -28,7 +28,6 @@ KASAN_SANITIZE_dumpstack_$(BITS).o := n KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_paravirt.o := n -OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y OBJECT_FILES_NON_STANDARD_test_nx.o := y OBJECT_FILES_NON_STANDARD_paravirt_patch.o := y @@ -53,6 +52,8 @@ obj-y += setup.o x86_init.o i8259.o irqinit.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-y += probe_roms.o +obj-$(CONFIG_X86_32) += sys_ia32.o +obj-$(CONFIG_IA32_EMULATION) += sys_ia32.o obj-$(CONFIG_X86_64) += sys_x86_64.o obj-$(CONFIG_X86_ESPFIX64) += espfix_64.o obj-$(CONFIG_SYSFS) += ksysfs.o diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 04205ce127a1..1ae5439a9a85 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -45,6 +45,7 @@ EXPORT_SYMBOL(acpi_disabled); #define PREFIX "ACPI: " int acpi_noirq; /* skip ACPI IRQ initialization */ +int acpi_nobgrt; /* skip ACPI BGRT */ int acpi_pci_disabled; /* skip ACPI PCI scan and IRQ initialization */ EXPORT_SYMBOL(acpi_pci_disabled); @@ -1619,7 +1620,7 @@ int __init acpi_boot_init(void) acpi_process_madt(); acpi_table_parse(ACPI_SIG_HPET, acpi_parse_hpet); - if (IS_ENABLED(CONFIG_ACPI_BGRT)) + if (IS_ENABLED(CONFIG_ACPI_BGRT) && !acpi_nobgrt) acpi_table_parse(ACPI_SIG_BGRT, acpi_parse_bgrt); if (!acpi_noirq) @@ -1671,6 +1672,13 @@ static int __init parse_acpi(char *arg) } early_param("acpi", parse_acpi); +static int __init parse_acpi_bgrt(char *arg) +{ + acpi_nobgrt = true; + return 0; +} +early_param("bgrt_disable", parse_acpi_bgrt); + /* FIXME: Using pci= for an ACPI parameter is a travesty. */ static int __init parse_pci(char *arg) { @@ -1740,7 +1748,7 @@ int __acpi_acquire_global_lock(unsigned int *lock) new = (((old & ~0x3) + 2) + ((old >> 1) & 0x1)); val = cmpxchg(lock, old, new); } while (unlikely (val != old)); - return (new < 3) ? -1 : 0; + return ((new & 0x3) < 3) ? -1 : 0; } int __acpi_release_global_lock(unsigned int *lock) diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 26b7256f590f..ed3b04483972 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -43,7 +43,7 @@ unsigned long acpi_get_wakeup_address(void) * * Wrapper around acpi_enter_sleep_state() to be called by assmebly. */ -acpi_status asmlinkage __visible x86_acpi_enter_sleep_state(u8 state) +asmlinkage acpi_status __visible x86_acpi_enter_sleep_state(u8 state) { return acpi_enter_sleep_state(state); } diff --git a/arch/x86/kernel/acpi/sleep.h b/arch/x86/kernel/acpi/sleep.h index d06c2079b6c1..171a40c74db6 100644 --- a/arch/x86/kernel/acpi/sleep.h +++ b/arch/x86/kernel/acpi/sleep.h @@ -19,4 +19,4 @@ extern void do_suspend_lowlevel(void); extern int x86_acpi_suspend_lowlevel(void); -acpi_status asmlinkage x86_acpi_enter_sleep_state(u8 state); +asmlinkage acpi_status x86_acpi_enter_sleep_state(u8 state); diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 15ac0d5f4b40..7867dfb3963e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1167,8 +1167,8 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries atomic_cond_read_acquire(&desc.refs, !VAL); } -void text_poke_loc_init(struct text_poke_loc *tp, void *addr, - const void *opcode, size_t len, const void *emulate) +static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, + const void *opcode, size_t len, const void *emulate) { struct insn insn; diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 69aed0ebbdfc..b6b3297851f3 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -36,10 +36,9 @@ static const struct pci_device_id amd_root_ids[] = { {} }; - #define PCI_DEVICE_ID_AMD_CNB17H_F4 0x1704 -const struct pci_device_id amd_nb_misc_ids[] = { +static const struct pci_device_id amd_nb_misc_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MISC) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_MISC) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F3) }, @@ -56,7 +55,6 @@ const struct pci_device_id amd_nb_misc_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) }, {} }; -EXPORT_SYMBOL_GPL(amd_nb_misc_ids); static const struct pci_device_id amd_nb_link_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F4) }, diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 5f973fed3c9f..81b9c63dae1b 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -546,12 +546,6 @@ static struct clock_event_device lapic_clockevent = { }; static DEFINE_PER_CPU(struct clock_event_device, lapic_events); -#define DEADLINE_MODEL_MATCH_FUNC(model, func) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&func } - -#define DEADLINE_MODEL_MATCH_REV(model, rev) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)rev } - static u32 hsx_deadline_rev(void) { switch (boot_cpu_data.x86_stepping) { @@ -588,23 +582,23 @@ static u32 skx_deadline_rev(void) } static const struct x86_cpu_id deadline_match[] = { - DEADLINE_MODEL_MATCH_FUNC( INTEL_FAM6_HASWELL_X, hsx_deadline_rev), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_BROADWELL_X, 0x0b000020), - DEADLINE_MODEL_MATCH_FUNC( INTEL_FAM6_BROADWELL_D, bdx_deadline_rev), - DEADLINE_MODEL_MATCH_FUNC( INTEL_FAM6_SKYLAKE_X, skx_deadline_rev), + X86_MATCH_INTEL_FAM6_MODEL( HASWELL_X, &hsx_deadline_rev), + X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X, 0x0b000020), + X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_D, &bdx_deadline_rev), + X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_X, &skx_deadline_rev), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_HASWELL, 0x22), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_HASWELL_L, 0x20), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_HASWELL_G, 0x17), + X86_MATCH_INTEL_FAM6_MODEL( HASWELL, 0x22), + X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L, 0x20), + X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G, 0x17), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_BROADWELL, 0x25), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_BROADWELL_G, 0x17), + X86_MATCH_INTEL_FAM6_MODEL( BROADWELL, 0x25), + X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G, 0x17), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_SKYLAKE_L, 0xb2), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_SKYLAKE, 0xb2), + X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L, 0xb2), + X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE, 0xb2), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_KABYLAKE_L, 0x52), - DEADLINE_MODEL_MATCH_REV ( INTEL_FAM6_KABYLAKE, 0x52), + X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L, 0x52), + X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE, 0x52), {}, }; diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 48293d15f1e1..67768e54438b 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -557,6 +557,12 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, irqd->hwirq = virq + i; irqd_set_single_target(irqd); /* + * Prevent that any of these interrupts is invoked in + * non interrupt context via e.g. generic_handle_irq() + * as that can corrupt the affinity move state. + */ + irqd_set_handle_enforce_irqctx(irqd); + /* * Legacy vectors are already assigned when the IOAPIC * takes them over. They stay on the same vector. This is * required for check_timer() to work correctly as it might diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 5c7ee3df4d0b..3ca07ad552ae 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -88,7 +88,6 @@ static void __used common(void) OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment); OFFSET(BP_init_size, boot_params, hdr.init_size); OFFSET(BP_pref_address, boot_params, hdr.pref_address); - OFFSET(BP_code32_start, boot_params, hdr.code32_start); BLANK(); DEFINE(PTREGS_SIZE, sizeof(struct pt_regs)); diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c index 82826f2275cc..6e043f295a60 100644 --- a/arch/x86/kernel/asm-offsets_32.c +++ b/arch/x86/kernel/asm-offsets_32.c @@ -3,12 +3,9 @@ # error "Please do not build this file directly, build asm-offsets.c instead" #endif -#include <asm/ucontext.h> +#include <linux/efi.h> -#define __SYSCALL_I386(nr, sym, qual) [nr] = 1, -static char syscalls[] = { -#include <asm/syscalls_32.h> -}; +#include <asm/ucontext.h> /* workaround for a warning with -Wmissing-prototypes */ void foo(void); @@ -62,6 +59,5 @@ void foo(void) #endif BLANK(); - DEFINE(__NR_syscall_max, sizeof(syscalls) - 1); - DEFINE(NR_syscalls, sizeof(syscalls)); + DEFINE(EFI_svam, offsetof(efi_runtime_services_t, set_virtual_address_map)); } diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index 24d2fde30d00..c2a47016f243 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -5,30 +5,6 @@ #include <asm/ia32.h> -#define __SYSCALL_64(nr, sym, qual) [nr] = 1, -#define __SYSCALL_X32(nr, sym, qual) -static char syscalls_64[] = { -#include <asm/syscalls_64.h> -}; -#undef __SYSCALL_64 -#undef __SYSCALL_X32 - -#ifdef CONFIG_X86_X32_ABI -#define __SYSCALL_64(nr, sym, qual) -#define __SYSCALL_X32(nr, sym, qual) [nr] = 1, -static char syscalls_x32[] = { -#include <asm/syscalls_64.h> -}; -#undef __SYSCALL_64 -#undef __SYSCALL_X32 -#endif - -#define __SYSCALL_I386(nr, sym, qual) [nr] = 1, -static char syscalls_ia32[] = { -#include <asm/syscalls_32.h> -}; -#undef __SYSCALL_I386 - #if defined(CONFIG_KVM_GUEST) && defined(CONFIG_PARAVIRT_SPINLOCKS) #include <asm/kvm_para.h> #endif @@ -90,17 +66,5 @@ int main(void) DEFINE(stack_canary_offset, offsetof(struct fixed_percpu_data, stack_canary)); BLANK(); #endif - - DEFINE(__NR_syscall_max, sizeof(syscalls_64) - 1); - DEFINE(NR_syscalls, sizeof(syscalls_64)); - -#ifdef CONFIG_X86_X32_ABI - DEFINE(__NR_syscall_x32_max, sizeof(syscalls_x32) - 1); - DEFINE(X32_NR_syscalls, sizeof(syscalls_x32)); -#endif - - DEFINE(__NR_syscall_compat_max, sizeof(syscalls_ia32) - 1); - DEFINE(IA32_NR_syscalls, sizeof(syscalls_ia32)); - return 0; } diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 1f875fbe1384..547ad7bbf0e0 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -394,6 +394,35 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c) per_cpu(cpu_llc_id, cpu) = c->phys_proc_id; } +static void amd_detect_ppin(struct cpuinfo_x86 *c) +{ + unsigned long long val; + + if (!cpu_has(c, X86_FEATURE_AMD_PPIN)) + return; + + /* When PPIN is defined in CPUID, still need to check PPIN_CTL MSR */ + if (rdmsrl_safe(MSR_AMD_PPIN_CTL, &val)) + goto clear_ppin; + + /* PPIN is locked in disabled mode, clear feature bit */ + if ((val & 3UL) == 1UL) + goto clear_ppin; + + /* If PPIN is disabled, try to enable it */ + if (!(val & 2UL)) { + wrmsrl_safe(MSR_AMD_PPIN_CTL, val | 2UL); + rdmsrl_safe(MSR_AMD_PPIN_CTL, &val); + } + + /* If PPIN_EN bit is 1, return from here; otherwise fall through */ + if (val & 2UL) + return; + +clear_ppin: + clear_cpu_cap(c, X86_FEATURE_AMD_PPIN); +} + u16 amd_get_nb_id(int cpu) { return per_cpu(cpu_llc_id, cpu); @@ -926,7 +955,8 @@ static void init_amd(struct cpuinfo_x86 *c) case 0x12: init_amd_ln(c); break; case 0x15: init_amd_bd(c); break; case 0x16: init_amd_jg(c); break; - case 0x17: init_amd_zn(c); break; + case 0x17: fallthrough; + case 0x19: init_amd_zn(c); break; } /* @@ -941,6 +971,7 @@ static void init_amd(struct cpuinfo_x86 *c) amd_detect_cmp(c); amd_get_topology(c); srat_detect_node(c); + amd_detect_ppin(c); init_amd_cacheinfo(c); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 4cdb123ff66a..bed0cb83fe24 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1008,8 +1008,8 @@ static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c) #define NO_ITLB_MULTIHIT BIT(7) #define NO_SPECTRE_V2 BIT(8) -#define VULNWL(_vendor, _family, _model, _whitelist) \ - { X86_VENDOR_##_vendor, _family, _model, X86_FEATURE_ANY, _whitelist } +#define VULNWL(vendor, family, model, whitelist) \ + X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist) #define VULNWL_INTEL(model, whitelist) \ VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist) @@ -1224,6 +1224,8 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c) cpu_set_bug_bits(c); + cpu_set_core_cap_bits(c); + fpu__init_system(c); #ifdef CONFIG_X86_32 diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index 0268185bef94..29a3bedabd06 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -5,6 +5,7 @@ #include <asm/msr-index.h> #include <asm/processor.h> #include <asm/vmx.h> +#include "cpu.h" #undef pr_fmt #define pr_fmt(fmt) "x86/cpu: " fmt diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index be82cd5841c3..9a26e972cdea 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -19,6 +19,8 @@ #include <asm/microcode_intel.h> #include <asm/hwcap2.h> #include <asm/elf.h> +#include <asm/cpu_device_id.h> +#include <asm/cmdline.h> #ifdef CONFIG_X86_64 #include <linux/topology.h> @@ -31,6 +33,20 @@ #include <asm/apic.h> #endif +enum split_lock_detect_state { + sld_off = 0, + sld_warn, + sld_fatal, +}; + +/* + * Default to sld_off because most systems do not support split lock detection + * split_lock_setup() will switch this to sld_warn on systems that support + * split lock detect, unless there is a command line override. + */ +static enum split_lock_detect_state sld_state __ro_after_init = sld_off; +static u64 msr_test_ctrl_cache __ro_after_init; + /* * Processors which have self-snooping capability can handle conflicting * memory type across CPUs by snooping its own cache. However, there exists @@ -570,6 +586,8 @@ static void init_intel_misc_features(struct cpuinfo_x86 *c) wrmsrl(MSR_MISC_FEATURES_ENABLES, msr); } +static void split_lock_init(void); + static void init_intel(struct cpuinfo_x86 *c) { early_init_intel(c); @@ -684,6 +702,8 @@ static void init_intel(struct cpuinfo_x86 *c) tsx_enable(); if (tsx_ctrl_state == TSX_CTRL_DISABLE) tsx_disable(); + + split_lock_init(); } #ifdef CONFIG_X86_32 @@ -945,3 +965,166 @@ static const struct cpu_dev intel_cpu_dev = { }; cpu_dev_register(intel_cpu_dev); + +#undef pr_fmt +#define pr_fmt(fmt) "x86/split lock detection: " fmt + +static const struct { + const char *option; + enum split_lock_detect_state state; +} sld_options[] __initconst = { + { "off", sld_off }, + { "warn", sld_warn }, + { "fatal", sld_fatal }, +}; + +static inline bool match_option(const char *arg, int arglen, const char *opt) +{ + int len = strlen(opt); + + return len == arglen && !strncmp(arg, opt, len); +} + +static bool split_lock_verify_msr(bool on) +{ + u64 ctrl, tmp; + + if (rdmsrl_safe(MSR_TEST_CTRL, &ctrl)) + return false; + if (on) + ctrl |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT; + else + ctrl &= ~MSR_TEST_CTRL_SPLIT_LOCK_DETECT; + if (wrmsrl_safe(MSR_TEST_CTRL, ctrl)) + return false; + rdmsrl(MSR_TEST_CTRL, tmp); + return ctrl == tmp; +} + +static void __init split_lock_setup(void) +{ + enum split_lock_detect_state state = sld_warn; + char arg[20]; + int i, ret; + + if (!split_lock_verify_msr(false)) { + pr_info("MSR access failed: Disabled\n"); + return; + } + + ret = cmdline_find_option(boot_command_line, "split_lock_detect", + arg, sizeof(arg)); + if (ret >= 0) { + for (i = 0; i < ARRAY_SIZE(sld_options); i++) { + if (match_option(arg, ret, sld_options[i].option)) { + state = sld_options[i].state; + break; + } + } + } + + switch (state) { + case sld_off: + pr_info("disabled\n"); + return; + case sld_warn: + pr_info("warning about user-space split_locks\n"); + break; + case sld_fatal: + pr_info("sending SIGBUS on user-space split_locks\n"); + break; + } + + rdmsrl(MSR_TEST_CTRL, msr_test_ctrl_cache); + + if (!split_lock_verify_msr(true)) { + pr_info("MSR access failed: Disabled\n"); + return; + } + + sld_state = state; + setup_force_cpu_cap(X86_FEATURE_SPLIT_LOCK_DETECT); +} + +/* + * MSR_TEST_CTRL is per core, but we treat it like a per CPU MSR. Locking + * is not implemented as one thread could undo the setting of the other + * thread immediately after dropping the lock anyway. + */ +static void sld_update_msr(bool on) +{ + u64 test_ctrl_val = msr_test_ctrl_cache; + + if (on) + test_ctrl_val |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT; + + wrmsrl(MSR_TEST_CTRL, test_ctrl_val); +} + +static void split_lock_init(void) +{ + split_lock_verify_msr(sld_state != sld_off); +} + +bool handle_user_split_lock(struct pt_regs *regs, long error_code) +{ + if ((regs->flags & X86_EFLAGS_AC) || sld_state == sld_fatal) + return false; + + pr_warn_ratelimited("#AC: %s/%d took a split_lock trap at address: 0x%lx\n", + current->comm, current->pid, regs->ip); + + /* + * Disable the split lock detection for this task so it can make + * progress and set TIF_SLD so the detection is re-enabled via + * switch_to_sld() when the task is scheduled out. + */ + sld_update_msr(false); + set_tsk_thread_flag(current, TIF_SLD); + return true; +} + +/* + * This function is called only when switching between tasks with + * different split-lock detection modes. It sets the MSR for the + * mode of the new task. This is right most of the time, but since + * the MSR is shared by hyperthreads on a physical core there can + * be glitches when the two threads need different modes. + */ +void switch_to_sld(unsigned long tifn) +{ + sld_update_msr(!(tifn & _TIF_SLD)); +} + +#define SPLIT_LOCK_CPU(model) {X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY} + +/* + * The following processors have the split lock detection feature. But + * since they don't have the IA32_CORE_CAPABILITIES MSR, the feature cannot + * be enumerated. Enable it by family and model matching on these + * processors. + */ +static const struct x86_cpu_id split_lock_cpu_ids[] __initconst = { + SPLIT_LOCK_CPU(INTEL_FAM6_ICELAKE_X), + SPLIT_LOCK_CPU(INTEL_FAM6_ICELAKE_L), + {} +}; + +void __init cpu_set_core_cap_bits(struct cpuinfo_x86 *c) +{ + u64 ia32_core_caps = 0; + + if (c->x86_vendor != X86_VENDOR_INTEL) + return; + if (cpu_has(c, X86_FEATURE_CORE_CAPABILITIES)) { + /* Enumerate features reported in IA32_CORE_CAPABILITIES MSR. */ + rdmsrl(MSR_IA32_CORE_CAPS, ia32_core_caps); + } else if (!boot_cpu_has(X86_FEATURE_HYPERVISOR)) { + /* Enumerate split lock detection by family and model. */ + if (x86_match_cpu(split_lock_cpu_ids)) + ia32_core_caps |= MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT; + } + + if (ia32_core_caps & MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT) + split_lock_setup(); +} diff --git a/arch/x86/kernel/cpu/match.c b/arch/x86/kernel/cpu/match.c index 6dd78d8235e4..d3482eb43ff3 100644 --- a/arch/x86/kernel/cpu/match.c +++ b/arch/x86/kernel/cpu/match.c @@ -16,12 +16,17 @@ * respective wildcard entries. * * A typical table entry would be to match a specific CPU - * { X86_VENDOR_INTEL, 6, 0x12 } - * or to match a specific CPU feature - * { X86_FEATURE_MATCH(X86_FEATURE_FOOBAR) } + * + * X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_BROADWELL, + * X86_FEATURE_ANY, NULL); * * Fields can be wildcarded with %X86_VENDOR_ANY, %X86_FAMILY_ANY, - * %X86_MODEL_ANY, %X86_FEATURE_ANY or 0 (except for vendor) + * %X86_MODEL_ANY, %X86_FEATURE_ANY (except for vendor) + * + * asm/cpu_device_id.h contains a set of useful macros which are shortcuts + * for various common selections. The above can be shortened to: + * + * X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, NULL); * * Arrays used to match for this should also be declared using * MODULE_DEVICE_TABLE(x86cpu, ...) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 2c4f949611e4..54165f3569e8 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -142,6 +142,8 @@ void mce_setup(struct mce *m) if (this_cpu_has(X86_FEATURE_INTEL_PPIN)) rdmsrl(MSR_PPIN, m->ppin); + else if (this_cpu_has(X86_FEATURE_AMD_PPIN)) + rdmsrl(MSR_AMD_PPIN, m->ppin); m->microcode = boot_cpu_data.microcode; } @@ -1213,8 +1215,14 @@ static void __mc_scan_banks(struct mce *m, struct mce *final, * On Intel systems this is entered on all CPUs in parallel through * MCE broadcast. However some CPUs might be broken beyond repair, * so be always careful when synchronizing with others. + * + * Tracing and kprobes are disabled: if we interrupted a kernel context + * with IF=1, we need to minimize stack usage. There are also recursion + * issues: if the machine check was due to a failure of the memory + * backing the user stack, tracing that reads the user stack will cause + * potentially infinite recursion. */ -void do_machine_check(struct pt_regs *regs, long error_code) +void notrace do_machine_check(struct pt_regs *regs, long error_code) { DECLARE_BITMAP(valid_banks, MAX_NR_BANKS); DECLARE_BITMAP(toclear, MAX_NR_BANKS); @@ -1360,6 +1368,7 @@ out_ist: ist_exit(regs); } EXPORT_SYMBOL_GPL(do_machine_check); +NOKPROBE_SYMBOL(do_machine_check); #ifndef CONFIG_MEMORY_FAILURE int memory_failure(unsigned long pfn, int flags) @@ -1877,6 +1886,8 @@ bool filter_mce(struct mce *m) { if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) return amd_filter_mce(m); + if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) + return intel_filter_mce(m); return false; } @@ -1892,10 +1903,11 @@ static void unexpected_machine_check(struct pt_regs *regs, long error_code) void (*machine_check_vector)(struct pt_regs *, long error_code) = unexpected_machine_check; -dotraplinkage void do_mce(struct pt_regs *regs, long error_code) +dotraplinkage notrace void do_mce(struct pt_regs *regs, long error_code) { machine_check_vector(regs, error_code); } +NOKPROBE_SYMBOL(do_mce); /* * Called for each booted CPU to set up machine checks. diff --git a/arch/x86/kernel/cpu/mce/dev-mcelog.c b/arch/x86/kernel/cpu/mce/dev-mcelog.c index 7c8958dee103..d089567a9ce8 100644 --- a/arch/x86/kernel/cpu/mce/dev-mcelog.c +++ b/arch/x86/kernel/cpu/mce/dev-mcelog.c @@ -29,11 +29,7 @@ static char *mce_helper_argv[2] = { mce_helper, NULL }; * separate MCEs from kernel messages to avoid bogus bug reports. */ -static struct mce_log_buffer mcelog = { - .signature = MCE_LOG_SIGNATURE, - .len = MCE_LOG_LEN, - .recordlen = sizeof(struct mce), -}; +static struct mce_log_buffer *mcelog; static DECLARE_WAIT_QUEUE_HEAD(mce_chrdev_wait); @@ -45,21 +41,21 @@ static int dev_mce_log(struct notifier_block *nb, unsigned long val, mutex_lock(&mce_chrdev_read_mutex); - entry = mcelog.next; + entry = mcelog->next; /* * When the buffer fills up discard new entries. Assume that the * earlier errors are the more interesting ones: */ - if (entry >= MCE_LOG_LEN) { - set_bit(MCE_OVERFLOW, (unsigned long *)&mcelog.flags); + if (entry >= mcelog->len) { + set_bit(MCE_OVERFLOW, (unsigned long *)&mcelog->flags); goto unlock; } - mcelog.next = entry + 1; + mcelog->next = entry + 1; - memcpy(mcelog.entry + entry, mce, sizeof(struct mce)); - mcelog.entry[entry].finished = 1; + memcpy(mcelog->entry + entry, mce, sizeof(struct mce)); + mcelog->entry[entry].finished = 1; /* wake processes polling /dev/mcelog */ wake_up_interruptible(&mce_chrdev_wait); @@ -214,21 +210,21 @@ static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf, /* Only supports full reads right now */ err = -EINVAL; - if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce)) + if (*off != 0 || usize < mcelog->len * sizeof(struct mce)) goto out; - next = mcelog.next; + next = mcelog->next; err = 0; for (i = 0; i < next; i++) { - struct mce *m = &mcelog.entry[i]; + struct mce *m = &mcelog->entry[i]; err |= copy_to_user(buf, m, sizeof(*m)); buf += sizeof(*m); } - memset(mcelog.entry, 0, next * sizeof(struct mce)); - mcelog.next = 0; + memset(mcelog->entry, 0, next * sizeof(struct mce)); + mcelog->next = 0; if (err) err = -EFAULT; @@ -242,7 +238,7 @@ out: static __poll_t mce_chrdev_poll(struct file *file, poll_table *wait) { poll_wait(file, &mce_chrdev_wait, wait); - if (READ_ONCE(mcelog.next)) + if (READ_ONCE(mcelog->next)) return EPOLLIN | EPOLLRDNORM; if (!mce_apei_read_done && apei_check_mce()) return EPOLLIN | EPOLLRDNORM; @@ -261,13 +257,13 @@ static long mce_chrdev_ioctl(struct file *f, unsigned int cmd, case MCE_GET_RECORD_LEN: return put_user(sizeof(struct mce), p); case MCE_GET_LOG_LEN: - return put_user(MCE_LOG_LEN, p); + return put_user(mcelog->len, p); case MCE_GETCLEAR_FLAGS: { unsigned flags; do { - flags = mcelog.flags; - } while (cmpxchg(&mcelog.flags, flags, 0) != flags); + flags = mcelog->flags; + } while (cmpxchg(&mcelog->flags, flags, 0) != flags); return put_user(flags, p); } @@ -339,8 +335,18 @@ static struct miscdevice mce_chrdev_device = { static __init int dev_mcelog_init_device(void) { + int mce_log_len; int err; + mce_log_len = max(MCE_LOG_MIN_LEN, num_online_cpus()); + mcelog = kzalloc(sizeof(*mcelog) + mce_log_len * sizeof(struct mce), GFP_KERNEL); + if (!mcelog) + return -ENOMEM; + + strncpy(mcelog->signature, MCE_LOG_SIGNATURE, sizeof(mcelog->signature)); + mcelog->len = mce_log_len; + mcelog->recordlen = sizeof(struct mce); + /* register character device /dev/mcelog */ err = misc_register(&mce_chrdev_device); if (err) { @@ -350,6 +356,7 @@ static __init int dev_mcelog_init_device(void) else pr_err("Unable to init device /dev/mcelog (rc: %d)\n", err); + kfree(mcelog); return err; } diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index f996ffb887bc..d8f9230d2034 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -521,3 +521,20 @@ void mce_intel_feature_clear(struct cpuinfo_x86 *c) { intel_clear_lmce(); } + +bool intel_filter_mce(struct mce *m) +{ + struct cpuinfo_x86 *c = &boot_cpu_data; + + /* MCE errata HSD131, HSM142, HSW131, BDM48, and HSM142 */ + if ((c->x86 == 6) && + ((c->x86_model == INTEL_FAM6_HASWELL) || + (c->x86_model == INTEL_FAM6_HASWELL_L) || + (c->x86_model == INTEL_FAM6_BROADWELL) || + (c->x86_model == INTEL_FAM6_HASWELL_G)) && + (m->bank == 0) && + ((m->status & 0xa0000000ffffffff) == 0x80000000000f0005)) + return true; + + return false; +} diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h index b785c0d0b590..3b008172ad73 100644 --- a/arch/x86/kernel/cpu/mce/internal.h +++ b/arch/x86/kernel/cpu/mce/internal.h @@ -8,6 +8,9 @@ #include <linux/device.h> #include <asm/mce.h> +/* Pointer to the installed machine check handler for this CPU setup. */ +extern void (*machine_check_vector)(struct pt_regs *, long error_code); + enum severity_level { MCE_NO_SEVERITY, MCE_DEFERRED_SEVERITY, @@ -48,6 +51,7 @@ void cmci_disable_bank(int bank); void intel_init_cmci(void); void intel_init_lmce(void); void intel_clear_lmce(void); +bool intel_filter_mce(struct mce *m); #else # define cmci_intel_adjust_timer mce_adjust_timer_default static inline bool mce_intel_cmci_poll(void) { return false; } @@ -56,6 +60,7 @@ static inline void cmci_disable_bank(int bank) { } static inline void intel_init_cmci(void) { } static inline void intel_init_lmce(void) { } static inline void intel_clear_lmce(void) { } +static inline bool intel_filter_mce(struct mce *m) { return false; }; #endif void mce_timer_kick(unsigned long interval); diff --git a/arch/x86/kernel/cpu/umwait.c b/arch/x86/kernel/cpu/umwait.c index c222f283b456..300e3fd5ade3 100644 --- a/arch/x86/kernel/cpu/umwait.c +++ b/arch/x86/kernel/cpu/umwait.c @@ -4,6 +4,7 @@ #include <linux/cpu.h> #include <asm/msr.h> +#include <asm/mwait.h> #define UMWAIT_C02_ENABLE 0 diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 46d732696c1c..9b6fafa69be9 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -25,6 +25,8 @@ #include <linux/init.h> #include <linux/export.h> #include <linux/clocksource.h> +#include <linux/cpu.h> +#include <linux/reboot.h> #include <asm/div64.h> #include <asm/x86_init.h> #include <asm/hypervisor.h> @@ -47,6 +49,11 @@ #define VMWARE_CMD_GETVCPU_INFO 68 #define VMWARE_CMD_LEGACY_X2APIC 3 #define VMWARE_CMD_VCPU_RESERVED 31 +#define VMWARE_CMD_STEALCLOCK 91 + +#define STEALCLOCK_NOT_AVAILABLE (-1) +#define STEALCLOCK_DISABLED 0 +#define STEALCLOCK_ENABLED 1 #define VMWARE_PORT(cmd, eax, ebx, ecx, edx) \ __asm__("inl (%%dx), %%eax" : \ @@ -86,6 +93,18 @@ } \ } while (0) +struct vmware_steal_time { + union { + uint64_t clock; /* stolen time counter in units of vtsc */ + struct { + /* only for little-endian */ + uint32_t clock_low; + uint32_t clock_high; + }; + }; + uint64_t reserved[7]; +}; + static unsigned long vmware_tsc_khz __ro_after_init; static u8 vmware_hypercall_mode __ro_after_init; @@ -103,15 +122,25 @@ static unsigned long vmware_get_tsc_khz(void) #ifdef CONFIG_PARAVIRT static struct cyc2ns_data vmware_cyc2ns __ro_after_init; -static int vmw_sched_clock __initdata = 1; +static bool vmw_sched_clock __initdata = true; +static DEFINE_PER_CPU_DECRYPTED(struct vmware_steal_time, vmw_steal_time) __aligned(64); +static bool has_steal_clock; +static bool steal_acc __initdata = true; /* steal time accounting */ static __init int setup_vmw_sched_clock(char *s) { - vmw_sched_clock = 0; + vmw_sched_clock = false; return 0; } early_param("no-vmw-sched-clock", setup_vmw_sched_clock); +static __init int parse_no_stealacc(char *arg) +{ + steal_acc = false; + return 0; +} +early_param("no-steal-acc", parse_no_stealacc); + static unsigned long long notrace vmware_sched_clock(void) { unsigned long long ns; @@ -122,7 +151,7 @@ static unsigned long long notrace vmware_sched_clock(void) return ns; } -static void __init vmware_sched_clock_setup(void) +static void __init vmware_cyc2ns_setup(void) { struct cyc2ns_data *d = &vmware_cyc2ns; unsigned long long tsc_now = rdtsc(); @@ -132,17 +161,201 @@ static void __init vmware_sched_clock_setup(void) d->cyc2ns_offset = mul_u64_u32_shr(tsc_now, d->cyc2ns_mul, d->cyc2ns_shift); - pv_ops.time.sched_clock = vmware_sched_clock; - pr_info("using sched offset of %llu ns\n", d->cyc2ns_offset); + pr_info("using clock offset of %llu ns\n", d->cyc2ns_offset); +} + +static int vmware_cmd_stealclock(uint32_t arg1, uint32_t arg2) +{ + uint32_t result, info; + + asm volatile (VMWARE_HYPERCALL : + "=a"(result), + "=c"(info) : + "a"(VMWARE_HYPERVISOR_MAGIC), + "b"(0), + "c"(VMWARE_CMD_STEALCLOCK), + "d"(0), + "S"(arg1), + "D"(arg2) : + "memory"); + return result; +} + +static bool stealclock_enable(phys_addr_t pa) +{ + return vmware_cmd_stealclock(upper_32_bits(pa), + lower_32_bits(pa)) == STEALCLOCK_ENABLED; +} + +static int __stealclock_disable(void) +{ + return vmware_cmd_stealclock(0, 1); +} + +static void stealclock_disable(void) +{ + __stealclock_disable(); +} + +static bool vmware_is_stealclock_available(void) +{ + return __stealclock_disable() != STEALCLOCK_NOT_AVAILABLE; +} + +/** + * vmware_steal_clock() - read the per-cpu steal clock + * @cpu: the cpu number whose steal clock we want to read + * + * The function reads the steal clock if we are on a 64-bit system, otherwise + * reads it in parts, checking that the high part didn't change in the + * meantime. + * + * Return: + * The steal clock reading in ns. + */ +static uint64_t vmware_steal_clock(int cpu) +{ + struct vmware_steal_time *steal = &per_cpu(vmw_steal_time, cpu); + uint64_t clock; + + if (IS_ENABLED(CONFIG_64BIT)) + clock = READ_ONCE(steal->clock); + else { + uint32_t initial_high, low, high; + + do { + initial_high = READ_ONCE(steal->clock_high); + /* Do not reorder initial_high and high readings */ + virt_rmb(); + low = READ_ONCE(steal->clock_low); + /* Keep low reading in between */ + virt_rmb(); + high = READ_ONCE(steal->clock_high); + } while (initial_high != high); + + clock = ((uint64_t)high << 32) | low; + } + + return mul_u64_u32_shr(clock, vmware_cyc2ns.cyc2ns_mul, + vmware_cyc2ns.cyc2ns_shift); +} + +static void vmware_register_steal_time(void) +{ + int cpu = smp_processor_id(); + struct vmware_steal_time *st = &per_cpu(vmw_steal_time, cpu); + + if (!has_steal_clock) + return; + + if (!stealclock_enable(slow_virt_to_phys(st))) { + has_steal_clock = false; + return; + } + + pr_info("vmware-stealtime: cpu %d, pa %llx\n", + cpu, (unsigned long long) slow_virt_to_phys(st)); } +static void vmware_disable_steal_time(void) +{ + if (!has_steal_clock) + return; + + stealclock_disable(); +} + +static void vmware_guest_cpu_init(void) +{ + if (has_steal_clock) + vmware_register_steal_time(); +} + +static void vmware_pv_guest_cpu_reboot(void *unused) +{ + vmware_disable_steal_time(); +} + +static int vmware_pv_reboot_notify(struct notifier_block *nb, + unsigned long code, void *unused) +{ + if (code == SYS_RESTART) + on_each_cpu(vmware_pv_guest_cpu_reboot, NULL, 1); + return NOTIFY_DONE; +} + +static struct notifier_block vmware_pv_reboot_nb = { + .notifier_call = vmware_pv_reboot_notify, +}; + +#ifdef CONFIG_SMP +static void __init vmware_smp_prepare_boot_cpu(void) +{ + vmware_guest_cpu_init(); + native_smp_prepare_boot_cpu(); +} + +static int vmware_cpu_online(unsigned int cpu) +{ + local_irq_disable(); + vmware_guest_cpu_init(); + local_irq_enable(); + return 0; +} + +static int vmware_cpu_down_prepare(unsigned int cpu) +{ + local_irq_disable(); + vmware_disable_steal_time(); + local_irq_enable(); + return 0; +} +#endif + +static __init int activate_jump_labels(void) +{ + if (has_steal_clock) { + static_key_slow_inc(¶virt_steal_enabled); + if (steal_acc) + static_key_slow_inc(¶virt_steal_rq_enabled); + } + + return 0; +} +arch_initcall(activate_jump_labels); + static void __init vmware_paravirt_ops_setup(void) { pv_info.name = "VMware hypervisor"; pv_ops.cpu.io_delay = paravirt_nop; - if (vmware_tsc_khz && vmw_sched_clock) - vmware_sched_clock_setup(); + if (vmware_tsc_khz == 0) + return; + + vmware_cyc2ns_setup(); + + if (vmw_sched_clock) + pv_ops.time.sched_clock = vmware_sched_clock; + + if (vmware_is_stealclock_available()) { + has_steal_clock = true; + pv_ops.time.steal_clock = vmware_steal_clock; + + /* We use reboot notifier only to disable steal clock */ + register_reboot_notifier(&vmware_pv_reboot_nb); + +#ifdef CONFIG_SMP + smp_ops.smp_prepare_boot_cpu = + vmware_smp_prepare_boot_cpu; + if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "x86/vmware:online", + vmware_cpu_online, + vmware_cpu_down_prepare) < 0) + pr_err("vmware_guest: Failed to install cpu hotplug callbacks\n"); +#else + vmware_guest_cpu_init(); +#endif + } } #else #define vmware_paravirt_ops_setup() do {} while (0) @@ -213,7 +426,7 @@ static void __init vmware_platform_setup(void) vmware_set_capabilities(); } -static u8 vmware_select_hypercall(void) +static u8 __init vmware_select_hypercall(void) { int eax, ebx, ecx, edx; diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index a1806598aaa4..32b153d38748 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -120,11 +120,6 @@ static bool xfeature_is_supervisor(int xfeature_nr) return ecx & 1; } -static bool xfeature_is_user(int xfeature_nr) -{ - return !xfeature_is_supervisor(xfeature_nr); -} - /* * When executing XSAVEOPT (or other optimized XSAVE instructions), if * a processor implementation detects that an FPU state component is still @@ -265,21 +260,25 @@ static void __init setup_xstate_features(void) cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); + xstate_sizes[i] = eax; + /* - * If an xfeature is supervisor state, the offset - * in EBX is invalid. We leave it to -1. + * If an xfeature is supervisor state, the offset in EBX is + * invalid, leave it to -1. */ - if (xfeature_is_user(i)) - xstate_offsets[i] = ebx; + if (xfeature_is_supervisor(i)) + continue; + + xstate_offsets[i] = ebx; - xstate_sizes[i] = eax; /* - * In our xstate size checks, we assume that the - * highest-numbered xstate feature has the - * highest offset in the buffer. Ensure it does. + * In our xstate size checks, we assume that the highest-numbered + * xstate feature has the highest offset in the buffer. Ensure + * it does. */ WARN_ONCE(last_good_offset > xstate_offsets[i], - "x86/fpu: misordered xstate at %d\n", last_good_offset); + "x86/fpu: misordered xstate at %d\n", last_good_offset); + last_good_offset = xstate_offsets[i]; } } @@ -326,6 +325,13 @@ static int xfeature_is_aligned(int xfeature_nr) u32 eax, ebx, ecx, edx; CHECK_XFEATURE(xfeature_nr); + + if (!xfeature_enabled(xfeature_nr)) { + WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n", + xfeature_nr); + return 0; + } + cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); /* * The value returned by ECX[1] indicates the alignment @@ -338,11 +344,11 @@ static int xfeature_is_aligned(int xfeature_nr) /* * This function sets up offsets and sizes of all extended states in * xsave area. This supports both standard format and compacted format - * of the xsave aread. + * of the xsave area. */ -static void __init setup_xstate_comp(void) +static void __init setup_xstate_comp_offsets(void) { - unsigned int xstate_comp_sizes[XFEATURE_MAX]; + unsigned int next_offset; int i; /* @@ -356,31 +362,23 @@ static void __init setup_xstate_comp(void) if (!boot_cpu_has(X86_FEATURE_XSAVES)) { for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (xfeature_enabled(i)) { + if (xfeature_enabled(i)) xstate_comp_offsets[i] = xstate_offsets[i]; - xstate_comp_sizes[i] = xstate_sizes[i]; - } } return; } - xstate_comp_offsets[FIRST_EXTENDED_XFEATURE] = - FXSAVE_SIZE + XSAVE_HDR_SIZE; + next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (xfeature_enabled(i)) - xstate_comp_sizes[i] = xstate_sizes[i]; - else - xstate_comp_sizes[i] = 0; + if (!xfeature_enabled(i)) + continue; - if (i > FIRST_EXTENDED_XFEATURE) { - xstate_comp_offsets[i] = xstate_comp_offsets[i-1] - + xstate_comp_sizes[i-1]; + if (xfeature_is_aligned(i)) + next_offset = ALIGN(next_offset, 64); - if (xfeature_is_aligned(i)) - xstate_comp_offsets[i] = - ALIGN(xstate_comp_offsets[i], 64); - } + xstate_comp_offsets[i] = next_offset; + next_offset += xstate_sizes[i]; } } @@ -774,7 +772,7 @@ void __init fpu__init_system_xstate(void) fpu__init_prepare_fx_sw_frame(); setup_init_fpu_buf(); - setup_xstate_comp(); + setup_xstate_comp_offsets(); print_xstate_offset_size(); pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", @@ -897,8 +895,6 @@ const void *get_xsave_field_ptr(int xfeature_nr) #ifdef CONFIG_ARCH_HAS_PKEYS -#define NR_VALID_PKRU_BITS (CONFIG_NR_PROTECTION_KEYS * 2) -#define PKRU_VALID_MASK (NR_VALID_PKRU_BITS - 1) /* * This will go out and modify PKRU register to set the access * rights for @pkey to @init_val. @@ -917,6 +913,13 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, if (!boot_cpu_has(X86_FEATURE_OSPKE)) return -EINVAL; + /* + * This code should only be called with valid 'pkey' + * values originating from in-kernel users. Complain + * if a bad value is observed. + */ + WARN_ON_ONCE(pkey >= arch_max_pkey()); + /* Set the bits we need in PKRU: */ if (init_val & PKEY_DISABLE_ACCESS) new_pkru_bits |= PKRU_AD_BIT; diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 3923ab4630d7..f66a6b90f954 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -67,11 +67,6 @@ __HEAD SYM_CODE_START(startup_32) movl pa(initial_stack),%ecx - /* test KEEP_SEGMENTS flag to see if the bootloader is asking - us to not reload segments */ - testb $KEEP_SEGMENTS, BP_loadflags(%esi) - jnz 2f - /* * Set segments to known values. */ @@ -82,7 +77,6 @@ SYM_CODE_START(startup_32) movl %eax,%fs movl %eax,%gs movl %eax,%ss -2: leal -__PAGE_OFFSET(%ecx),%esp /* diff --git a/arch/x86/kernel/ima_arch.c b/arch/x86/kernel/ima_arch.c index 23054909c8dd..7dfb1e808928 100644 --- a/arch/x86/kernel/ima_arch.c +++ b/arch/x86/kernel/ima_arch.c @@ -17,7 +17,7 @@ static enum efi_secureboot_mode get_sb_mode(void) size = sizeof(secboot); - if (!efi_enabled(EFI_RUNTIME_SERVICES)) { + if (!efi_rt_services_supported(EFI_RT_SUPPORTED_GET_VARIABLE)) { pr_info("ima: secureboot mode unknown, no efi\n"); return efi_secureboot_mode_unknown; } diff --git a/arch/x86/kernel/ioport.c b/arch/x86/kernel/ioport.c index 8abeee0dd7bf..a53e7b4a7419 100644 --- a/arch/x86/kernel/ioport.c +++ b/arch/x86/kernel/ioport.c @@ -13,6 +13,7 @@ #include <asm/io_bitmap.h> #include <asm/desc.h> +#include <asm/syscalls.h> #ifdef CONFIG_X86_IOPL_IOPERM diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 21efee32e2b1..c7965ff429c5 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -230,7 +230,7 @@ u64 arch_irq_stat(void) * SMP cross-CPU interrupts have their own specific * handlers). */ -__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs) +__visible void __irq_entry do_IRQ(struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); struct irq_desc * desc; @@ -263,7 +263,6 @@ __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs) exiting_irq(); set_irq_regs(old_regs); - return 1; } #ifdef CONFIG_X86_LOCAL_APIC diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index 16919a9671fa..5aa523c2d573 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -44,15 +44,6 @@ * (these are usually mapped into the 0x30-0xff vector range) */ -/* - * IRQ2 is cascade interrupt to second interrupt controller - */ -static struct irqaction irq2 = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; - DEFINE_PER_CPU(vector_irq_t, vector_irq) = { [0 ... NR_VECTORS - 1] = VECTOR_UNUSED, }; @@ -84,7 +75,7 @@ void __init init_IRQ(void) * On cpu 0, Assign ISA_IRQ_VECTOR(irq) to IRQ 0..15. * If these IRQ's are handled by legacy interrupt-controllers like PIC, * then this configuration will likely be static after the boot. If - * these IRQ's are handled by more mordern controllers like IO-APIC, + * these IRQs are handled by more modern controllers like IO-APIC, * then this vector space can be freed and re-used dynamically as the * irq's migrate etc. */ @@ -104,6 +95,9 @@ void __init native_init_IRQ(void) idt_setup_apic_and_irq_gates(); lapic_assign_system_vectors(); - if (!acpi_ioapic && !of_ioapic && nr_legacy_irqs()) - setup_irq(2, &irq2); + if (!acpi_ioapic && !of_ioapic && nr_legacy_irqs()) { + /* IRQ2 is cascade interrupt to second interrupt controller */ + if (request_irq(2, no_action, IRQF_NO_THREAD, "cascade", NULL)) + pr_err("%s: request_irq() failed\n", "cascade"); + } } diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 9c4498ea0b3c..5ba8477c2cb7 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -58,7 +58,7 @@ __jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, return code; } -static void inline __jump_label_transform(struct jump_entry *entry, +static inline void __jump_label_transform(struct jump_entry *entry, enum jump_label_type type, int init) { diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index f293d872602a..db6578d45157 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -141,9 +141,8 @@ prepare_add_efi_setup_data(struct boot_params *params, struct setup_data *sd = (void *)params + efi_setup_data_offset; struct efi_setup_data *esd = (void *)sd + sizeof(struct setup_data); - esd->fw_vendor = efi.fw_vendor; - esd->runtime = efi.runtime; - esd->tables = efi.config_table; + esd->fw_vendor = efi_fw_vendor; + esd->tables = efi_config_table; esd->smbios = efi.smbios; sd->type = SETUP_EFI; diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 3f45b5c43a71..ea13f6888284 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -71,6 +71,21 @@ found: return (unsigned long)buf; } +static void synthesize_clac(kprobe_opcode_t *addr) +{ + /* + * Can't be static_cpu_has() due to how objtool treats this feature bit. + * This isn't a fast path anyway. + */ + if (!boot_cpu_has(X86_FEATURE_SMAP)) + return; + + /* Replace the NOP3 with CLAC */ + addr[0] = 0x0f; + addr[1] = 0x01; + addr[2] = 0xca; +} + /* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) { @@ -92,6 +107,9 @@ asm ( /* We don't bother saving the ss register */ " pushq %rsp\n" " pushfq\n" + ".global optprobe_template_clac\n" + "optprobe_template_clac:\n" + ASM_NOP3 SAVE_REGS_STRING " movq %rsp, %rsi\n" ".global optprobe_template_val\n" @@ -111,6 +129,9 @@ asm ( #else /* CONFIG_X86_32 */ " pushl %esp\n" " pushfl\n" + ".global optprobe_template_clac\n" + "optprobe_template_clac:\n" + ASM_NOP3 SAVE_REGS_STRING " movl %esp, %edx\n" ".global optprobe_template_val\n" @@ -134,6 +155,8 @@ asm ( void optprobe_template_func(void); STACK_FRAME_NON_STANDARD(optprobe_template_func); +#define TMPL_CLAC_IDX \ + ((long)optprobe_template_clac - (long)optprobe_template_entry) #define TMPL_MOVE_IDX \ ((long)optprobe_template_val - (long)optprobe_template_entry) #define TMPL_CALL_IDX \ @@ -389,6 +412,8 @@ int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, op->optinsn.size = ret; len = TMPL_END_IDX + op->optinsn.size; + synthesize_clac(buf + TMPL_CLAC_IDX); + /* Set probe information */ synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op); diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 904494b924c1..34b18f6eeb2c 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -159,12 +159,19 @@ bool kvm_check_and_clear_guest_paused(void) return ret; } +static int kvm_cs_enable(struct clocksource *cs) +{ + vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK); + return 0; +} + struct clocksource kvm_clock = { .name = "kvm-clock", .read = kvm_clock_get_cycles, .rating = 400, .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .enable = kvm_cs_enable, }; EXPORT_SYMBOL_GPL(kvm_clock); @@ -272,7 +279,7 @@ static int __init kvm_setup_vsyscall_timeinfo(void) if (!(flags & PVCLOCK_TSC_STABLE_BIT)) return 0; - kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK; + kvm_clock.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK; #endif kvmclock_init_mem(); diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index c57e1ca70fd1..84c3ba32f211 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -27,7 +27,6 @@ #include <asm/tlb.h> #include <asm/desc.h> #include <asm/mmu_context.h> -#include <asm/syscalls.h> #include <asm/pgtable_areas.h> /* This is a multiple of PAGE_SIZE. */ diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 54c21d6abd5a..6407ea21fa1b 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -403,9 +403,9 @@ static void default_do_nmi(struct pt_regs *regs) * a 'real' unknown NMI. For example, while processing * a perf NMI another perf NMI comes in along with a * 'real' unknown NMI. These two NMIs get combined into - * one (as descibed above). When the next NMI gets + * one (as described above). When the next NMI gets * processed, it will be flagged by perf as handled, but - * noone will know that there was a 'real' unknown NMI sent + * no one will know that there was a 'real' unknown NMI sent * also. As a result it gets swallowed. Or if the first * perf NMI returns two events handled then the second * NMI will get eaten by the logic below, again losing a diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 3053c85e0e42..9da70b279dad 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -28,7 +28,6 @@ #include <linux/hw_breakpoint.h> #include <asm/cpu.h> #include <asm/apic.h> -#include <asm/syscalls.h> #include <linux/uaccess.h> #include <asm/mwait.h> #include <asm/fpu/internal.h> @@ -650,6 +649,9 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) /* Enforce MSR update to ensure consistent state */ __speculation_ctrl_update(~tifn, tifn); } + + if ((tifp ^ tifn) & _TIF_SLD) + switch_to_sld(tifn); } /* diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 5052ced43373..954b013cc585 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -49,7 +49,6 @@ #include <asm/tlbflush.h> #include <asm/cpu.h> -#include <asm/syscalls.h> #include <asm/debugreg.h> #include <asm/switch_to.h> #include <asm/vm86.h> diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index ffd497804dbc..5ef9d8f25b0e 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -48,7 +48,6 @@ #include <asm/desc.h> #include <asm/proto.h> #include <asm/ia32.h> -#include <asm/syscalls.h> #include <asm/debugreg.h> #include <asm/switch_to.h> #include <asm/xen/hypervisor.h> diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c index 10125358b9c4..11065dc03f5b 100644 --- a/arch/x86/kernel/pvclock.c +++ b/arch/x86/kernel/pvclock.c @@ -145,7 +145,7 @@ void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock, void pvclock_set_pvti_cpu0_va(struct pvclock_vsyscall_time_info *pvti) { - WARN_ON(vclock_was_used(VCLOCK_PVCLOCK)); + WARN_ON(vclock_was_used(VDSO_CLOCKMODE_PVCLOCK)); pvti_cpu0_va = pvti; } diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 0cc7c0b106bb..3ca43be4f9cf 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -531,7 +531,7 @@ static void emergency_vmx_disable_all(void) /* * We need to disable VMX on all CPUs before rebooting, otherwise - * we risk hanging up the machine, because the CPU ignore INIT + * we risk hanging up the machine, because the CPU ignores INIT * signals when VMX is enabled. * * We can't take any locks and we may be on an inconsistent diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index ef3ba99068d3..a4d9a261425b 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -9,6 +9,8 @@ #include <asm/kexec.h> #include <asm/processor-flags.h> #include <asm/pgtable_types.h> +#include <asm/nospec-branch.h> +#include <asm/unwind_hints.h> /* * Must be relocatable PIC code callable as a C function @@ -39,6 +41,7 @@ .align PAGE_SIZE .code64 SYM_CODE_START_NOALIGN(relocate_kernel) + UNWIND_HINT_EMPTY /* * %rdi indirection_page * %rsi page_list @@ -105,6 +108,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) + UNWIND_HINT_EMPTY /* set return address to 0 if not preserving context */ pushq $0 /* store the start address on the stack */ @@ -192,14 +196,12 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) 1: popq %rdx leaq PAGE_SIZE(%r10), %rsp + ANNOTATE_RETPOLINE_SAFE call *%rdx /* get the re-entry point of the peer system */ movq 0(%rsp), %rbp - call 1f -1: - popq %r8 - subq $(1b - relocate_kernel), %r8 + leaq relocate_kernel(%rip), %r8 movq CP_PA_SWAP_PAGE(%r8), %r10 movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi movq CP_PA_TABLE_PAGE(%r8), %rax @@ -212,6 +214,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) + UNWIND_HINT_EMPTY movq RSP(%r8), %rsp movq CR4(%r8), %rax movq %rax, %cr4 @@ -233,6 +236,7 @@ SYM_CODE_END(virtual_mapped) /* Do the copies */ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) + UNWIND_HINT_EMPTY movq %rdi, %rcx /* Put the page_list in %rcx */ xorl %edi, %edi xorl %esi, %esi diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index a74262c71484..e6b545047f38 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -64,7 +64,6 @@ RESERVE_BRK(dmi_alloc, 65536); * at link time, with RESERVE_BRK*() facility reserving additional * chunks. */ -static __initdata unsigned long _brk_start = (unsigned long)__brk_base; unsigned long _brk_end = (unsigned long)__brk_base; diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 8a29573851a3..83b74fb38c8f 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -42,29 +42,9 @@ #endif /* CONFIG_X86_64 */ #include <asm/syscall.h> -#include <asm/syscalls.h> - #include <asm/sigframe.h> #include <asm/signal.h> -#define COPY(x) do { \ - get_user_ex(regs->x, &sc->x); \ -} while (0) - -#define GET_SEG(seg) ({ \ - unsigned short tmp; \ - get_user_ex(tmp, &sc->seg); \ - tmp; \ -}) - -#define COPY_SEG(seg) do { \ - regs->seg = GET_SEG(seg); \ -} while (0) - -#define COPY_SEG_CPL3(seg) do { \ - regs->seg = GET_SEG(seg) | 3; \ -} while (0) - #ifdef CONFIG_X86_64 /* * If regs->ss will cause an IRET fault, change it. Otherwise leave it @@ -92,53 +72,58 @@ static void force_valid_ss(struct pt_regs *regs) ar != (AR_DPL3 | AR_S | AR_P | AR_TYPE_RWDATA_EXPDOWN)) regs->ss = __USER_DS; } +# define CONTEXT_COPY_SIZE offsetof(struct sigcontext, reserved1) +#else +# define CONTEXT_COPY_SIZE sizeof(struct sigcontext) #endif static int restore_sigcontext(struct pt_regs *regs, - struct sigcontext __user *sc, + struct sigcontext __user *usc, unsigned long uc_flags) { - unsigned long buf_val; - void __user *buf; - unsigned int tmpflags; - unsigned int err = 0; + struct sigcontext sc; /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - get_user_try { + if (copy_from_user(&sc, usc, CONTEXT_COPY_SIZE)) + return -EFAULT; #ifdef CONFIG_X86_32 - set_user_gs(regs, GET_SEG(gs)); - COPY_SEG(fs); - COPY_SEG(es); - COPY_SEG(ds); + set_user_gs(regs, sc.gs); + regs->fs = sc.fs; + regs->es = sc.es; + regs->ds = sc.ds; #endif /* CONFIG_X86_32 */ - COPY(di); COPY(si); COPY(bp); COPY(sp); COPY(bx); - COPY(dx); COPY(cx); COPY(ip); COPY(ax); + regs->bx = sc.bx; + regs->cx = sc.cx; + regs->dx = sc.dx; + regs->si = sc.si; + regs->di = sc.di; + regs->bp = sc.bp; + regs->ax = sc.ax; + regs->sp = sc.sp; + regs->ip = sc.ip; #ifdef CONFIG_X86_64 - COPY(r8); - COPY(r9); - COPY(r10); - COPY(r11); - COPY(r12); - COPY(r13); - COPY(r14); - COPY(r15); + regs->r8 = sc.r8; + regs->r9 = sc.r9; + regs->r10 = sc.r10; + regs->r11 = sc.r11; + regs->r12 = sc.r12; + regs->r13 = sc.r13; + regs->r14 = sc.r14; + regs->r15 = sc.r15; #endif /* CONFIG_X86_64 */ - COPY_SEG_CPL3(cs); - COPY_SEG_CPL3(ss); - - get_user_ex(tmpflags, &sc->flags); - regs->flags = (regs->flags & ~FIX_EFLAGS) | (tmpflags & FIX_EFLAGS); - regs->orig_ax = -1; /* disable syscall checks */ + /* Get CS/SS and force CPL3 */ + regs->cs = sc.cs | 0x03; + regs->ss = sc.ss | 0x03; - get_user_ex(buf_val, &sc->fpstate); - buf = (void __user *)buf_val; - } get_user_catch(err); + regs->flags = (regs->flags & ~FIX_EFLAGS) | (sc.flags & FIX_EFLAGS); + /* disable syscall checks */ + regs->orig_ax = -1; #ifdef CONFIG_X86_64 /* @@ -149,70 +134,78 @@ static int restore_sigcontext(struct pt_regs *regs, force_valid_ss(regs); #endif - err |= fpu__restore_sig(buf, IS_ENABLED(CONFIG_X86_32)); - - return err; + return fpu__restore_sig((void __user *)sc.fpstate, + IS_ENABLED(CONFIG_X86_32)); } -int setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate, +static __always_inline int +__unsafe_setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate, struct pt_regs *regs, unsigned long mask) { - int err = 0; - - put_user_try { - #ifdef CONFIG_X86_32 - put_user_ex(get_user_gs(regs), (unsigned int __user *)&sc->gs); - put_user_ex(regs->fs, (unsigned int __user *)&sc->fs); - put_user_ex(regs->es, (unsigned int __user *)&sc->es); - put_user_ex(regs->ds, (unsigned int __user *)&sc->ds); + unsafe_put_user(get_user_gs(regs), + (unsigned int __user *)&sc->gs, Efault); + unsafe_put_user(regs->fs, (unsigned int __user *)&sc->fs, Efault); + unsafe_put_user(regs->es, (unsigned int __user *)&sc->es, Efault); + unsafe_put_user(regs->ds, (unsigned int __user *)&sc->ds, Efault); #endif /* CONFIG_X86_32 */ - put_user_ex(regs->di, &sc->di); - put_user_ex(regs->si, &sc->si); - put_user_ex(regs->bp, &sc->bp); - put_user_ex(regs->sp, &sc->sp); - put_user_ex(regs->bx, &sc->bx); - put_user_ex(regs->dx, &sc->dx); - put_user_ex(regs->cx, &sc->cx); - put_user_ex(regs->ax, &sc->ax); + unsafe_put_user(regs->di, &sc->di, Efault); + unsafe_put_user(regs->si, &sc->si, Efault); + unsafe_put_user(regs->bp, &sc->bp, Efault); + unsafe_put_user(regs->sp, &sc->sp, Efault); + unsafe_put_user(regs->bx, &sc->bx, Efault); + unsafe_put_user(regs->dx, &sc->dx, Efault); + unsafe_put_user(regs->cx, &sc->cx, Efault); + unsafe_put_user(regs->ax, &sc->ax, Efault); #ifdef CONFIG_X86_64 - put_user_ex(regs->r8, &sc->r8); - put_user_ex(regs->r9, &sc->r9); - put_user_ex(regs->r10, &sc->r10); - put_user_ex(regs->r11, &sc->r11); - put_user_ex(regs->r12, &sc->r12); - put_user_ex(regs->r13, &sc->r13); - put_user_ex(regs->r14, &sc->r14); - put_user_ex(regs->r15, &sc->r15); + unsafe_put_user(regs->r8, &sc->r8, Efault); + unsafe_put_user(regs->r9, &sc->r9, Efault); + unsafe_put_user(regs->r10, &sc->r10, Efault); + unsafe_put_user(regs->r11, &sc->r11, Efault); + unsafe_put_user(regs->r12, &sc->r12, Efault); + unsafe_put_user(regs->r13, &sc->r13, Efault); + unsafe_put_user(regs->r14, &sc->r14, Efault); + unsafe_put_user(regs->r15, &sc->r15, Efault); #endif /* CONFIG_X86_64 */ - put_user_ex(current->thread.trap_nr, &sc->trapno); - put_user_ex(current->thread.error_code, &sc->err); - put_user_ex(regs->ip, &sc->ip); + unsafe_put_user(current->thread.trap_nr, &sc->trapno, Efault); + unsafe_put_user(current->thread.error_code, &sc->err, Efault); + unsafe_put_user(regs->ip, &sc->ip, Efault); #ifdef CONFIG_X86_32 - put_user_ex(regs->cs, (unsigned int __user *)&sc->cs); - put_user_ex(regs->flags, &sc->flags); - put_user_ex(regs->sp, &sc->sp_at_signal); - put_user_ex(regs->ss, (unsigned int __user *)&sc->ss); + unsafe_put_user(regs->cs, (unsigned int __user *)&sc->cs, Efault); + unsafe_put_user(regs->flags, &sc->flags, Efault); + unsafe_put_user(regs->sp, &sc->sp_at_signal, Efault); + unsafe_put_user(regs->ss, (unsigned int __user *)&sc->ss, Efault); #else /* !CONFIG_X86_32 */ - put_user_ex(regs->flags, &sc->flags); - put_user_ex(regs->cs, &sc->cs); - put_user_ex(0, &sc->gs); - put_user_ex(0, &sc->fs); - put_user_ex(regs->ss, &sc->ss); + unsafe_put_user(regs->flags, &sc->flags, Efault); + unsafe_put_user(regs->cs, &sc->cs, Efault); + unsafe_put_user(0, &sc->gs, Efault); + unsafe_put_user(0, &sc->fs, Efault); + unsafe_put_user(regs->ss, &sc->ss, Efault); #endif /* CONFIG_X86_32 */ - put_user_ex(fpstate, (unsigned long __user *)&sc->fpstate); - - /* non-iBCS2 extensions.. */ - put_user_ex(mask, &sc->oldmask); - put_user_ex(current->thread.cr2, &sc->cr2); - } put_user_catch(err); + unsafe_put_user(fpstate, (unsigned long __user *)&sc->fpstate, Efault); - return err; + /* non-iBCS2 extensions.. */ + unsafe_put_user(mask, &sc->oldmask, Efault); + unsafe_put_user(current->thread.cr2, &sc->cr2, Efault); + return 0; +Efault: + return -EFAULT; } +#define unsafe_put_sigcontext(sc, fp, regs, set, label) \ +do { \ + if (__unsafe_setup_sigcontext(sc, fp, regs, set->sig[0])) \ + goto label; \ +} while(0); + +#define unsafe_put_sigmask(set, frame, label) \ + unsafe_put_user(*(__u64 *)(set), \ + (__u64 __user *)&(frame)->uc.uc_sigmask, \ + label) + /* * Set up a signal frame. */ @@ -312,26 +305,16 @@ __setup_frame(int sig, struct ksignal *ksig, sigset_t *set, { struct sigframe __user *frame; void __user *restorer; - int err = 0; - void __user *fpstate = NULL; - - frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate); - - if (!access_ok(frame, sizeof(*frame))) - return -EFAULT; + void __user *fp = NULL; - if (__put_user(sig, &frame->sig)) - return -EFAULT; + frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fp); - if (setup_sigcontext(&frame->sc, fpstate, regs, set->sig[0])) + if (!user_access_begin(frame, sizeof(*frame))) return -EFAULT; - if (_NSIG_WORDS > 1) { - if (__copy_to_user(&frame->extramask, &set->sig[1], - sizeof(frame->extramask))) - return -EFAULT; - } - + unsafe_put_user(sig, &frame->sig, Efault); + unsafe_put_sigcontext(&frame->sc, fp, regs, set, Efault); + unsafe_put_user(set->sig[1], &frame->extramask[0], Efault); if (current->mm->context.vdso) restorer = current->mm->context.vdso + vdso_image_32.sym___kernel_sigreturn; @@ -341,7 +324,7 @@ __setup_frame(int sig, struct ksignal *ksig, sigset_t *set, restorer = ksig->ka.sa.sa_restorer; /* Set up to return from userspace. */ - err |= __put_user(restorer, &frame->pretcode); + unsafe_put_user(restorer, &frame->pretcode, Efault); /* * This is popl %eax ; movl $__NR_sigreturn, %eax ; int $0x80 @@ -350,10 +333,8 @@ __setup_frame(int sig, struct ksignal *ksig, sigset_t *set, * reasons and because gdb uses it as a signature to notice * signal handler stack frames. */ - err |= __put_user(*((u64 *)&retcode), (u64 *)frame->retcode); - - if (err) - return -EFAULT; + unsafe_put_user(*((u64 *)&retcode), (u64 *)frame->retcode, Efault); + user_access_end(); /* Set up registers for signal handler */ regs->sp = (unsigned long)frame; @@ -368,6 +349,10 @@ __setup_frame(int sig, struct ksignal *ksig, sigset_t *set, regs->cs = __USER_CS; return 0; + +Efault: + user_access_end(); + return -EFAULT; } static int __setup_rt_frame(int sig, struct ksignal *ksig, @@ -375,50 +360,45 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig, { struct rt_sigframe __user *frame; void __user *restorer; - int err = 0; - void __user *fpstate = NULL; + void __user *fp = NULL; - frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate); + frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fp); - if (!access_ok(frame, sizeof(*frame))) + if (!user_access_begin(frame, sizeof(*frame))) return -EFAULT; - put_user_try { - put_user_ex(sig, &frame->sig); - put_user_ex(&frame->info, &frame->pinfo); - put_user_ex(&frame->uc, &frame->puc); + unsafe_put_user(sig, &frame->sig, Efault); + unsafe_put_user(&frame->info, &frame->pinfo, Efault); + unsafe_put_user(&frame->uc, &frame->puc, Efault); - /* Create the ucontext. */ - if (static_cpu_has(X86_FEATURE_XSAVE)) - put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags); - else - put_user_ex(0, &frame->uc.uc_flags); - put_user_ex(0, &frame->uc.uc_link); - save_altstack_ex(&frame->uc.uc_stack, regs->sp); + /* Create the ucontext. */ + if (static_cpu_has(X86_FEATURE_XSAVE)) + unsafe_put_user(UC_FP_XSTATE, &frame->uc.uc_flags, Efault); + else + unsafe_put_user(0, &frame->uc.uc_flags, Efault); + unsafe_put_user(0, &frame->uc.uc_link, Efault); + unsafe_save_altstack(&frame->uc.uc_stack, regs->sp, Efault); - /* Set up to return from userspace. */ - restorer = current->mm->context.vdso + - vdso_image_32.sym___kernel_rt_sigreturn; - if (ksig->ka.sa.sa_flags & SA_RESTORER) - restorer = ksig->ka.sa.sa_restorer; - put_user_ex(restorer, &frame->pretcode); + /* Set up to return from userspace. */ + restorer = current->mm->context.vdso + + vdso_image_32.sym___kernel_rt_sigreturn; + if (ksig->ka.sa.sa_flags & SA_RESTORER) + restorer = ksig->ka.sa.sa_restorer; + unsafe_put_user(restorer, &frame->pretcode, Efault); - /* - * This is movl $__NR_rt_sigreturn, %ax ; int $0x80 - * - * WE DO NOT USE IT ANY MORE! It's only left here for historical - * reasons and because gdb uses it as a signature to notice - * signal handler stack frames. - */ - put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode); - } put_user_catch(err); + /* + * This is movl $__NR_rt_sigreturn, %ax ; int $0x80 + * + * WE DO NOT USE IT ANY MORE! It's only left here for historical + * reasons and because gdb uses it as a signature to notice + * signal handler stack frames. + */ + unsafe_put_user(*((u64 *)&rt_retcode), (u64 *)frame->retcode, Efault); + unsafe_put_sigcontext(&frame->uc.uc_mcontext, fp, regs, set, Efault); + unsafe_put_sigmask(set, frame, Efault); + user_access_end(); - err |= copy_siginfo_to_user(&frame->info, &ksig->info); - err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate, - regs, set->sig[0]); - err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); - - if (err) + if (copy_siginfo_to_user(&frame->info, &ksig->info)) return -EFAULT; /* Set up registers for signal handler */ @@ -434,6 +414,9 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig, regs->cs = __USER_CS; return 0; +Efault: + user_access_end(); + return -EFAULT; } #else /* !CONFIG_X86_32 */ static unsigned long frame_uc_flags(struct pt_regs *regs) @@ -457,43 +440,34 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig, struct rt_sigframe __user *frame; void __user *fp = NULL; unsigned long uc_flags; - int err = 0; + + /* x86-64 should always use SA_RESTORER. */ + if (!(ksig->ka.sa.sa_flags & SA_RESTORER)) + return -EFAULT; frame = get_sigframe(&ksig->ka, regs, sizeof(struct rt_sigframe), &fp); + uc_flags = frame_uc_flags(regs); - if (!access_ok(frame, sizeof(*frame))) + if (!user_access_begin(frame, sizeof(*frame))) return -EFAULT; + /* Create the ucontext. */ + unsafe_put_user(uc_flags, &frame->uc.uc_flags, Efault); + unsafe_put_user(0, &frame->uc.uc_link, Efault); + unsafe_save_altstack(&frame->uc.uc_stack, regs->sp, Efault); + + /* Set up to return from userspace. If provided, use a stub + already in userspace. */ + unsafe_put_user(ksig->ka.sa.sa_restorer, &frame->pretcode, Efault); + unsafe_put_sigcontext(&frame->uc.uc_mcontext, fp, regs, set, Efault); + unsafe_put_sigmask(set, frame, Efault); + user_access_end(); + if (ksig->ka.sa.sa_flags & SA_SIGINFO) { if (copy_siginfo_to_user(&frame->info, &ksig->info)) return -EFAULT; } - uc_flags = frame_uc_flags(regs); - - put_user_try { - /* Create the ucontext. */ - put_user_ex(uc_flags, &frame->uc.uc_flags); - put_user_ex(0, &frame->uc.uc_link); - save_altstack_ex(&frame->uc.uc_stack, regs->sp); - - /* Set up to return from userspace. If provided, use a stub - already in userspace. */ - /* x86-64 should always use SA_RESTORER. */ - if (ksig->ka.sa.sa_flags & SA_RESTORER) { - put_user_ex(ksig->ka.sa.sa_restorer, &frame->pretcode); - } else { - /* could use a vstub here */ - err |= -EFAULT; - } - } put_user_catch(err); - - err |= setup_sigcontext(&frame->uc.uc_mcontext, fp, regs, set->sig[0]); - err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); - - if (err) - return -EFAULT; - /* Set up registers for signal handler */ regs->di = sig; /* In case the signal handler was declared without prototypes */ @@ -530,6 +504,10 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig, force_valid_ss(regs); return 0; + +Efault: + user_access_end(); + return -EFAULT; } #endif /* CONFIG_X86_32 */ @@ -541,44 +519,33 @@ static int x32_setup_rt_frame(struct ksignal *ksig, struct rt_sigframe_x32 __user *frame; unsigned long uc_flags; void __user *restorer; - int err = 0; - void __user *fpstate = NULL; - - frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate); + void __user *fp = NULL; - if (!access_ok(frame, sizeof(*frame))) + if (!(ksig->ka.sa.sa_flags & SA_RESTORER)) return -EFAULT; - if (ksig->ka.sa.sa_flags & SA_SIGINFO) { - if (__copy_siginfo_to_user32(&frame->info, &ksig->info, true)) - return -EFAULT; - } + frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fp); uc_flags = frame_uc_flags(regs); - put_user_try { - /* Create the ucontext. */ - put_user_ex(uc_flags, &frame->uc.uc_flags); - put_user_ex(0, &frame->uc.uc_link); - compat_save_altstack_ex(&frame->uc.uc_stack, regs->sp); - put_user_ex(0, &frame->uc.uc__pad0); - - if (ksig->ka.sa.sa_flags & SA_RESTORER) { - restorer = ksig->ka.sa.sa_restorer; - } else { - /* could use a vstub here */ - restorer = NULL; - err |= -EFAULT; - } - put_user_ex(restorer, (unsigned long __user *)&frame->pretcode); - } put_user_catch(err); + if (!user_access_begin(frame, sizeof(*frame))) + return -EFAULT; - err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate, - regs, set->sig[0]); - err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); + /* Create the ucontext. */ + unsafe_put_user(uc_flags, &frame->uc.uc_flags, Efault); + unsafe_put_user(0, &frame->uc.uc_link, Efault); + unsafe_compat_save_altstack(&frame->uc.uc_stack, regs->sp, Efault); + unsafe_put_user(0, &frame->uc.uc__pad0, Efault); + restorer = ksig->ka.sa.sa_restorer; + unsafe_put_user(restorer, (unsigned long __user *)&frame->pretcode, Efault); + unsafe_put_sigcontext(&frame->uc.uc_mcontext, fp, regs, set, Efault); + unsafe_put_sigmask(set, frame, Efault); + user_access_end(); - if (err) - return -EFAULT; + if (ksig->ka.sa.sa_flags & SA_SIGINFO) { + if (__copy_siginfo_to_user32(&frame->info, &ksig->info, true)) + return -EFAULT; + } /* Set up registers for signal handler */ regs->sp = (unsigned long) frame; @@ -597,6 +564,11 @@ static int x32_setup_rt_frame(struct ksignal *ksig, #endif /* CONFIG_X86_X32_ABI */ return 0; +#ifdef CONFIG_X86_X32_ABI +Efault: + user_access_end(); + return -EFAULT; +#endif } /* @@ -613,9 +585,8 @@ SYSCALL_DEFINE0(sigreturn) if (!access_ok(frame, sizeof(*frame))) goto badframe; - if (__get_user(set.sig[0], &frame->sc.oldmask) || (_NSIG_WORDS > 1 - && __copy_from_user(&set.sig[1], &frame->extramask, - sizeof(frame->extramask)))) + if (__get_user(set.sig[0], &frame->sc.oldmask) || + __get_user(set.sig[1], &frame->extramask[0])) goto badframe; set_current_blocked(&set); @@ -645,7 +616,7 @@ SYSCALL_DEFINE0(rt_sigreturn) frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long)); if (!access_ok(frame, sizeof(*frame))) goto badframe; - if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) + if (__get_user(*(__u64 *)&set, (__u64 __user *)&frame->uc.uc_sigmask)) goto badframe; if (__get_user(uc_flags, &frame->uc.uc_flags)) goto badframe; @@ -859,7 +830,7 @@ void signal_fault(struct pt_regs *regs, void __user *frame, char *where) } #ifdef CONFIG_X86_X32_ABI -asmlinkage long sys32_x32_rt_sigreturn(void) +COMPAT_SYSCALL_DEFINE0(x32_rt_sigreturn) { struct pt_regs *regs = current_pt_regs(); struct rt_sigframe_x32 __user *frame; @@ -870,7 +841,7 @@ asmlinkage long sys32_x32_rt_sigreturn(void) if (!access_ok(frame, sizeof(*frame))) goto badframe; - if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) + if (__get_user(set.sig[0], (__u64 __user *)&frame->uc.uc_sigmask)) goto badframe; if (__get_user(uc_flags, &frame->uc.uc_flags)) goto badframe; diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 69881b2d446c..fe3ab9632f3b 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -147,6 +147,8 @@ static inline void smpboot_restore_warm_reset_vector(void) *((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0; } +static void init_freq_invariance(void); + /* * Report back to the Boot Processor during boot time or to the caller processor * during CPU online. @@ -183,6 +185,8 @@ static void smp_callin(void) */ set_cpu_sibling_map(raw_smp_processor_id()); + init_freq_invariance(); + /* * Get our bogomips. * Update loops_per_jiffy in cpu_data. Previous call to @@ -466,7 +470,7 @@ static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) */ static const struct x86_cpu_id snc_cpu[] = { - { X86_VENDOR_INTEL, 6, INTEL_FAM6_SKYLAKE_X }, + X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL), {} }; @@ -1337,7 +1341,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) set_sched_topology(x86_topology); set_cpu_sibling_map(0); - + init_freq_invariance(); smp_sanity_check(); switch (apic_intr_mode) { @@ -1434,7 +1438,7 @@ early_param("possible_cpus", _setup_possible_cpus); /* * cpu_possible_mask should be static, it cannot change as cpu's * are onlined, or offlined. The reason is per-cpu data-structures - * are allocated by some modules at init time, and dont expect to + * are allocated by some modules at init time, and don't expect to * do this dynamically on cpu arrival/departure. * cpu_present_mask on the other hand can change dynamically. * In case when cpu_hotplug is not compiled, then we resort to current @@ -1764,3 +1768,287 @@ void native_play_dead(void) } #endif + +/* + * APERF/MPERF frequency ratio computation. + * + * The scheduler wants to do frequency invariant accounting and needs a <1 + * ratio to account for the 'current' frequency, corresponding to + * freq_curr / freq_max. + * + * Since the frequency freq_curr on x86 is controlled by micro-controller and + * our P-state setting is little more than a request/hint, we need to observe + * the effective frequency 'BusyMHz', i.e. the average frequency over a time + * interval after discarding idle time. This is given by: + * + * BusyMHz = delta_APERF / delta_MPERF * freq_base + * + * where freq_base is the max non-turbo P-state. + * + * The freq_max term has to be set to a somewhat arbitrary value, because we + * can't know which turbo states will be available at a given point in time: + * it all depends on the thermal headroom of the entire package. We set it to + * the turbo level with 4 cores active. + * + * Benchmarks show that's a good compromise between the 1C turbo ratio + * (freq_curr/freq_max would rarely reach 1) and something close to freq_base, + * which would ignore the entire turbo range (a conspicuous part, making + * freq_curr/freq_max always maxed out). + * + * An exception to the heuristic above is the Atom uarch, where we choose the + * highest turbo level for freq_max since Atom's are generally oriented towards + * power efficiency. + * + * Setting freq_max to anything less than the 1C turbo ratio makes the ratio + * freq_curr / freq_max to eventually grow >1, in which case we clip it to 1. + */ + +DEFINE_STATIC_KEY_FALSE(arch_scale_freq_key); + +static DEFINE_PER_CPU(u64, arch_prev_aperf); +static DEFINE_PER_CPU(u64, arch_prev_mperf); +static u64 arch_turbo_freq_ratio = SCHED_CAPACITY_SCALE; +static u64 arch_max_freq_ratio = SCHED_CAPACITY_SCALE; + +void arch_set_max_freq_ratio(bool turbo_disabled) +{ + arch_max_freq_ratio = turbo_disabled ? SCHED_CAPACITY_SCALE : + arch_turbo_freq_ratio; +} + +static bool turbo_disabled(void) +{ + u64 misc_en; + int err; + + err = rdmsrl_safe(MSR_IA32_MISC_ENABLE, &misc_en); + if (err) + return false; + + return (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); +} + +static bool slv_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq) +{ + int err; + + err = rdmsrl_safe(MSR_ATOM_CORE_RATIOS, base_freq); + if (err) + return false; + + err = rdmsrl_safe(MSR_ATOM_CORE_TURBO_RATIOS, turbo_freq); + if (err) + return false; + + *base_freq = (*base_freq >> 16) & 0x3F; /* max P state */ + *turbo_freq = *turbo_freq & 0x3F; /* 1C turbo */ + + return true; +} + +#include <asm/cpu_device_id.h> +#include <asm/intel-family.h> + +#define ICPU(model) \ + {X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF, 0} + +static const struct x86_cpu_id has_knl_turbo_ratio_limits[] = { + ICPU(INTEL_FAM6_XEON_PHI_KNL), + ICPU(INTEL_FAM6_XEON_PHI_KNM), + {} +}; + +static const struct x86_cpu_id has_skx_turbo_ratio_limits[] = { + ICPU(INTEL_FAM6_SKYLAKE_X), + {} +}; + +static const struct x86_cpu_id has_glm_turbo_ratio_limits[] = { + ICPU(INTEL_FAM6_ATOM_GOLDMONT), + ICPU(INTEL_FAM6_ATOM_GOLDMONT_D), + ICPU(INTEL_FAM6_ATOM_GOLDMONT_PLUS), + {} +}; + +static bool knl_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, + int num_delta_fratio) +{ + int fratio, delta_fratio, found; + int err, i; + u64 msr; + + if (!x86_match_cpu(has_knl_turbo_ratio_limits)) + return false; + + err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq); + if (err) + return false; + + *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */ + + err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr); + if (err) + return false; + + fratio = (msr >> 8) & 0xFF; + i = 16; + found = 0; + do { + if (found >= num_delta_fratio) { + *turbo_freq = fratio; + return true; + } + + delta_fratio = (msr >> (i + 5)) & 0x7; + + if (delta_fratio) { + found += 1; + fratio -= delta_fratio; + } + + i += 8; + } while (i < 64); + + return true; +} + +static bool skx_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int size) +{ + u64 ratios, counts; + u32 group_size; + int err, i; + + err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq); + if (err) + return false; + + *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */ + + err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &ratios); + if (err) + return false; + + err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT1, &counts); + if (err) + return false; + + for (i = 0; i < 64; i += 8) { + group_size = (counts >> i) & 0xFF; + if (group_size >= size) { + *turbo_freq = (ratios >> i) & 0xFF; + return true; + } + } + + return false; +} + +static bool core_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq) +{ + int err; + + err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq); + if (err) + return false; + + err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, turbo_freq); + if (err) + return false; + + *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */ + *turbo_freq = (*turbo_freq >> 24) & 0xFF; /* 4C turbo */ + + return true; +} + +static bool intel_set_max_freq_ratio(void) +{ + u64 base_freq, turbo_freq; + + if (slv_set_max_freq_ratio(&base_freq, &turbo_freq)) + goto out; + + if (x86_match_cpu(has_glm_turbo_ratio_limits) && + skx_set_max_freq_ratio(&base_freq, &turbo_freq, 1)) + goto out; + + if (knl_set_max_freq_ratio(&base_freq, &turbo_freq, 1)) + goto out; + + if (x86_match_cpu(has_skx_turbo_ratio_limits) && + skx_set_max_freq_ratio(&base_freq, &turbo_freq, 4)) + goto out; + + if (core_set_max_freq_ratio(&base_freq, &turbo_freq)) + goto out; + + return false; + +out: + arch_turbo_freq_ratio = div_u64(turbo_freq * SCHED_CAPACITY_SCALE, + base_freq); + arch_set_max_freq_ratio(turbo_disabled()); + return true; +} + +static void init_counter_refs(void *arg) +{ + u64 aperf, mperf; + + rdmsrl(MSR_IA32_APERF, aperf); + rdmsrl(MSR_IA32_MPERF, mperf); + + this_cpu_write(arch_prev_aperf, aperf); + this_cpu_write(arch_prev_mperf, mperf); +} + +static void init_freq_invariance(void) +{ + bool ret = false; + + if (smp_processor_id() != 0 || !boot_cpu_has(X86_FEATURE_APERFMPERF)) + return; + + if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) + ret = intel_set_max_freq_ratio(); + + if (ret) { + on_each_cpu(init_counter_refs, NULL, 1); + static_branch_enable(&arch_scale_freq_key); + } else { + pr_debug("Couldn't determine max cpu frequency, necessary for scale-invariant accounting.\n"); + } +} + +DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE; + +void arch_scale_freq_tick(void) +{ + u64 freq_scale; + u64 aperf, mperf; + u64 acnt, mcnt; + + if (!arch_scale_freq_invariant()) + return; + + rdmsrl(MSR_IA32_APERF, aperf); + rdmsrl(MSR_IA32_MPERF, mperf); + + acnt = aperf - this_cpu_read(arch_prev_aperf); + mcnt = mperf - this_cpu_read(arch_prev_mperf); + if (!mcnt) + return; + + this_cpu_write(arch_prev_aperf, aperf); + this_cpu_write(arch_prev_mperf, mperf); + + acnt <<= 2*SCHED_CAPACITY_SHIFT; + mcnt *= arch_max_freq_ratio; + + freq_scale = div64_u64(acnt, mcnt); + + if (freq_scale > SCHED_CAPACITY_SCALE) + freq_scale = SCHED_CAPACITY_SCALE; + + this_cpu_write(arch_freq_scale, freq_scale); +} diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c index 2d6898c2cb64..6ad43fc44556 100644 --- a/arch/x86/kernel/stacktrace.c +++ b/arch/x86/kernel/stacktrace.c @@ -96,7 +96,8 @@ struct stack_frame_user { }; static int -copy_stack_frame(const void __user *fp, struct stack_frame_user *frame) +copy_stack_frame(const struct stack_frame_user __user *fp, + struct stack_frame_user *frame) { int ret; @@ -105,7 +106,8 @@ copy_stack_frame(const void __user *fp, struct stack_frame_user *frame) ret = 1; pagefault_disable(); - if (__copy_from_user_inatomic(frame, fp, sizeof(*frame))) + if (__get_user(frame->next_fp, &fp->next_fp) || + __get_user(frame->ret_addr, &fp->ret_addr)) ret = 0; pagefault_enable(); diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/kernel/sys_ia32.c index 21790307121e..ab03fede1422 100644 --- a/arch/x86/ia32/sys_ia32.c +++ b/arch/x86/kernel/sys_ia32.c @@ -51,20 +51,80 @@ #define AA(__x) ((unsigned long)(__x)) - -COMPAT_SYSCALL_DEFINE3(x86_truncate64, const char __user *, filename, - unsigned long, offset_low, unsigned long, offset_high) +SYSCALL_DEFINE3(ia32_truncate64, const char __user *, filename, + unsigned long, offset_low, unsigned long, offset_high) { return ksys_truncate(filename, ((loff_t) offset_high << 32) | offset_low); } -COMPAT_SYSCALL_DEFINE3(x86_ftruncate64, unsigned int, fd, - unsigned long, offset_low, unsigned long, offset_high) +SYSCALL_DEFINE3(ia32_ftruncate64, unsigned int, fd, + unsigned long, offset_low, unsigned long, offset_high) { return ksys_ftruncate(fd, ((loff_t) offset_high << 32) | offset_low); } +/* warning: next two assume little endian */ +SYSCALL_DEFINE5(ia32_pread64, unsigned int, fd, char __user *, ubuf, + u32, count, u32, poslo, u32, poshi) +{ + return ksys_pread64(fd, ubuf, count, + ((loff_t)AA(poshi) << 32) | AA(poslo)); +} + +SYSCALL_DEFINE5(ia32_pwrite64, unsigned int, fd, const char __user *, ubuf, + u32, count, u32, poslo, u32, poshi) +{ + return ksys_pwrite64(fd, ubuf, count, + ((loff_t)AA(poshi) << 32) | AA(poslo)); +} + + +/* + * Some system calls that need sign extended arguments. This could be + * done by a generic wrapper. + */ +SYSCALL_DEFINE6(ia32_fadvise64_64, int, fd, __u32, offset_low, + __u32, offset_high, __u32, len_low, __u32, len_high, + int, advice) +{ + return ksys_fadvise64_64(fd, + (((u64)offset_high)<<32) | offset_low, + (((u64)len_high)<<32) | len_low, + advice); +} + +SYSCALL_DEFINE4(ia32_readahead, int, fd, unsigned int, off_lo, + unsigned int, off_hi, size_t, count) +{ + return ksys_readahead(fd, ((u64)off_hi << 32) | off_lo, count); +} + +SYSCALL_DEFINE6(ia32_sync_file_range, int, fd, unsigned int, off_low, + unsigned int, off_hi, unsigned int, n_low, + unsigned int, n_hi, int, flags) +{ + return ksys_sync_file_range(fd, + ((u64)off_hi << 32) | off_low, + ((u64)n_hi << 32) | n_low, flags); +} + +SYSCALL_DEFINE5(ia32_fadvise64, int, fd, unsigned int, offset_lo, + unsigned int, offset_hi, size_t, len, int, advice) +{ + return ksys_fadvise64_64(fd, ((u64)offset_hi << 32) | offset_lo, + len, advice); +} + +SYSCALL_DEFINE6(ia32_fallocate, int, fd, int, mode, + unsigned int, offset_lo, unsigned int, offset_hi, + unsigned int, len_lo, unsigned int, len_hi) +{ + return ksys_fallocate(fd, mode, ((u64)offset_hi << 32) | offset_lo, + ((u64)len_hi << 32) | len_lo); +} + +#ifdef CONFIG_IA32_EMULATION /* * Another set for IA32/LFS -- x86_64 struct stat is different due to * support for 64bit inode numbers. @@ -97,7 +157,7 @@ static int cp_stat64(struct stat64 __user *ubuf, struct kstat *stat) return 0; } -COMPAT_SYSCALL_DEFINE2(x86_stat64, const char __user *, filename, +COMPAT_SYSCALL_DEFINE2(ia32_stat64, const char __user *, filename, struct stat64 __user *, statbuf) { struct kstat stat; @@ -108,7 +168,7 @@ COMPAT_SYSCALL_DEFINE2(x86_stat64, const char __user *, filename, return ret; } -COMPAT_SYSCALL_DEFINE2(x86_lstat64, const char __user *, filename, +COMPAT_SYSCALL_DEFINE2(ia32_lstat64, const char __user *, filename, struct stat64 __user *, statbuf) { struct kstat stat; @@ -118,7 +178,7 @@ COMPAT_SYSCALL_DEFINE2(x86_lstat64, const char __user *, filename, return ret; } -COMPAT_SYSCALL_DEFINE2(x86_fstat64, unsigned int, fd, +COMPAT_SYSCALL_DEFINE2(ia32_fstat64, unsigned int, fd, struct stat64 __user *, statbuf) { struct kstat stat; @@ -128,7 +188,7 @@ COMPAT_SYSCALL_DEFINE2(x86_fstat64, unsigned int, fd, return ret; } -COMPAT_SYSCALL_DEFINE4(x86_fstatat, unsigned int, dfd, +COMPAT_SYSCALL_DEFINE4(ia32_fstatat64, unsigned int, dfd, const char __user *, filename, struct stat64 __user *, statbuf, int, flag) { @@ -156,7 +216,7 @@ struct mmap_arg_struct32 { unsigned int offset; }; -COMPAT_SYSCALL_DEFINE1(x86_mmap, struct mmap_arg_struct32 __user *, arg) +COMPAT_SYSCALL_DEFINE1(ia32_mmap, struct mmap_arg_struct32 __user *, arg) { struct mmap_arg_struct32 a; @@ -170,70 +230,10 @@ COMPAT_SYSCALL_DEFINE1(x86_mmap, struct mmap_arg_struct32 __user *, arg) a.offset>>PAGE_SHIFT); } -/* warning: next two assume little endian */ -COMPAT_SYSCALL_DEFINE5(x86_pread, unsigned int, fd, char __user *, ubuf, - u32, count, u32, poslo, u32, poshi) -{ - return ksys_pread64(fd, ubuf, count, - ((loff_t)AA(poshi) << 32) | AA(poslo)); -} - -COMPAT_SYSCALL_DEFINE5(x86_pwrite, unsigned int, fd, const char __user *, ubuf, - u32, count, u32, poslo, u32, poshi) -{ - return ksys_pwrite64(fd, ubuf, count, - ((loff_t)AA(poshi) << 32) | AA(poslo)); -} - - -/* - * Some system calls that need sign extended arguments. This could be - * done by a generic wrapper. - */ -COMPAT_SYSCALL_DEFINE6(x86_fadvise64_64, int, fd, __u32, offset_low, - __u32, offset_high, __u32, len_low, __u32, len_high, - int, advice) -{ - return ksys_fadvise64_64(fd, - (((u64)offset_high)<<32) | offset_low, - (((u64)len_high)<<32) | len_low, - advice); -} - -COMPAT_SYSCALL_DEFINE4(x86_readahead, int, fd, unsigned int, off_lo, - unsigned int, off_hi, size_t, count) -{ - return ksys_readahead(fd, ((u64)off_hi << 32) | off_lo, count); -} - -COMPAT_SYSCALL_DEFINE6(x86_sync_file_range, int, fd, unsigned int, off_low, - unsigned int, off_hi, unsigned int, n_low, - unsigned int, n_hi, int, flags) -{ - return ksys_sync_file_range(fd, - ((u64)off_hi << 32) | off_low, - ((u64)n_hi << 32) | n_low, flags); -} - -COMPAT_SYSCALL_DEFINE5(x86_fadvise64, int, fd, unsigned int, offset_lo, - unsigned int, offset_hi, size_t, len, int, advice) -{ - return ksys_fadvise64_64(fd, ((u64)offset_hi << 32) | offset_lo, - len, advice); -} - -COMPAT_SYSCALL_DEFINE6(x86_fallocate, int, fd, int, mode, - unsigned int, offset_lo, unsigned int, offset_hi, - unsigned int, len_lo, unsigned int, len_hi) -{ - return ksys_fallocate(fd, mode, ((u64)offset_hi << 32) | offset_lo, - ((u64)len_hi << 32) | len_lo); -} - /* * The 32-bit clone ABI is CONFIG_CLONE_BACKWARDS */ -COMPAT_SYSCALL_DEFINE5(x86_clone, unsigned long, clone_flags, +COMPAT_SYSCALL_DEFINE5(ia32_clone, unsigned long, clone_flags, unsigned long, newsp, int __user *, parent_tidptr, unsigned long, tls_val, int __user *, child_tidptr) { @@ -252,3 +252,4 @@ COMPAT_SYSCALL_DEFINE5(x86_clone, unsigned long, clone_flags, return _do_fork(&args); } +#endif /* CONFIG_IA32_EMULATION */ diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index ca3c11a17b5a..504fa5425bce 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -21,7 +21,6 @@ #include <asm/elf.h> #include <asm/ia32.h> -#include <asm/syscalls.h> /* * Align a virtual address to avoid aliasing in the I$ on AMD F15h. diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index d8673d8a779b..106e7f87f534 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -62,19 +62,16 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq0 = { - .handler = timer_interrupt, - .flags = IRQF_NOBALANCING | IRQF_IRQPOLL | IRQF_TIMER, - .name = "timer" -}; - static void __init setup_default_timer_irq(void) { + unsigned long flags = IRQF_NOBALANCING | IRQF_IRQPOLL | IRQF_TIMER; + /* - * Unconditionally register the legacy timer; even without legacy - * PIC/PIT we need this for the HPET0 in legacy replacement mode. + * Unconditionally register the legacy timer interrupt; even + * without legacy PIC/PIT we need this for the HPET0 in legacy + * replacement mode. */ - if (setup_irq(0, &irq0)) + if (request_irq(0, timer_interrupt, flags, "timer", NULL)) pr_info("Failed to register legacy timer interrupt\n"); } @@ -122,18 +119,12 @@ void __init time_init(void) */ void clocksource_arch_init(struct clocksource *cs) { - if (cs->archdata.vclock_mode == VCLOCK_NONE) + if (cs->vdso_clock_mode == VDSO_CLOCKMODE_NONE) return; - if (cs->archdata.vclock_mode > VCLOCK_MAX) { - pr_warn("clocksource %s registered with invalid vclock_mode %d. Disabling vclock.\n", - cs->name, cs->archdata.vclock_mode); - cs->archdata.vclock_mode = VCLOCK_NONE; - } - if (cs->mask != CLOCKSOURCE_MASK(64)) { - pr_warn("clocksource %s registered with invalid mask %016llx. Disabling vclock.\n", + pr_warn("clocksource %s registered with invalid mask %016llx for VDSO. Disabling VDSO support.\n", cs->name, cs->mask); - cs->archdata.vclock_mode = VCLOCK_NONE; + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; } } diff --git a/arch/x86/kernel/topology.c b/arch/x86/kernel/topology.c index be5bc2e47c71..b8810ebbc8ae 100644 --- a/arch/x86/kernel/topology.c +++ b/arch/x86/kernel/topology.c @@ -59,39 +59,29 @@ __setup("cpu0_hotplug", enable_cpu0_hotplug); */ int _debug_hotplug_cpu(int cpu, int action) { - struct device *dev = get_cpu_device(cpu); int ret; if (!cpu_is_hotpluggable(cpu)) return -EINVAL; - lock_device_hotplug(); - switch (action) { case 0: - ret = cpu_down(cpu); - if (!ret) { + ret = remove_cpu(cpu); + if (!ret) pr_info("DEBUG_HOTPLUG_CPU0: CPU %u is now offline\n", cpu); - dev->offline = true; - kobject_uevent(&dev->kobj, KOBJ_OFFLINE); - } else + else pr_debug("Can't offline CPU%d.\n", cpu); break; case 1: - ret = cpu_up(cpu); - if (!ret) { - dev->offline = false; - kobject_uevent(&dev->kobj, KOBJ_ONLINE); - } else { + ret = add_cpu(cpu); + if (ret) pr_debug("Can't online CPU%d.\n", cpu); - } + break; default: ret = -EINVAL; } - unlock_device_hotplug(); - return ret; } diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 6ef00eb6fbb9..d54cffdc7cac 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -46,6 +46,7 @@ #include <asm/traps.h> #include <asm/desc.h> #include <asm/fpu/internal.h> +#include <asm/cpu.h> #include <asm/cpu_entry_area.h> #include <asm/mce.h> #include <asm/fixmap.h> @@ -242,7 +243,6 @@ do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, { struct task_struct *tsk = current; - if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code)) return; @@ -288,9 +288,29 @@ DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, 0, NULL, "coprocessor segment overru DO_ERROR(X86_TRAP_TS, SIGSEGV, 0, NULL, "invalid TSS", invalid_TSS) DO_ERROR(X86_TRAP_NP, SIGBUS, 0, NULL, "segment not present", segment_not_present) DO_ERROR(X86_TRAP_SS, SIGBUS, 0, NULL, "stack segment", stack_segment) -DO_ERROR(X86_TRAP_AC, SIGBUS, BUS_ADRALN, NULL, "alignment check", alignment_check) #undef IP +dotraplinkage void do_alignment_check(struct pt_regs *regs, long error_code) +{ + char *str = "alignment check"; + + RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); + + if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP) + return; + + if (!user_mode(regs)) + die("Split lock detected\n", regs, error_code); + + local_irq_enable(); + + if (handle_user_split_lock(regs, error_code)) + return; + + do_trap(X86_TRAP_AC, SIGBUS, "alignment check", regs, + error_code, BUS_ADRALN, NULL); +} + #ifdef CONFIG_VMAP_STACK __visible void __noreturn handle_stack_overflow(const char *message, struct pt_regs *regs, @@ -572,14 +592,20 @@ dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code) return; /* - * Use ist_enter despite the fact that we don't use an IST stack. - * We can be called from a kprobe in non-CONTEXT_KERNEL kernel - * mode or even during context tracking state changes. + * Unlike any other non-IST entry, we can be called from a kprobe in + * non-CONTEXT_KERNEL kernel mode or even during context tracking + * state changes. Make sure that we wake up RCU even if we're coming + * from kernel code. * - * This means that we can't schedule. That's okay. + * This means that we can't schedule even if we came from a + * preemptible kernel context. That's okay. */ - ist_enter(regs); + if (!user_mode(regs)) { + rcu_nmi_enter(); + preempt_disable(); + } RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); + #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, SIGTRAP) == NOTIFY_STOP) @@ -600,7 +626,10 @@ dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code) cond_local_irq_disable(regs); exit: - ist_exit(regs); + if (!user_mode(regs)) { + preempt_enable_no_resched(); + rcu_nmi_exit(); + } } NOKPROBE_SYMBOL(do_int3); @@ -862,7 +891,25 @@ do_simd_coprocessor_error(struct pt_regs *regs, long error_code) dotraplinkage void do_spurious_interrupt_bug(struct pt_regs *regs, long error_code) { - cond_local_irq_enable(regs); + /* + * This addresses a Pentium Pro Erratum: + * + * PROBLEM: If the APIC subsystem is configured in mixed mode with + * Virtual Wire mode implemented through the local APIC, an + * interrupt vector of 0Fh (Intel reserved encoding) may be + * generated by the local APIC (Int 15). This vector may be + * generated upon receipt of a spurious interrupt (an interrupt + * which is removed before the system receives the INTA sequence) + * instead of the programmed 8259 spurious interrupt vector. + * + * IMPLICATION: The spurious interrupt vector programmed in the + * 8259 is normally handled by an operating system's spurious + * interrupt handler. However, a vector of 0Fh is unknown to some + * operating systems, which would crash if this erratum occurred. + * + * In theory this could be limited to 32bit, but the handler is not + * hurting and who knows which other CPUs suffer from this. + */ } dotraplinkage void diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 7e322e2daaf5..fdd4c1078632 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -477,7 +477,7 @@ static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin) * transition from one expected value to another with a fairly * high accuracy, and we didn't miss any events. We can thus * use the TSC value at the transitions to calculate a pretty - * good value for the TSC frequencty. + * good value for the TSC frequency. */ static inline int pit_verify_msb(unsigned char val) { @@ -1108,17 +1108,24 @@ static void tsc_cs_tick_stable(struct clocksource *cs) sched_clock_tick_stable(); } +static int tsc_cs_enable(struct clocksource *cs) +{ + vclocks_set_used(VDSO_CLOCKMODE_TSC); + return 0; +} + /* * .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc() */ static struct clocksource clocksource_tsc_early = { - .name = "tsc-early", - .rating = 299, - .read = read_tsc, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS | + .name = "tsc-early", + .rating = 299, + .read = read_tsc, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_MUST_VERIFY, - .archdata = { .vclock_mode = VCLOCK_TSC }, + .vdso_clock_mode = VDSO_CLOCKMODE_TSC, + .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable, .tick_stable = tsc_cs_tick_stable, @@ -1131,14 +1138,15 @@ static struct clocksource clocksource_tsc_early = { * been found good. */ static struct clocksource clocksource_tsc = { - .name = "tsc", - .rating = 300, - .read = read_tsc, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS | + .name = "tsc", + .rating = 300, + .read = read_tsc, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_MUST_VERIFY, - .archdata = { .vclock_mode = VCLOCK_TSC }, + .vdso_clock_mode = VDSO_CLOCKMODE_TSC, + .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable, .tick_stable = tsc_cs_tick_stable, diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c index e0cbe4f2af49..4fec6f3a1858 100644 --- a/arch/x86/kernel/tsc_msr.c +++ b/arch/x86/kernel/tsc_msr.c @@ -15,18 +15,46 @@ #include <asm/param.h> #include <asm/tsc.h> -#define MAX_NUM_FREQS 9 +#define MAX_NUM_FREQS 16 /* 4 bits to select the frequency */ + +/* + * The frequency numbers in the SDM are e.g. 83.3 MHz, which does not contain a + * lot of accuracy which leads to clock drift. As far as we know Bay Trail SoCs + * use a 25 MHz crystal and Cherry Trail uses a 19.2 MHz crystal, the crystal + * is the source clk for a root PLL which outputs 1600 and 100 MHz. It is + * unclear if the root PLL outputs are used directly by the CPU clock PLL or + * if there is another PLL in between. + * This does not matter though, we can model the chain of PLLs as a single PLL + * with a quotient equal to the quotients of all PLLs in the chain multiplied. + * So we can create a simplified model of the CPU clock setup using a reference + * clock of 100 MHz plus a quotient which gets us as close to the frequency + * from the SDM as possible. + * For the 83.3 MHz example from above this would give us 100 MHz * 5 / 6 = + * 83 and 1/3 MHz, which matches exactly what has been measured on actual hw. + */ +#define TSC_REFERENCE_KHZ 100000 + +struct muldiv { + u32 multiplier; + u32 divider; +}; /* * If MSR_PERF_STAT[31] is set, the maximum resolved bus ratio can be * read in MSR_PLATFORM_ID[12:8], otherwise in MSR_PERF_STAT[44:40]. * Unfortunately some Intel Atom SoCs aren't quite compliant to this, * so we need manually differentiate SoC families. This is what the - * field msr_plat does. + * field use_msr_plat does. */ struct freq_desc { - u8 msr_plat; /* 1: use MSR_PLATFORM_INFO, 0: MSR_IA32_PERF_STATUS */ + bool use_msr_plat; + struct muldiv muldiv[MAX_NUM_FREQS]; + /* + * Some CPU frequencies in the SDM do not map to known PLL freqs, in + * that case the muldiv array is empty and the freqs array is used. + */ u32 freqs[MAX_NUM_FREQS]; + u32 mask; }; /* @@ -35,41 +63,91 @@ struct freq_desc { * by MSR based on SDM. */ static const struct freq_desc freq_desc_pnw = { - 0, { 0, 0, 0, 0, 0, 99840, 0, 83200 } + .use_msr_plat = false, + .freqs = { 0, 0, 0, 0, 0, 99840, 0, 83200 }, + .mask = 0x07, }; static const struct freq_desc freq_desc_clv = { - 0, { 0, 133200, 0, 0, 0, 99840, 0, 83200 } + .use_msr_plat = false, + .freqs = { 0, 133200, 0, 0, 0, 99840, 0, 83200 }, + .mask = 0x07, }; +/* + * Bay Trail SDM MSR_FSB_FREQ frequencies simplified PLL model: + * 000: 100 * 5 / 6 = 83.3333 MHz + * 001: 100 * 1 / 1 = 100.0000 MHz + * 010: 100 * 4 / 3 = 133.3333 MHz + * 011: 100 * 7 / 6 = 116.6667 MHz + * 100: 100 * 4 / 5 = 80.0000 MHz + */ static const struct freq_desc freq_desc_byt = { - 1, { 83300, 100000, 133300, 116700, 80000, 0, 0, 0 } + .use_msr_plat = true, + .muldiv = { { 5, 6 }, { 1, 1 }, { 4, 3 }, { 7, 6 }, + { 4, 5 } }, + .mask = 0x07, }; +/* + * Cherry Trail SDM MSR_FSB_FREQ frequencies simplified PLL model: + * 0000: 100 * 5 / 6 = 83.3333 MHz + * 0001: 100 * 1 / 1 = 100.0000 MHz + * 0010: 100 * 4 / 3 = 133.3333 MHz + * 0011: 100 * 7 / 6 = 116.6667 MHz + * 0100: 100 * 4 / 5 = 80.0000 MHz + * 0101: 100 * 14 / 15 = 93.3333 MHz + * 0110: 100 * 9 / 10 = 90.0000 MHz + * 0111: 100 * 8 / 9 = 88.8889 MHz + * 1000: 100 * 7 / 8 = 87.5000 MHz + */ static const struct freq_desc freq_desc_cht = { - 1, { 83300, 100000, 133300, 116700, 80000, 93300, 90000, 88900, 87500 } + .use_msr_plat = true, + .muldiv = { { 5, 6 }, { 1, 1 }, { 4, 3 }, { 7, 6 }, + { 4, 5 }, { 14, 15 }, { 9, 10 }, { 8, 9 }, + { 7, 8 } }, + .mask = 0x0f, }; +/* + * Merriefield SDM MSR_FSB_FREQ frequencies simplified PLL model: + * 0001: 100 * 1 / 1 = 100.0000 MHz + * 0010: 100 * 4 / 3 = 133.3333 MHz + */ static const struct freq_desc freq_desc_tng = { - 1, { 0, 100000, 133300, 0, 0, 0, 0, 0 } + .use_msr_plat = true, + .muldiv = { { 0, 0 }, { 1, 1 }, { 4, 3 } }, + .mask = 0x07, }; +/* + * Moorefield SDM MSR_FSB_FREQ frequencies simplified PLL model: + * 0000: 100 * 5 / 6 = 83.3333 MHz + * 0001: 100 * 1 / 1 = 100.0000 MHz + * 0010: 100 * 4 / 3 = 133.3333 MHz + * 0011: 100 * 1 / 1 = 100.0000 MHz + */ static const struct freq_desc freq_desc_ann = { - 1, { 83300, 100000, 133300, 100000, 0, 0, 0, 0 } + .use_msr_plat = true, + .muldiv = { { 5, 6 }, { 1, 1 }, { 4, 3 }, { 1, 1 } }, + .mask = 0x0f, }; +/* 24 MHz crystal? : 24 * 13 / 4 = 78 MHz */ static const struct freq_desc freq_desc_lgm = { - 1, { 78000, 78000, 78000, 78000, 78000, 78000, 78000, 78000 } + .use_msr_plat = true, + .freqs = { 78000, 78000, 78000, 78000, 78000, 78000, 78000, 78000 }, + .mask = 0x0f, }; static const struct x86_cpu_id tsc_msr_cpu_ids[] = { - INTEL_CPU_FAM6(ATOM_SALTWELL_MID, freq_desc_pnw), - INTEL_CPU_FAM6(ATOM_SALTWELL_TABLET, freq_desc_clv), - INTEL_CPU_FAM6(ATOM_SILVERMONT, freq_desc_byt), - INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, freq_desc_tng), - INTEL_CPU_FAM6(ATOM_AIRMONT, freq_desc_cht), - INTEL_CPU_FAM6(ATOM_AIRMONT_MID, freq_desc_ann), - INTEL_CPU_FAM6(ATOM_AIRMONT_NP, freq_desc_lgm), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL_MID, &freq_desc_pnw), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL_TABLET,&freq_desc_clv), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &freq_desc_byt), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &freq_desc_tng), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &freq_desc_cht), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT_MID, &freq_desc_ann), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT_NP, &freq_desc_lgm), {} }; @@ -81,17 +159,19 @@ static const struct x86_cpu_id tsc_msr_cpu_ids[] = { */ unsigned long cpu_khz_from_msr(void) { - u32 lo, hi, ratio, freq; + u32 lo, hi, ratio, freq, tscref; const struct freq_desc *freq_desc; const struct x86_cpu_id *id; + const struct muldiv *md; unsigned long res; + int index; id = x86_match_cpu(tsc_msr_cpu_ids); if (!id) return 0; freq_desc = (struct freq_desc *)id->driver_data; - if (freq_desc->msr_plat) { + if (freq_desc->use_msr_plat) { rdmsr(MSR_PLATFORM_INFO, lo, hi); ratio = (lo >> 8) & 0xff; } else { @@ -101,12 +181,28 @@ unsigned long cpu_khz_from_msr(void) /* Get FSB FREQ ID */ rdmsr(MSR_FSB_FREQ, lo, hi); + index = lo & freq_desc->mask; + md = &freq_desc->muldiv[index]; - /* Map CPU reference clock freq ID(0-7) to CPU reference clock freq(KHz) */ - freq = freq_desc->freqs[lo & 0x7]; + /* + * Note this also catches cases where the index points to an unpopulated + * part of muldiv, in that case the else will set freq and res to 0. + */ + if (md->divider) { + tscref = TSC_REFERENCE_KHZ * md->multiplier; + freq = DIV_ROUND_CLOSEST(tscref, md->divider); + /* + * Multiplying by ratio before the division has better + * accuracy than just calculating freq * ratio. + */ + res = DIV_ROUND_CLOSEST(tscref * ratio, md->divider); + } else { + freq = freq_desc->freqs[index]; + res = freq * ratio; + } - /* TSC frequency = maximum resolved freq * maximum resolved bus ratio */ - res = freq * ratio; + if (freq == 0) + pr_err("Error MSR_FSB_FREQ index %d is unknown\n", index); #ifdef CONFIG_X86_LOCAL_APIC lapic_timer_period = (freq * 1000) / HZ; diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index 32a818764e03..3d3c761eb74a 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -295,7 +295,7 @@ static cycles_t check_tsc_warp(unsigned int timeout) * But as the TSC is per-logical CPU and can potentially be modified wrongly * by the bios, TSC sync test for smaller duration should be able * to catch such errors. Also this will catch the condition where all the - * cores in the socket doesn't get reset at the same time. + * cores in the socket don't get reset at the same time. */ static inline unsigned int loop_timeout(int cpu) { diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c index 91d55454e702..47a8676c7395 100644 --- a/arch/x86/kernel/vm86_32.c +++ b/arch/x86/kernel/vm86_32.c @@ -98,7 +98,6 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) struct task_struct *tsk = current; struct vm86plus_struct __user *user; struct vm86 *vm86 = current->thread.vm86; - long err = 0; /* * This gets called from entry.S with interrupts disabled, but @@ -114,37 +113,30 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) set_flags(regs->pt.flags, VEFLAGS, X86_EFLAGS_VIF | vm86->veflags_mask); user = vm86->user_vm86; - if (!access_ok(user, vm86->vm86plus.is_vm86pus ? + if (!user_access_begin(user, vm86->vm86plus.is_vm86pus ? sizeof(struct vm86plus_struct) : - sizeof(struct vm86_struct))) { - pr_alert("could not access userspace vm86 info\n"); - do_exit(SIGSEGV); - } - - put_user_try { - put_user_ex(regs->pt.bx, &user->regs.ebx); - put_user_ex(regs->pt.cx, &user->regs.ecx); - put_user_ex(regs->pt.dx, &user->regs.edx); - put_user_ex(regs->pt.si, &user->regs.esi); - put_user_ex(regs->pt.di, &user->regs.edi); - put_user_ex(regs->pt.bp, &user->regs.ebp); - put_user_ex(regs->pt.ax, &user->regs.eax); - put_user_ex(regs->pt.ip, &user->regs.eip); - put_user_ex(regs->pt.cs, &user->regs.cs); - put_user_ex(regs->pt.flags, &user->regs.eflags); - put_user_ex(regs->pt.sp, &user->regs.esp); - put_user_ex(regs->pt.ss, &user->regs.ss); - put_user_ex(regs->es, &user->regs.es); - put_user_ex(regs->ds, &user->regs.ds); - put_user_ex(regs->fs, &user->regs.fs); - put_user_ex(regs->gs, &user->regs.gs); - - put_user_ex(vm86->screen_bitmap, &user->screen_bitmap); - } put_user_catch(err); - if (err) { - pr_alert("could not access userspace vm86 info\n"); - do_exit(SIGSEGV); - } + sizeof(struct vm86_struct))) + goto Efault; + + unsafe_put_user(regs->pt.bx, &user->regs.ebx, Efault_end); + unsafe_put_user(regs->pt.cx, &user->regs.ecx, Efault_end); + unsafe_put_user(regs->pt.dx, &user->regs.edx, Efault_end); + unsafe_put_user(regs->pt.si, &user->regs.esi, Efault_end); + unsafe_put_user(regs->pt.di, &user->regs.edi, Efault_end); + unsafe_put_user(regs->pt.bp, &user->regs.ebp, Efault_end); + unsafe_put_user(regs->pt.ax, &user->regs.eax, Efault_end); + unsafe_put_user(regs->pt.ip, &user->regs.eip, Efault_end); + unsafe_put_user(regs->pt.cs, &user->regs.cs, Efault_end); + unsafe_put_user(regs->pt.flags, &user->regs.eflags, Efault_end); + unsafe_put_user(regs->pt.sp, &user->regs.esp, Efault_end); + unsafe_put_user(regs->pt.ss, &user->regs.ss, Efault_end); + unsafe_put_user(regs->es, &user->regs.es, Efault_end); + unsafe_put_user(regs->ds, &user->regs.ds, Efault_end); + unsafe_put_user(regs->fs, &user->regs.fs, Efault_end); + unsafe_put_user(regs->gs, &user->regs.gs, Efault_end); + unsafe_put_user(vm86->screen_bitmap, &user->screen_bitmap, Efault_end); + + user_access_end(); preempt_disable(); tsk->thread.sp0 = vm86->saved_sp0; @@ -159,6 +151,13 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) lazy_load_gs(vm86->regs32.gs); regs->pt.ax = retval; + return; + +Efault_end: + user_access_end(); +Efault: + pr_alert("could not access userspace vm86 info\n"); + do_exit(SIGSEGV); } static void mark_screen_rdonly(struct mm_struct *mm) @@ -243,6 +242,7 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus) struct kernel_vm86_regs vm86regs; struct pt_regs *regs = current_pt_regs(); unsigned long err = 0; + struct vm86_struct v; err = security_mmap_addr(0); if (err) { @@ -278,39 +278,32 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus) if (vm86->saved_sp0) return -EPERM; - if (!access_ok(user_vm86, plus ? - sizeof(struct vm86_struct) : - sizeof(struct vm86plus_struct))) + if (copy_from_user(&v, user_vm86, + offsetof(struct vm86_struct, int_revectored))) return -EFAULT; memset(&vm86regs, 0, sizeof(vm86regs)); - get_user_try { - unsigned short seg; - get_user_ex(vm86regs.pt.bx, &user_vm86->regs.ebx); - get_user_ex(vm86regs.pt.cx, &user_vm86->regs.ecx); - get_user_ex(vm86regs.pt.dx, &user_vm86->regs.edx); - get_user_ex(vm86regs.pt.si, &user_vm86->regs.esi); - get_user_ex(vm86regs.pt.di, &user_vm86->regs.edi); - get_user_ex(vm86regs.pt.bp, &user_vm86->regs.ebp); - get_user_ex(vm86regs.pt.ax, &user_vm86->regs.eax); - get_user_ex(vm86regs.pt.ip, &user_vm86->regs.eip); - get_user_ex(seg, &user_vm86->regs.cs); - vm86regs.pt.cs = seg; - get_user_ex(vm86regs.pt.flags, &user_vm86->regs.eflags); - get_user_ex(vm86regs.pt.sp, &user_vm86->regs.esp); - get_user_ex(seg, &user_vm86->regs.ss); - vm86regs.pt.ss = seg; - get_user_ex(vm86regs.es, &user_vm86->regs.es); - get_user_ex(vm86regs.ds, &user_vm86->regs.ds); - get_user_ex(vm86regs.fs, &user_vm86->regs.fs); - get_user_ex(vm86regs.gs, &user_vm86->regs.gs); - - get_user_ex(vm86->flags, &user_vm86->flags); - get_user_ex(vm86->screen_bitmap, &user_vm86->screen_bitmap); - get_user_ex(vm86->cpu_type, &user_vm86->cpu_type); - } get_user_catch(err); - if (err) - return err; + + vm86regs.pt.bx = v.regs.ebx; + vm86regs.pt.cx = v.regs.ecx; + vm86regs.pt.dx = v.regs.edx; + vm86regs.pt.si = v.regs.esi; + vm86regs.pt.di = v.regs.edi; + vm86regs.pt.bp = v.regs.ebp; + vm86regs.pt.ax = v.regs.eax; + vm86regs.pt.ip = v.regs.eip; + vm86regs.pt.cs = v.regs.cs; + vm86regs.pt.flags = v.regs.eflags; + vm86regs.pt.sp = v.regs.esp; + vm86regs.pt.ss = v.regs.ss; + vm86regs.es = v.regs.es; + vm86regs.ds = v.regs.ds; + vm86regs.fs = v.regs.fs; + vm86regs.gs = v.regs.gs; + + vm86->flags = v.flags; + vm86->screen_bitmap = v.screen_bitmap; + vm86->cpu_type = v.cpu_type; if (copy_from_user(&vm86->int_revectored, &user_vm86->int_revectored, diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index e3296aa028fe..1bf7e312361f 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -21,6 +21,7 @@ #define LOAD_OFFSET __START_KERNEL_map #endif +#define RUNTIME_DISCARD_EXIT #define EMITS_PT_NOTE #define RO_EXCEPTION_TABLE_ALIGN 16 @@ -313,8 +314,8 @@ SECTIONS . = ALIGN(8); /* - * .exit.text is discard at runtime, not link time, to deal with - * references from .altinstructions and .eh_frame + * .exit.text is discarded at runtime, not link time, to deal with + * references from .altinstructions */ .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) { EXIT_TEXT @@ -412,9 +413,6 @@ SECTIONS DWARF_DEBUG DISCARDS - /DISCARD/ : { - *(.eh_frame) - } } diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index e3099c642fec..7356a56e6282 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1445,6 +1445,8 @@ static void limit_periodic_timer_frequency(struct kvm_lapic *apic) } } +static void cancel_hv_timer(struct kvm_lapic *apic); + static void apic_update_lvtt(struct kvm_lapic *apic) { u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & @@ -1454,6 +1456,10 @@ static void apic_update_lvtt(struct kvm_lapic *apic) if (apic_lvtt_tscdeadline(apic) != (timer_mode == APIC_LVT_TIMER_TSCDEADLINE)) { hrtimer_cancel(&apic->lapic_timer.timer); + preempt_disable(); + if (apic->lapic_timer.hv_timer_in_use) + cancel_hv_timer(apic); + preempt_enable(); kvm_lapic_set_reg(apic, APIC_TMICT, 0); apic->lapic_timer.period = 0; apic->lapic_timer.tscdeadline = 0; @@ -1715,7 +1721,7 @@ static void start_sw_period(struct kvm_lapic *apic) hrtimer_start(&apic->lapic_timer.timer, apic->lapic_timer.target_expiration, - HRTIMER_MODE_ABS); + HRTIMER_MODE_ABS_HARD); } bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index e4c8a4cbf407..21a3320f166a 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -400,7 +400,7 @@ retry_walk: goto error; ptep_user = (pt_element_t __user *)((void *)host_addr + offset); - if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte)))) + if (unlikely(__get_user(pte, ptep_user))) goto error; walker->ptep_user[walker->level - 1] = ptep_user; diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 91000501756e..216364cb65a3 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -48,6 +48,7 @@ #include <asm/kvm_para.h> #include <asm/irq_remapping.h> #include <asm/spec-ctrl.h> +#include <asm/cpu_device_id.h> #include <asm/virtext.h> #include "trace.h" @@ -59,7 +60,7 @@ MODULE_LICENSE("GPL"); #ifdef MODULE static const struct x86_cpu_id svm_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_SVM), + X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); @@ -1933,14 +1934,6 @@ static void sev_clflush_pages(struct page *pages[], unsigned long npages) static void __unregister_enc_region_locked(struct kvm *kvm, struct enc_region *region) { - /* - * The guest may change the memory encryption attribute from C=0 -> C=1 - * or vice versa for this memory range. Lets make sure caches are - * flushed to ensure that guest data gets written into memory with - * correct C-bit. - */ - sev_clflush_pages(region->pages, region->npages); - sev_unpin_memory(kvm, region->pages, region->npages); list_del(®ion->list); kfree(region); @@ -1971,6 +1964,13 @@ static void sev_vm_destroy(struct kvm *kvm) mutex_lock(&kvm->lock); /* + * Ensure that all guest tagged cache entries are flushed before + * releasing the pages back to the system for use. CLFLUSH will + * not do this, so issue a WBINVD. + */ + wbinvd_on_all_cpus(); + + /* * if userspace was terminated before unregistering the memory regions * then lets unpin all the registered memory. */ @@ -7158,6 +7158,9 @@ static int svm_mem_enc_op(struct kvm *kvm, void __user *argp) if (!svm_sev_enabled()) return -ENOTTY; + if (!argp) + return 0; + if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd))) return -EFAULT; @@ -7285,6 +7288,13 @@ static int svm_unregister_enc_region(struct kvm *kvm, goto failed; } + /* + * Ensure that all guest tagged cache entries are flushed before + * releasing the pages back to the system for use. CLFLUSH will + * not do this, so issue a WBINVD. + */ + wbinvd_on_all_cpus(); + __unregister_enc_region_locked(kvm, region); mutex_unlock(&kvm->lock); diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index f194dd058470..cef5a344fedb 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -815,8 +815,8 @@ TRACE_EVENT(kvm_write_tsc_offset, #ifdef CONFIG_X86_64 #define host_clocks \ - {VCLOCK_NONE, "none"}, \ - {VCLOCK_TSC, "tsc"} \ + {VDSO_CLOCKMODE_NONE, "none"}, \ + {VDSO_CLOCKMODE_TSC, "tsc"} \ TRACE_EVENT(kvm_update_master_clock, TP_PROTO(bool use_master_clock, unsigned int host_clock, bool offset_matched), diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 26f8f31563e9..458e684dfbdc 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -31,6 +31,7 @@ #include <asm/apic.h> #include <asm/asm.h> #include <asm/cpu.h> +#include <asm/cpu_device_id.h> #include <asm/debugreg.h> #include <asm/desc.h> #include <asm/fpu/internal.h> @@ -41,6 +42,7 @@ #include <asm/mce.h> #include <asm/mmu_context.h> #include <asm/mshyperv.h> +#include <asm/mwait.h> #include <asm/spec-ctrl.h> #include <asm/virtext.h> #include <asm/vmx.h> @@ -66,7 +68,7 @@ MODULE_LICENSE("GPL"); #ifdef MODULE static const struct x86_cpu_id vmx_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_VMX), + X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); @@ -6287,7 +6289,7 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) #endif ASM_CALL_CONSTRAINT : - THUNK_TARGET(entry), + [thunk_target]"r"(entry), [ss]"i"(__KERNEL_DS), [cs]"i"(__KERNEL_CS) ); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index e64da06c7009..0695ea177e22 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -14,8 +14,6 @@ extern const u32 vmx_msr_index[]; extern u64 host_efer; -extern u32 get_umwait_control_msr(void); - #define MSR_TYPE_R 1 #define MSR_TYPE_W 2 #define MSR_TYPE_RW 3 diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3156e25b0774..bf8564d73fc3 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1554,7 +1554,10 @@ EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr); */ static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data) { - if (lapic_in_kernel(vcpu) && apic_x2apic_mode(vcpu->arch.apic) && + if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic)) + return 1; + + if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) && ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) && ((data & APIC_MODE_MASK) == APIC_DM_FIXED)) { @@ -1631,7 +1634,7 @@ static void update_pvclock_gtod(struct timekeeper *tk) write_seqcount_begin(&vdata->seq); /* copy pvclock gtod data */ - vdata->clock.vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; + vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode; vdata->clock.cycle_last = tk->tkr_mono.cycle_last; vdata->clock.mask = tk->tkr_mono.mask; vdata->clock.mult = tk->tkr_mono.mult; @@ -1639,7 +1642,7 @@ static void update_pvclock_gtod(struct timekeeper *tk) vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec; vdata->clock.offset = tk->tkr_mono.base; - vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->archdata.vclock_mode; + vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode; vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last; vdata->raw_clock.mask = tk->tkr_raw.mask; vdata->raw_clock.mult = tk->tkr_raw.mult; @@ -1840,7 +1843,7 @@ static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) static inline int gtod_is_based_on_tsc(int mode) { - return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK; + return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK; } static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) @@ -1933,7 +1936,7 @@ static inline bool kvm_check_tsc_unstable(void) * TSC is marked unstable when we're running on Hyper-V, * 'TSC page' clocksource is good. */ - if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK) + if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK) return false; #endif return check_tsc_unstable(); @@ -2088,30 +2091,30 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, u64 tsc_pg_val; switch (clock->vclock_mode) { - case VCLOCK_HVCLOCK: + case VDSO_CLOCKMODE_HVCLOCK: tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(), tsc_timestamp); if (tsc_pg_val != U64_MAX) { /* TSC page valid */ - *mode = VCLOCK_HVCLOCK; + *mode = VDSO_CLOCKMODE_HVCLOCK; v = (tsc_pg_val - clock->cycle_last) & clock->mask; } else { /* TSC page invalid */ - *mode = VCLOCK_NONE; + *mode = VDSO_CLOCKMODE_NONE; } break; - case VCLOCK_TSC: - *mode = VCLOCK_TSC; + case VDSO_CLOCKMODE_TSC: + *mode = VDSO_CLOCKMODE_TSC; *tsc_timestamp = read_tsc(); v = (*tsc_timestamp - clock->cycle_last) & clock->mask; break; default: - *mode = VCLOCK_NONE; + *mode = VDSO_CLOCKMODE_NONE; } - if (*mode == VCLOCK_NONE) + if (*mode == VDSO_CLOCKMODE_NONE) *tsc_timestamp = v = 0; return v * clock->mult; @@ -2444,7 +2447,6 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) vcpu->hv_clock.tsc_timestamp = tsc_timestamp; vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; vcpu->last_guest_tsc = tsc_timestamp; - WARN_ON((s64)vcpu->hv_clock.system_time < 0); /* If the host uses TSC clocksource, then it is stable */ pvclock_flags = 0; diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index 53adc1762ec0..ec31f5b60323 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -366,7 +366,7 @@ AVXcode: 1 1b: BNDCN Gv,Ev (F2) | BNDMOV Ev,Gv (66) | BNDMK Gv,Ev (F3) | BNDSTX Ev,Gv 1c: Grp20 (1A),(1C) 1d: -1e: +1e: Grp21 (1A) 1f: NOP Ev # 0x0f 0x20-0x2f 20: MOV Rd,Cd @@ -803,8 +803,8 @@ f0: MOVBE Gy,My | MOVBE Gw,Mw (66) | CRC32 Gd,Eb (F2) | CRC32 Gd,Eb (66&F2) f1: MOVBE My,Gy | MOVBE Mw,Gw (66) | CRC32 Gd,Ey (F2) | CRC32 Gd,Ew (66&F2) f2: ANDN Gy,By,Ey (v) f3: Grp17 (1A) -f5: BZHI Gy,Ey,By (v) | PEXT Gy,By,Ey (F3),(v) | PDEP Gy,By,Ey (F2),(v) -f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) +f5: BZHI Gy,Ey,By (v) | PEXT Gy,By,Ey (F3),(v) | PDEP Gy,By,Ey (F2),(v) | WRUSSD/Q My,Gy (66) +f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSSD/Q My,Gy f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) f8: MOVDIR64B Gv,Mdqq (66) | ENQCMD Gv,Mdqq (F2) | ENQCMDS Gv,Mdqq (F3) f9: MOVDIRI My,Gy @@ -970,7 +970,7 @@ GrpTable: Grp7 2: LGDT Ms | XGETBV (000),(11B) | XSETBV (001),(11B) | VMFUNC (100),(11B) | XEND (101)(11B) | XTEST (110)(11B) | ENCLU (111),(11B) 3: LIDT Ms 4: SMSW Mw/Rv -5: rdpkru (110),(11B) | wrpkru (111),(11B) +5: rdpkru (110),(11B) | wrpkru (111),(11B) | SAVEPREVSSP (F3),(010),(11B) | RSTORSSP Mq (F3) | SETSSBSY (F3),(000),(11B) 6: LMSW Ew 7: INVLPG Mb | SWAPGS (o64),(000),(11B) | RDTSCP (001),(11B) EndTable @@ -1041,8 +1041,8 @@ GrpTable: Grp15 2: vldmxcsr Md (v1) | WRFSBASE Ry (F3),(11B) 3: vstmxcsr Md (v1) | WRGSBASE Ry (F3),(11B) 4: XSAVE | ptwrite Ey (F3),(11B) -5: XRSTOR | lfence (11B) -6: XSAVEOPT | clwb (66) | mfence (11B) | TPAUSE Rd (66),(11B) | UMONITOR Rv (F3),(11B) | UMWAIT Rd (F2),(11B) +5: XRSTOR | lfence (11B) | INCSSPD/Q Ry (F3),(11B) +6: XSAVEOPT | clwb (66) | mfence (11B) | TPAUSE Rd (66),(11B) | UMONITOR Rv (F3),(11B) | UMWAIT Rd (F2),(11B) | CLRSSBSY Mq (F3) 7: clflush | clflushopt (66) | sfence (11B) EndTable @@ -1077,6 +1077,11 @@ GrpTable: Grp20 0: cldemote Mb EndTable +GrpTable: Grp21 +1: RDSSPD/Q Ry (F3),(11B) +7: ENDBR64 (F3),(010),(11B) | ENDBR32 (F3),(011),(11B) +EndTable + # AMD's Prefetch Group GrpTable: GrpP 0: PREFETCH diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 30bb0bd3b1b8..b991aa4bdfae 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -80,18 +80,6 @@ __visible bool ex_handler_uaccess(const struct exception_table_entry *fixup, } EXPORT_SYMBOL(ex_handler_uaccess); -__visible bool ex_handler_ext(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) -{ - /* Special hack for uaccess_err */ - current->thread.uaccess_err = 1; - regs->ip = ex_fixup_addr(fixup); - return true; -} -EXPORT_SYMBOL(ex_handler_ext); - __visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, struct pt_regs *regs, int trapnr, unsigned long error_code, diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index 23df4885bbed..8ae0272c1c51 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -788,44 +788,6 @@ void __init mem_init(void) x86_init.hyper.init_after_bootmem(); mem_init_print_info(NULL); - printk(KERN_INFO "virtual kernel memory layout:\n" - " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" - " cpu_entry : 0x%08lx - 0x%08lx (%4ld kB)\n" -#ifdef CONFIG_HIGHMEM - " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n" -#endif - " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" - " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" - " .init : 0x%08lx - 0x%08lx (%4ld kB)\n" - " .data : 0x%08lx - 0x%08lx (%4ld kB)\n" - " .text : 0x%08lx - 0x%08lx (%4ld kB)\n", - FIXADDR_START, FIXADDR_TOP, - (FIXADDR_TOP - FIXADDR_START) >> 10, - - CPU_ENTRY_AREA_BASE, - CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE, - CPU_ENTRY_AREA_MAP_SIZE >> 10, - -#ifdef CONFIG_HIGHMEM - PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, - (LAST_PKMAP*PAGE_SIZE) >> 10, -#endif - - VMALLOC_START, VMALLOC_END, - (VMALLOC_END - VMALLOC_START) >> 20, - - (unsigned long)__va(0), (unsigned long)high_memory, - ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20, - - (unsigned long)&__init_begin, (unsigned long)&__init_end, - ((unsigned long)&__init_end - - (unsigned long)&__init_begin) >> 10, - - (unsigned long)&_etext, (unsigned long)&_edata, - ((unsigned long)&_edata - (unsigned long)&_etext) >> 10, - - (unsigned long)&_text, (unsigned long)&_etext, - ((unsigned long)&_etext - (unsigned long)&_text) >> 10); /* * Check boundaries twice: Some fundamental inconsistencies can diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index abbdecb75fad..0a14711d3a93 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -818,8 +818,7 @@ void __init paging_init(void) * will not set it back. */ node_clear_state(0, N_MEMORY); - if (N_MEMORY != N_NORMAL_MEMORY) - node_clear_state(0, N_NORMAL_MEMORY); + node_clear_state(0, N_NORMAL_MEMORY); zone_sizes_init(); } diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index 935a91e1fd77..18c637c0dc6f 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c @@ -115,6 +115,9 @@ static void __ioremap_check_other(resource_size_t addr, struct ioremap_desc *des if (!sev_active()) return; + if (!IS_ENABLED(CONFIG_EFI)) + return; + if (efi_mem_type(addr) == EFI_RUNTIME_SERVICES_DATA) desc->flags |= IORES_MAP_ENCRYPTED; } diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c index 49d7814b59a9..9994353fb75d 100644 --- a/arch/x86/mm/kmmio.c +++ b/arch/x86/mm/kmmio.c @@ -260,7 +260,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) goto no_kmmio; } - ctx = &get_cpu_var(kmmio_ctx); + ctx = this_cpu_ptr(&kmmio_ctx); if (ctx->active) { if (page_base == ctx->addr) { /* @@ -285,7 +285,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) pr_emerg("previous hit was at 0x%08lx.\n", ctx->addr); disarm_kmmio_fault_page(faultpage); } - goto no_kmmio_ctx; + goto no_kmmio; } ctx->active++; @@ -314,11 +314,8 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) * the user should drop to single cpu before tracing. */ - put_cpu_var(kmmio_ctx); return 1; /* fault handled */ -no_kmmio_ctx: - put_cpu_var(kmmio_ctx); no_kmmio: rcu_read_unlock(); preempt_enable_no_resched(); @@ -333,7 +330,7 @@ no_kmmio: static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs) { int ret = 0; - struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx); + struct kmmio_context *ctx = this_cpu_ptr(&kmmio_ctx); if (!ctx->active) { /* @@ -371,7 +368,6 @@ static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs) if (!(regs->flags & X86_EFLAGS_TF)) ret = 1; out: - put_cpu_var(kmmio_ctx); return ret; } diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c index 673de6063345..109325d77b3e 100644 --- a/arch/x86/mm/mmio-mod.c +++ b/arch/x86/mm/mmio-mod.c @@ -386,7 +386,7 @@ static void enter_uniprocessor(void) put_online_cpus(); for_each_cpu(cpu, downed_cpus) { - err = cpu_down(cpu); + err = remove_cpu(cpu); if (!err) pr_info("CPU%d is down.\n", cpu); else @@ -406,7 +406,7 @@ static void leave_uniprocessor(void) return; pr_notice("Re-enabling CPUs...\n"); for_each_cpu(cpu, downed_cpus) { - err = cpu_up(cpu); + err = add_cpu(cpu); if (!err) pr_info("enabled CPU%d.\n", cpu); else diff --git a/arch/x86/mm/numa_emulation.c b/arch/x86/mm/numa_emulation.c index 7f1d2034df1e..c5174b4e318b 100644 --- a/arch/x86/mm/numa_emulation.c +++ b/arch/x86/mm/numa_emulation.c @@ -324,7 +324,7 @@ static int __init split_nodes_size_interleave(struct numa_meminfo *ei, 0, NULL, NUMA_NO_NODE); } -int __init setup_emu2phys_nid(int *dfl_phys_nid) +static int __init setup_emu2phys_nid(int *dfl_phys_nid) { int i, max_emu_nid = 0; diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index c4aedd00c1ba..6d5424069e2b 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -15,6 +15,7 @@ #include <linux/gfp.h> #include <linux/pci.h> #include <linux/vmalloc.h> +#include <linux/libnvdimm.h> #include <asm/e820/api.h> #include <asm/processor.h> @@ -304,11 +305,13 @@ void clflush_cache_range(void *vaddr, unsigned int size) } EXPORT_SYMBOL_GPL(clflush_cache_range); +#ifdef CONFIG_ARCH_HAS_PMEM_API void arch_invalidate_pmem(void *addr, size_t size) { clflush_cache_range(addr, size); } EXPORT_SYMBOL_GPL(arch_invalidate_pmem); +#endif static void __cpa_flush_all(void *arg) { diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index 44a9f068eee0..843aa10a4cb6 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -39,6 +39,7 @@ #include <asm/tlbflush.h> #include <asm/desc.h> #include <asm/sections.h> +#include <asm/set_memory.h> #undef pr_fmt #define pr_fmt(fmt) "Kernel/User page tables isolation: " fmt @@ -555,13 +556,6 @@ static inline bool pti_kernel_image_global_ok(void) } /* - * This is the only user for these and it is not arch-generic - * like the other set_memory.h functions. Just extern them. - */ -extern int set_memory_nonglobal(unsigned long addr, int numpages); -extern int set_memory_global(unsigned long addr, int numpages); - -/* * For some configurations, map all of kernel text into the user page * tables. This reduces TLB misses, especially on non-PCID systems. */ diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index 393d251798c0..4d2a7a764602 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -2039,10 +2039,12 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, } /* and dreg_lo,sreg_lo */ EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo)); - /* and dreg_hi,sreg_hi */ - EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); - /* or dreg_lo,dreg_hi */ - EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); + if (is_jmp64) { + /* and dreg_hi,sreg_hi */ + EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); + /* or dreg_lo,dreg_hi */ + EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); + } goto emit_cond_jmp; } case BPF_JMP | BPF_JSET | BPF_K: diff --git a/arch/x86/platform/atom/punit_atom_debug.c b/arch/x86/platform/atom/punit_atom_debug.c index ee6b0780bea1..f8ed5f66cd20 100644 --- a/arch/x86/platform/atom/punit_atom_debug.c +++ b/arch/x86/platform/atom/punit_atom_debug.c @@ -117,17 +117,16 @@ static void punit_dbgfs_unregister(void) debugfs_remove_recursive(punit_dbg_file); } -#define ICPU(model, drv_data) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT,\ - (kernel_ulong_t)&drv_data } +#define X86_MATCH(model, data) \ + X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \ + X86_FEATURE_MWAIT, data) static const struct x86_cpu_id intel_punit_cpu_ids[] = { - ICPU(INTEL_FAM6_ATOM_SILVERMONT, punit_device_byt), - ICPU(INTEL_FAM6_ATOM_SILVERMONT_MID, punit_device_tng), - ICPU(INTEL_FAM6_ATOM_AIRMONT, punit_device_cht), + X86_MATCH(ATOM_SILVERMONT, &punit_device_byt), + X86_MATCH(ATOM_SILVERMONT_MID, &punit_device_tng), + X86_MATCH(ATOM_AIRMONT, &punit_device_cht), {} }; - MODULE_DEVICE_TABLE(x86cpu, intel_punit_cpu_ids); static int __init punit_atom_debug_init(void) diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index ae923ee8e2b4..1aae5302501d 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -54,10 +54,16 @@ #include <asm/x86_init.h> #include <asm/uv/uv.h> -static efi_system_table_t efi_systab __initdata; -static u64 efi_systab_phys __initdata; +static unsigned long efi_systab_phys __initdata; +static unsigned long prop_phys = EFI_INVALID_TABLE_ADDR; +static unsigned long uga_phys = EFI_INVALID_TABLE_ADDR; +static unsigned long efi_runtime, efi_nr_tables; -static efi_config_table_type_t arch_tables[] __initdata = { +unsigned long efi_fw_vendor, efi_config_table; + +static const efi_config_table_type_t arch_tables[] __initconst = { + {EFI_PROPERTIES_TABLE_GUID, "PROP", &prop_phys}, + {UGA_IO_PROTOCOL_GUID, "UGA", &uga_phys}, #ifdef CONFIG_X86_UV {UV_SYSTEM_TABLE_GUID, "UVsystab", &uv_systab_phys}, #endif @@ -65,26 +71,26 @@ static efi_config_table_type_t arch_tables[] __initdata = { }; static const unsigned long * const efi_tables[] = { - &efi.mps, &efi.acpi, &efi.acpi20, &efi.smbios, &efi.smbios3, - &efi.boot_info, - &efi.hcdp, - &efi.uga, + &uga_phys, #ifdef CONFIG_X86_UV &uv_systab_phys, #endif - &efi.fw_vendor, - &efi.runtime, - &efi.config_table, + &efi_fw_vendor, + &efi_runtime, + &efi_config_table, &efi.esrt, - &efi.properties_table, - &efi.mem_attr_table, + &prop_phys, + &efi_mem_attr_table, #ifdef CONFIG_EFI_RCI2_TABLE &rci2_table_phys, #endif + &efi.tpm_log, + &efi.tpm_final_log, + &efi_rng_seed, }; u64 efi_setup; /* efi setup_data physical address */ @@ -214,16 +220,13 @@ int __init efi_memblock_x86_reserve_range(void) if (efi_enabled(EFI_PARAVIRT)) return 0; -#ifdef CONFIG_X86_32 - /* Can't handle data above 4GB at this time */ - if (e->efi_memmap_hi) { + /* Can't handle firmware tables above 4GB on i386 */ + if (IS_ENABLED(CONFIG_X86_32) && e->efi_memmap_hi > 0) { pr_err("Memory map is above 4GB, disabling EFI.\n"); return -EINVAL; } - pmap = e->efi_memmap; -#else - pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32)); -#endif + pmap = (phys_addr_t)(e->efi_memmap | ((u64)e->efi_memmap_hi << 32)); + data.phys_map = pmap; data.size = e->efi_memmap_size; data.desc_size = e->efi_memdesc_size; @@ -243,6 +246,7 @@ int __init efi_memblock_x86_reserve_range(void) efi.memmap.desc_version); memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size); + set_bit(EFI_PRESERVE_BS_REGIONS, &efi.flags); return 0; } @@ -305,11 +309,11 @@ static void __init efi_clean_memmap(void) if (n_removal > 0) { struct efi_memory_map_data data = { - .phys_map = efi.memmap.phys_map, - .desc_version = efi.memmap.desc_version, - .desc_size = efi.memmap.desc_size, - .size = efi.memmap.desc_size * (efi.memmap.nr_map - n_removal), - .flags = 0, + .phys_map = efi.memmap.phys_map, + .desc_version = efi.memmap.desc_version, + .desc_size = efi.memmap.desc_size, + .size = efi.memmap.desc_size * (efi.memmap.nr_map - n_removal), + .flags = 0, }; pr_warn("Removing %d invalid memory map entries.\n", n_removal); @@ -333,43 +337,32 @@ void __init efi_print_memmap(void) } } -static int __init efi_systab_init(u64 phys) +static int __init efi_systab_init(unsigned long phys) { int size = efi_enabled(EFI_64BIT) ? sizeof(efi_system_table_64_t) : sizeof(efi_system_table_32_t); + const efi_table_hdr_t *hdr; bool over4g = false; void *p; + int ret; - p = early_memremap_ro(phys, size); + hdr = p = early_memremap_ro(phys, size); if (p == NULL) { pr_err("Couldn't map the system table!\n"); return -ENOMEM; } + ret = efi_systab_check_header(hdr, 1); + if (ret) { + early_memunmap(p, size); + return ret; + } + if (efi_enabled(EFI_64BIT)) { const efi_system_table_64_t *systab64 = p; - efi_systab.hdr = systab64->hdr; - efi_systab.fw_vendor = systab64->fw_vendor; - efi_systab.fw_revision = systab64->fw_revision; - efi_systab.con_in_handle = systab64->con_in_handle; - efi_systab.con_in = systab64->con_in; - efi_systab.con_out_handle = systab64->con_out_handle; - efi_systab.con_out = (void *)(unsigned long)systab64->con_out; - efi_systab.stderr_handle = systab64->stderr_handle; - efi_systab.stderr = systab64->stderr; - efi_systab.runtime = (void *)(unsigned long)systab64->runtime; - efi_systab.boottime = (void *)(unsigned long)systab64->boottime; - efi_systab.nr_tables = systab64->nr_tables; - efi_systab.tables = systab64->tables; - - over4g = systab64->con_in_handle > U32_MAX || - systab64->con_in > U32_MAX || - systab64->con_out_handle > U32_MAX || - systab64->con_out > U32_MAX || - systab64->stderr_handle > U32_MAX || - systab64->stderr > U32_MAX || - systab64->boottime > U32_MAX; + efi_runtime = systab64->runtime; + over4g = systab64->runtime > U32_MAX; if (efi_setup) { struct efi_setup_data *data; @@ -380,38 +373,33 @@ static int __init efi_systab_init(u64 phys) return -ENOMEM; } - efi_systab.fw_vendor = (unsigned long)data->fw_vendor; - efi_systab.runtime = (void *)(unsigned long)data->runtime; - efi_systab.tables = (unsigned long)data->tables; + efi_fw_vendor = (unsigned long)data->fw_vendor; + efi_config_table = (unsigned long)data->tables; over4g |= data->fw_vendor > U32_MAX || - data->runtime > U32_MAX || data->tables > U32_MAX; early_memunmap(data, sizeof(*data)); } else { + efi_fw_vendor = systab64->fw_vendor; + efi_config_table = systab64->tables; + over4g |= systab64->fw_vendor > U32_MAX || - systab64->runtime > U32_MAX || systab64->tables > U32_MAX; } + efi_nr_tables = systab64->nr_tables; } else { const efi_system_table_32_t *systab32 = p; - efi_systab.hdr = systab32->hdr; - efi_systab.fw_vendor = systab32->fw_vendor; - efi_systab.fw_revision = systab32->fw_revision; - efi_systab.con_in_handle = systab32->con_in_handle; - efi_systab.con_in = systab32->con_in; - efi_systab.con_out_handle = systab32->con_out_handle; - efi_systab.con_out = (void *)(unsigned long)systab32->con_out; - efi_systab.stderr_handle = systab32->stderr_handle; - efi_systab.stderr = systab32->stderr; - efi_systab.runtime = (void *)(unsigned long)systab32->runtime; - efi_systab.boottime = (void *)(unsigned long)systab32->boottime; - efi_systab.nr_tables = systab32->nr_tables; - efi_systab.tables = systab32->tables; + efi_fw_vendor = systab32->fw_vendor; + efi_runtime = systab32->runtime; + efi_config_table = systab32->tables; + efi_nr_tables = systab32->nr_tables; } + efi.runtime_version = hdr->revision; + + efi_systab_report_header(hdr, efi_fw_vendor); early_memunmap(p, size); if (IS_ENABLED(CONFIG_X86_32) && over4g) { @@ -419,29 +407,40 @@ static int __init efi_systab_init(u64 phys) return -EINVAL; } - efi.systab = &efi_systab; + return 0; +} + +static int __init efi_config_init(const efi_config_table_type_t *arch_tables) +{ + void *config_tables; + int sz, ret; + + if (efi_nr_tables == 0) + return 0; + + if (efi_enabled(EFI_64BIT)) + sz = sizeof(efi_config_table_64_t); + else + sz = sizeof(efi_config_table_32_t); /* - * Verify the EFI Table + * Let's see what config tables the firmware passed to us. */ - if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) { - pr_err("System table signature incorrect!\n"); - return -EINVAL; + config_tables = early_memremap(efi_config_table, efi_nr_tables * sz); + if (config_tables == NULL) { + pr_err("Could not map Configuration table!\n"); + return -ENOMEM; } - if ((efi.systab->hdr.revision >> 16) == 0) - pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n", - efi.systab->hdr.revision >> 16, - efi.systab->hdr.revision & 0xffff); - return 0; + ret = efi_config_parse_tables(config_tables, efi_nr_tables, + arch_tables); + + early_memunmap(config_tables, efi_nr_tables * sz); + return ret; } void __init efi_init(void) { - efi_char16_t *c16; - char vendor[100] = "unknown"; - int i = 0; - if (IS_ENABLED(CONFIG_X86_32) && (boot_params.efi_info.efi_systab_hi || boot_params.efi_info.efi_memmap_hi)) { @@ -455,29 +454,7 @@ void __init efi_init(void) if (efi_systab_init(efi_systab_phys)) return; - efi.config_table = (unsigned long)efi.systab->tables; - efi.fw_vendor = (unsigned long)efi.systab->fw_vendor; - efi.runtime = (unsigned long)efi.systab->runtime; - - /* - * Show what we know for posterity - */ - c16 = early_memremap_ro(efi.systab->fw_vendor, - sizeof(vendor) * sizeof(efi_char16_t)); - if (c16) { - for (i = 0; i < sizeof(vendor) - 1 && c16[i]; ++i) - vendor[i] = c16[i]; - vendor[i] = '\0'; - early_memunmap(c16, sizeof(vendor) * sizeof(efi_char16_t)); - } else { - pr_err("Could not map the firmware vendor!\n"); - } - - pr_info("EFI v%u.%.02u by %s\n", - efi.systab->hdr.revision >> 16, - efi.systab->hdr.revision & 0xffff, vendor); - - if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables)) + if (efi_reuse_config(efi_config_table, efi_nr_tables)) return; if (efi_config_init(arch_tables)) @@ -496,6 +473,22 @@ void __init efi_init(void) return; } + /* Parse the EFI Properties table if it exists */ + if (prop_phys != EFI_INVALID_TABLE_ADDR) { + efi_properties_table_t *tbl; + + tbl = early_memremap_ro(prop_phys, sizeof(*tbl)); + if (tbl == NULL) { + pr_err("Could not map Properties table!\n"); + } else { + if (tbl->memory_protection_attribute & + EFI_PROPERTIES_RUNTIME_MEMORY_PROTECTION_NON_EXECUTABLE_PE_DATA) + set_bit(EFI_NX_PE_DATA, &efi.flags); + + early_memunmap(tbl, sizeof(*tbl)); + } + } + set_bit(EFI_RUNTIME_SERVICES, &efi.flags); efi_clean_memmap(); @@ -602,20 +595,6 @@ static void __init efi_merge_regions(void) } } -static void __init get_systab_virt_addr(efi_memory_desc_t *md) -{ - unsigned long size; - u64 end, systab; - - size = md->num_pages << EFI_PAGE_SHIFT; - end = md->phys_addr + size; - systab = efi_systab_phys; - if (md->phys_addr <= systab && systab < end) { - systab += md->virt_addr - md->phys_addr; - efi.systab = (efi_system_table_t *)(unsigned long)systab; - } -} - static void *realloc_pages(void *old_memmap, int old_shift) { void *ret; @@ -771,7 +750,6 @@ static void * __init efi_map_regions(int *count, int *pg_shift) continue; efi_map_region(md); - get_systab_virt_addr(md); if (left < desc_size) { new_memmap = realloc_pages(new_memmap, *pg_shift); @@ -797,8 +775,6 @@ static void __init kexec_enter_virtual_mode(void) efi_memory_desc_t *md; unsigned int num_pages; - efi.systab = NULL; - /* * We don't do virtual mode, since we don't do runtime services, on * non-native EFI. With the UV1 memmap, we don't do runtime services in @@ -821,10 +797,8 @@ static void __init kexec_enter_virtual_mode(void) * Map efi regions which were passed via setup_data. The virt_addr is a * fixed addr which was used in first kernel of a kexec boot. */ - for_each_efi_memory_desc(md) { + for_each_efi_memory_desc(md) efi_map_region_fixed(md); /* FIXME: add error handling */ - get_systab_virt_addr(md); - } /* * Unregister the early EFI memmap from efi_init() and install @@ -839,8 +813,6 @@ static void __init kexec_enter_virtual_mode(void) return; } - BUG_ON(!efi.systab); - num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE); num_pages >>= PAGE_SHIFT; @@ -850,15 +822,6 @@ static void __init kexec_enter_virtual_mode(void) } efi_sync_low_kernel_mappings(); - - /* - * Now that EFI is in virtual mode, update the function - * pointers in the runtime service table to the new virtual addresses. - * - * Call EFI services through wrapper functions. - */ - efi.runtime_version = efi_systab.hdr.revision; - efi_native_runtime_setup(); #endif } @@ -892,8 +855,6 @@ static void __init __efi_enter_virtual_mode(void) efi_status_t status; unsigned long pa; - efi.systab = NULL; - if (efi_alloc_page_tables()) { pr_err("Failed to allocate EFI page tables\n"); goto err; @@ -925,9 +886,6 @@ static void __init __efi_enter_virtual_mode(void) efi_print_memmap(); } - if (WARN_ON(!efi.systab)) - goto err; - if (efi_setup_page_tables(pa, 1 << pg_shift)) goto err; @@ -936,23 +894,17 @@ static void __init __efi_enter_virtual_mode(void) status = efi_set_virtual_address_map(efi.memmap.desc_size * count, efi.memmap.desc_size, efi.memmap.desc_version, - (efi_memory_desc_t *)pa); + (efi_memory_desc_t *)pa, + efi_systab_phys); if (status != EFI_SUCCESS) { pr_err("Unable to switch EFI into virtual mode (status=%lx)!\n", status); goto err; } + efi_check_for_embedded_firmwares(); efi_free_boot_services(); - /* - * Now that EFI is in virtual mode, update the function - * pointers in the runtime service table to the new virtual addresses. - * - * Call EFI services through wrapper functions. - */ - efi.runtime_version = efi_systab.hdr.revision; - if (!efi_is_mixed()) efi_native_runtime_setup(); else @@ -978,6 +930,8 @@ void __init efi_enter_virtual_mode(void) if (efi_enabled(EFI_PARAVIRT)) return; + efi.runtime = (efi_runtime_services_t *)efi_runtime; + if (efi_setup) kexec_enter_virtual_mode(); else @@ -999,3 +953,43 @@ bool efi_is_table_address(unsigned long phys_addr) return false; } + +char *efi_systab_show_arch(char *str) +{ + if (uga_phys != EFI_INVALID_TABLE_ADDR) + str += sprintf(str, "UGA=0x%lx\n", uga_phys); + return str; +} + +#define EFI_FIELD(var) efi_ ## var + +#define EFI_ATTR_SHOW(name) \ +static ssize_t name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "0x%lx\n", EFI_FIELD(name)); \ +} + +EFI_ATTR_SHOW(fw_vendor); +EFI_ATTR_SHOW(runtime); +EFI_ATTR_SHOW(config_table); + +struct kobj_attribute efi_attr_fw_vendor = __ATTR_RO(fw_vendor); +struct kobj_attribute efi_attr_runtime = __ATTR_RO(runtime); +struct kobj_attribute efi_attr_config_table = __ATTR_RO(config_table); + +umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + if (attr == &efi_attr_fw_vendor.attr) { + if (efi_enabled(EFI_PARAVIRT) || + efi_fw_vendor == EFI_INVALID_TABLE_ADDR) + return 0; + } else if (attr == &efi_attr_runtime.attr) { + if (efi_runtime == EFI_INVALID_TABLE_ADDR) + return 0; + } else if (attr == &efi_attr_config_table.attr) { + if (efi_config_table == EFI_INVALID_TABLE_ADDR) + return 0; + } + return attr->mode; +} diff --git a/arch/x86/platform/efi/efi_32.c b/arch/x86/platform/efi/efi_32.c index 081d466002c9..c049c432745d 100644 --- a/arch/x86/platform/efi/efi_32.c +++ b/arch/x86/platform/efi/efi_32.c @@ -66,14 +66,16 @@ void __init efi_map_region(efi_memory_desc_t *md) void __init efi_map_region_fixed(efi_memory_desc_t *md) {} void __init parse_efi_setup(u64 phys_addr, u32 data_len) {} -efi_status_t efi_call_svam(efi_set_virtual_address_map_t *__efiapi *, - u32, u32, u32, void *); +efi_status_t efi_call_svam(efi_runtime_services_t * const *, + u32, u32, u32, void *, u32); efi_status_t __init efi_set_virtual_address_map(unsigned long memory_map_size, unsigned long descriptor_size, u32 descriptor_version, - efi_memory_desc_t *virtual_map) + efi_memory_desc_t *virtual_map, + unsigned long systab_phys) { + const efi_system_table_t *systab = (efi_system_table_t *)systab_phys; struct desc_ptr gdt_descr; efi_status_t status; unsigned long flags; @@ -90,9 +92,10 @@ efi_status_t __init efi_set_virtual_address_map(unsigned long memory_map_size, /* Disable interrupts around EFI calls: */ local_irq_save(flags); - status = efi_call_svam(&efi.systab->runtime->set_virtual_address_map, + status = efi_call_svam(&systab->runtime, memory_map_size, descriptor_size, - descriptor_version, virtual_map); + descriptor_version, virtual_map, + __pa(&efi.runtime)); local_irq_restore(flags); load_fixmap_gdt(0); diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index d19a2edd63cb..211bb9358b73 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -497,12 +497,9 @@ static DEFINE_SPINLOCK(efi_runtime_lock); */ #define __efi_thunk(func, ...) \ ({ \ - efi_runtime_services_32_t *__rt; \ unsigned short __ds, __es; \ efi_status_t ____s; \ \ - __rt = (void *)(unsigned long)efi.systab->mixed_mode.runtime; \ - \ savesegment(ds, __ds); \ savesegment(es, __es); \ \ @@ -510,7 +507,7 @@ static DEFINE_SPINLOCK(efi_runtime_lock); loadsegment(ds, __KERNEL_DS); \ loadsegment(es, __KERNEL_DS); \ \ - ____s = efi64_thunk(__rt->func, __VA_ARGS__); \ + ____s = efi64_thunk(efi.runtime->mixed_mode.func, __VA_ARGS__); \ \ loadsegment(ds, __ds); \ loadsegment(es, __es); \ @@ -839,8 +836,10 @@ efi_status_t __init __no_sanitize_address efi_set_virtual_address_map(unsigned long memory_map_size, unsigned long descriptor_size, u32 descriptor_version, - efi_memory_desc_t *virtual_map) + efi_memory_desc_t *virtual_map, + unsigned long systab_phys) { + const efi_system_table_t *systab = (efi_system_table_t *)systab_phys; efi_status_t status; unsigned long flags; pgd_t *save_pgd = NULL; @@ -863,13 +862,16 @@ efi_set_virtual_address_map(unsigned long memory_map_size, /* Disable interrupts around EFI calls: */ local_irq_save(flags); - status = efi_call(efi.systab->runtime->set_virtual_address_map, + status = efi_call(efi.runtime->set_virtual_address_map, memory_map_size, descriptor_size, descriptor_version, virtual_map); local_irq_restore(flags); kernel_fpu_end(); + /* grab the virtually remapped EFI runtime services table pointer */ + efi.runtime = READ_ONCE(systab->runtime); + if (save_pgd) efi_uv1_memmap_phys_epilog(save_pgd); else diff --git a/arch/x86/platform/efi/efi_stub_32.S b/arch/x86/platform/efi/efi_stub_32.S index 75c46e7a809f..09ec84f6ef51 100644 --- a/arch/x86/platform/efi/efi_stub_32.S +++ b/arch/x86/platform/efi/efi_stub_32.S @@ -8,14 +8,20 @@ #include <linux/linkage.h> #include <linux/init.h> +#include <asm/asm-offsets.h> #include <asm/page_types.h> __INIT SYM_FUNC_START(efi_call_svam) - push 8(%esp) - push 8(%esp) + push %ebp + movl %esp, %ebp + push %ebx + + push 16(%esp) + push 16(%esp) push %ecx push %edx + movl %eax, %ebx // &systab_phys->runtime /* * Switch to the flat mapped alias of this routine, by jumping to the @@ -35,15 +41,20 @@ SYM_FUNC_START(efi_call_svam) subl $__PAGE_OFFSET, %esp /* call the EFI routine */ - call *(%eax) + movl (%eax), %eax + call *EFI_svam(%eax) - /* convert ESP back to a kernel VA, and pop the outgoing args */ - addl $__PAGE_OFFSET + 16, %esp + /* grab the virtually remapped EFI runtime services table pointer */ + movl (%ebx), %ecx + movl 36(%esp), %edx // &efi.runtime + movl %ecx, (%edx) /* re-enable paging */ movl %cr0, %edx orl $0x80000000, %edx movl %edx, %cr0 + movl 16(%esp), %ebx + leave ret SYM_FUNC_END(efi_call_svam) diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 88d32c06cffa..a5a469cdf5bf 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -410,6 +410,10 @@ void __init efi_free_boot_services(void) int num_entries = 0; void *new, *new_md; + /* Keep all regions for /sys/kernel/debug/efi */ + if (efi_enabled(EFI_DBG)) + return; + for_each_efi_memory_desc(md) { unsigned long long start = md->phys_addr; unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; @@ -537,7 +541,7 @@ int __init efi_reuse_config(u64 tables, int nr_tables) goto out_memremap; } - for (i = 0; i < efi.systab->nr_tables; i++) { + for (i = 0; i < nr_tables; i++) { efi_guid_t guid; guid = ((efi_config_table_64_t *)p)->guid; @@ -659,12 +663,9 @@ static int qrk_capsule_setup_info(struct capsule_info *cap_info, void **pkbuff, return 1; } -#define ICPU(family, model, quirk_handler) \ - { X86_VENDOR_INTEL, family, model, X86_FEATURE_ANY, \ - (unsigned long)&quirk_handler } - static const struct x86_cpu_id efi_capsule_quirk_ids[] = { - ICPU(5, 9, qrk_capsule_setup_info), /* Intel Quark X1000 */ + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 5, INTEL_FAM5_QUARK_X1000, + &qrk_capsule_setup_info), { } }; diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bt.c b/arch/x86/platform/intel-mid/device_libs/platform_bt.c index e3f4bfc08f78..31dda18bb370 100644 --- a/arch/x86/platform/intel-mid/device_libs/platform_bt.c +++ b/arch/x86/platform/intel-mid/device_libs/platform_bt.c @@ -60,11 +60,8 @@ static struct bt_sfi_data tng_bt_sfi_data __initdata = { .setup = tng_bt_sfi_setup, }; -#define ICPU(model, ddata) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (kernel_ulong_t)&ddata } - static const struct x86_cpu_id bt_sfi_cpu_ids[] = { - ICPU(INTEL_FAM6_ATOM_SILVERMONT_MID, tng_bt_sfi_data), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &tng_bt_sfi_data), {} }; diff --git a/arch/x86/platform/intel-quark/imr.c b/arch/x86/platform/intel-quark/imr.c index e9d97d52475e..0286fe1b14b5 100644 --- a/arch/x86/platform/intel-quark/imr.c +++ b/arch/x86/platform/intel-quark/imr.c @@ -569,7 +569,7 @@ static void __init imr_fixup_memmap(struct imr_device *idev) } static const struct x86_cpu_id imr_ids[] __initconst = { - { X86_VENDOR_INTEL, 5, 9 }, /* Intel Quark SoC X1000. */ + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 5, INTEL_FAM5_QUARK_X1000, NULL), {} }; diff --git a/arch/x86/platform/intel-quark/imr_selftest.c b/arch/x86/platform/intel-quark/imr_selftest.c index 4307830e1b6f..570e3062faac 100644 --- a/arch/x86/platform/intel-quark/imr_selftest.c +++ b/arch/x86/platform/intel-quark/imr_selftest.c @@ -105,7 +105,7 @@ static void __init imr_self_test(void) } static const struct x86_cpu_id imr_ids[] __initconst = { - { X86_VENDOR_INTEL, 5, 9 }, /* Intel Quark SoC X1000. */ + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 5, INTEL_FAM5_QUARK_X1000, NULL), {} }; diff --git a/arch/x86/platform/intel/iosf_mbi.c b/arch/x86/platform/intel/iosf_mbi.c index 9e2444500428..526f70f27c1c 100644 --- a/arch/x86/platform/intel/iosf_mbi.c +++ b/arch/x86/platform/intel/iosf_mbi.c @@ -265,7 +265,7 @@ static void iosf_mbi_reset_semaphore(void) iosf_mbi_sem_address, 0, PUNIT_SEMAPHORE_BIT)) dev_err(&mbi_pdev->dev, "Error P-Unit semaphore reset failed\n"); - pm_qos_update_request(&iosf_mbi_pm_qos, PM_QOS_DEFAULT_VALUE); + cpu_latency_qos_update_request(&iosf_mbi_pm_qos, PM_QOS_DEFAULT_VALUE); blocking_notifier_call_chain(&iosf_mbi_pmic_bus_access_notifier, MBI_PMIC_BUS_ACCESS_END, NULL); @@ -301,8 +301,8 @@ static void iosf_mbi_reset_semaphore(void) * 4) When CPU cores enter C6 or C7 the P-Unit needs to talk to the PMIC * if this happens while the kernel itself is accessing the PMIC I2C bus * the SoC hangs. - * As the third step we call pm_qos_update_request() to disallow the CPU - * to enter C6 or C7. + * As the third step we call cpu_latency_qos_update_request() to disallow the + * CPU to enter C6 or C7. * * 5) The P-Unit has a PMIC bus semaphore which we can request to stop * autonomous P-Unit tasks from accessing the PMIC I2C bus while we hold it. @@ -338,7 +338,7 @@ int iosf_mbi_block_punit_i2c_access(void) * requires the P-Unit to talk to the PMIC and if this happens while * we're holding the semaphore, the SoC hangs. */ - pm_qos_update_request(&iosf_mbi_pm_qos, 0); + cpu_latency_qos_update_request(&iosf_mbi_pm_qos, 0); /* host driver writes to side band semaphore register */ ret = iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_REG_WRITE, @@ -547,8 +547,7 @@ static int __init iosf_mbi_init(void) { iosf_debugfs_init(); - pm_qos_add_request(&iosf_mbi_pm_qos, PM_QOS_CPU_DMA_LATENCY, - PM_QOS_DEFAULT_VALUE); + cpu_latency_qos_add_request(&iosf_mbi_pm_qos, PM_QOS_DEFAULT_VALUE); return pci_register_driver(&iosf_mbi_pci_driver); } @@ -561,7 +560,7 @@ static void __exit iosf_mbi_exit(void) pci_dev_put(mbi_pdev); mbi_pdev = NULL; - pm_qos_remove_request(&iosf_mbi_pm_qos); + cpu_latency_qos_remove_request(&iosf_mbi_pm_qos); } module_init(iosf_mbi_init); diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 915bb1639763..aaff9ed7ff45 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -475,20 +475,8 @@ static int msr_save_cpuid_features(const struct x86_cpu_id *c) } static const struct x86_cpu_id msr_save_cpu_table[] = { - { - .vendor = X86_VENDOR_AMD, - .family = 0x15, - .model = X86_MODEL_ANY, - .feature = X86_FEATURE_ANY, - .driver_data = (kernel_ulong_t)msr_save_cpuid_features, - }, - { - .vendor = X86_VENDOR_AMD, - .family = 0x16, - .model = X86_MODEL_ANY, - .feature = X86_FEATURE_ANY, - .driver_data = (kernel_ulong_t)msr_save_cpuid_features, - }, + X86_MATCH_VENDOR_FAM(AMD, 0x15, &msr_save_cpuid_features), + X86_MATCH_VENDOR_FAM(AMD, 0x16, &msr_save_cpuid_features), {} }; diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile index 99b6332ba540..b11ec5d8f8ac 100644 --- a/arch/x86/realmode/rm/Makefile +++ b/arch/x86/realmode/rm/Makefile @@ -71,5 +71,6 @@ $(obj)/realmode.relocs: $(obj)/realmode.elf FORCE KBUILD_CFLAGS := $(REALMODE_CFLAGS) -D_SETUP -D_WAKEUP \ -I$(srctree)/arch/x86/boot KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ +KBUILD_CFLAGS += -fno-asynchronous-unwind-tables GCOV_PROFILE := n UBSAN_SANITIZE := n diff --git a/arch/x86/realmode/rm/realmode.lds.S b/arch/x86/realmode/rm/realmode.lds.S index 64d135d1ee63..63aa51875ba0 100644 --- a/arch/x86/realmode/rm/realmode.lds.S +++ b/arch/x86/realmode/rm/realmode.lds.S @@ -71,7 +71,6 @@ SECTIONS /DISCARD/ : { *(.note*) *(.debug*) - *(.eh_frame*) } #include "pasyms.h" diff --git a/arch/x86/um/Makefile b/arch/x86/um/Makefile index 33c51c064c77..77f70b969d14 100644 --- a/arch/x86/um/Makefile +++ b/arch/x86/um/Makefile @@ -21,6 +21,7 @@ obj-y += checksum_32.o syscalls_32.o obj-$(CONFIG_ELF_CORE) += elfcore.o subarch-y = ../lib/string_32.o ../lib/atomic64_32.o ../lib/atomic64_cx8_32.o +subarch-y += ../kernel/sys_ia32.o else diff --git a/arch/x86/um/sys_call_table_32.c b/arch/x86/um/sys_call_table_32.c index 9649b5ad2ca2..2ed81e581755 100644 --- a/arch/x86/um/sys_call_table_32.c +++ b/arch/x86/um/sys_call_table_32.c @@ -7,7 +7,7 @@ #include <linux/linkage.h> #include <linux/sys.h> #include <linux/cache.h> -#include <generated/user_constants.h> +#include <asm/unistd.h> #include <asm/syscall.h> #define __NO_STUBS @@ -26,11 +26,11 @@ #define old_mmap sys_old_mmap -#define __SYSCALL_I386(nr, sym, qual) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ; +#define __SYSCALL_I386(nr, sym) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ; #include <asm/syscalls_32.h> #undef __SYSCALL_I386 -#define __SYSCALL_I386(nr, sym, qual) [ nr ] = sym, +#define __SYSCALL_I386(nr, sym) [ nr ] = sym, extern asmlinkage long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); diff --git a/arch/x86/um/sys_call_table_64.c b/arch/x86/um/sys_call_table_64.c index c8bc7fb8cbd6..2e8544dafbb0 100644 --- a/arch/x86/um/sys_call_table_64.c +++ b/arch/x86/um/sys_call_table_64.c @@ -7,7 +7,7 @@ #include <linux/linkage.h> #include <linux/sys.h> #include <linux/cache.h> -#include <generated/user_constants.h> +#include <asm/unistd.h> #include <asm/syscall.h> #define __NO_STUBS @@ -36,11 +36,14 @@ #define stub_execveat sys_execveat #define stub_rt_sigreturn sys_rt_sigreturn -#define __SYSCALL_64(nr, sym, qual) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ; +#define __SYSCALL_X32(nr, sym) +#define __SYSCALL_COMMON(nr, sym) __SYSCALL_64(nr, sym) + +#define __SYSCALL_64(nr, sym) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ; #include <asm/syscalls_64.h> #undef __SYSCALL_64 -#define __SYSCALL_64(nr, sym, qual) [ nr ] = sym, +#define __SYSCALL_64(nr, sym) [ nr ] = sym, extern asmlinkage long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); diff --git a/arch/x86/um/user-offsets.c b/arch/x86/um/user-offsets.c index 5b37b7f952dd..c51dd8363d25 100644 --- a/arch/x86/um/user-offsets.c +++ b/arch/x86/um/user-offsets.c @@ -9,18 +9,6 @@ #include <linux/ptrace.h> #include <asm/types.h> -#ifdef __i386__ -#define __SYSCALL_I386(nr, sym, qual) [nr] = 1, -static char syscalls[] = { -#include <asm/syscalls_32.h> -}; -#else -#define __SYSCALL_64(nr, sym, qual) [nr] = 1, -static char syscalls[] = { -#include <asm/syscalls_64.h> -}; -#endif - #define DEFINE(sym, val) \ asm volatile("\n->" #sym " %0 " #val : : "i" (val)) @@ -94,7 +82,4 @@ void foo(void) DEFINE(UM_PROT_READ, PROT_READ); DEFINE(UM_PROT_WRITE, PROT_WRITE); DEFINE(UM_PROT_EXEC, PROT_EXEC); - - DEFINE(__NR_syscall_max, sizeof(syscalls) - 1); - DEFINE(NR_syscalls, sizeof(syscalls)); } diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 7a43b2ae19f1..2097fa0ebdb5 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -132,7 +132,7 @@ void __init xen_smp_cpus_done(unsigned int max_cpus) if (xen_vcpu_nr(cpu) < MAX_VIRT_CPUS) continue; - rc = cpu_down(cpu); + rc = remove_cpu(cpu); if (rc == 0) { /* diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index befbdd8b17f0..c8897aad13cd 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -145,12 +145,19 @@ static struct notifier_block xen_pvclock_gtod_notifier = { .notifier_call = xen_pvclock_gtod_notify, }; +static int xen_cs_enable(struct clocksource *cs) +{ + vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK); + return 0; +} + static struct clocksource xen_clocksource __read_mostly = { - .name = "xen", - .rating = 400, - .read = xen_clocksource_get_cycles, - .mask = ~0, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .name = "xen", + .rating = 400, + .read = xen_clocksource_get_cycles, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .enable = xen_cs_enable, }; /* @@ -412,12 +419,13 @@ void xen_restore_time_memory_area(void) ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t); /* - * We don't disable VCLOCK_PVCLOCK entirely if it fails to register the - * secondary time info with Xen or if we migrated to a host without the - * necessary flags. On both of these cases what happens is either - * process seeing a zeroed out pvti or seeing no PVCLOCK_TSC_STABLE_BIT - * bit set. Userspace checks the latter and if 0, it discards the data - * in pvti and fallbacks to a system call for a reliable timestamp. + * We don't disable VDSO_CLOCKMODE_PVCLOCK entirely if it fails to + * register the secondary time info with Xen or if we migrated to a + * host without the necessary flags. On both of these cases what + * happens is either process seeing a zeroed out pvti or seeing no + * PVCLOCK_TSC_STABLE_BIT bit set. Userspace checks the latter and + * if 0, it discards the data in pvti and fallbacks to a system + * call for a reliable timestamp. */ if (ret != 0) pr_notice("Cannot restore secondary vcpu_time_info (err %d)", @@ -443,7 +451,7 @@ static void xen_setup_vsyscall_time_info(void) ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t); if (ret) { - pr_notice("xen: VCLOCK_PVCLOCK not supported (err %d)\n", ret); + pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (err %d)\n", ret); free_page((unsigned long)ti); return; } @@ -460,14 +468,14 @@ static void xen_setup_vsyscall_time_info(void) if (!ret) free_page((unsigned long)ti); - pr_notice("xen: VCLOCK_PVCLOCK not supported (tsc unstable)\n"); + pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (tsc unstable)\n"); return; } xen_clock = ti; pvclock_set_pvti_cpu0_va(xen_clock); - xen_clocksource.archdata.vclock_mode = VCLOCK_PVCLOCK; + xen_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK; } static void __init xen_time_init(void) diff --git a/arch/xtensa/include/asm/futex.h b/arch/xtensa/include/asm/futex.h index 964611083224..a1a27b2ea460 100644 --- a/arch/xtensa/include/asm/futex.h +++ b/arch/xtensa/include/asm/futex.h @@ -72,7 +72,8 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, #if XCHAL_HAVE_S32C1I || XCHAL_HAVE_EXCLUSIVE int oldval = 0, ret; - pagefault_disable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; switch (op) { case FUTEX_OP_SET: @@ -99,8 +100,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, ret = -ENOSYS; } - pagefault_enable(); - if (!ret) *oval = oldval; diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 833109880165..49322b66cda9 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -267,13 +267,12 @@ static int __init simdisk_setup(struct simdisk *dev, int which, spin_lock_init(&dev->lock); dev->users = 0; - dev->queue = blk_alloc_queue(GFP_KERNEL); + dev->queue = blk_alloc_queue(simdisk_make_request, NUMA_NO_NODE); if (dev->queue == NULL) { pr_err("blk_alloc_queue failed\n"); goto out_alloc_queue; } - blk_queue_make_request(dev->queue, simdisk_make_request); dev->queue->queuedata = dev; dev->gd = alloc_disk(SIMDISK_MINORS); |