| b.liu | e958203 | 2025-04-17 19:18:16 +0800 | [diff] [blame^] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | #ifndef _ASM_X86_JUMP_LABEL_H |
| 3 | #define _ASM_X86_JUMP_LABEL_H |
| 4 | |
| 5 | #define HAVE_JUMP_LABEL_BATCH |
| 6 | |
| 7 | #define JUMP_LABEL_NOP_SIZE 5 |
| 8 | |
| 9 | #ifdef CONFIG_X86_64 |
| 10 | # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC |
| 11 | #else |
| 12 | # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC |
| 13 | #endif |
| 14 | |
| 15 | #include <asm/asm.h> |
| 16 | #include <asm/nops.h> |
| 17 | |
| 18 | #ifndef __ASSEMBLY__ |
| 19 | |
| 20 | #include <linux/stringify.h> |
| 21 | #include <linux/types.h> |
| 22 | |
| 23 | static __always_inline bool arch_static_branch(struct static_key *key, bool branch) |
| 24 | { |
| 25 | asm_volatile_goto("1:" |
| 26 | ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" |
| 27 | ".pushsection __jump_table, \"aw\" \n\t" |
| 28 | _ASM_ALIGN "\n\t" |
| 29 | ".long 1b - ., %l[l_yes] - . \n\t" |
| 30 | _ASM_PTR "%c0 + %c1 - .\n\t" |
| 31 | ".popsection \n\t" |
| 32 | : : "i" (key), "i" (branch) : : l_yes); |
| 33 | |
| 34 | return false; |
| 35 | l_yes: |
| 36 | return true; |
| 37 | } |
| 38 | |
| 39 | static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) |
| 40 | { |
| 41 | asm_volatile_goto("1:" |
| 42 | ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" |
| 43 | "2:\n\t" |
| 44 | ".pushsection __jump_table, \"aw\" \n\t" |
| 45 | _ASM_ALIGN "\n\t" |
| 46 | ".long 1b - ., %l[l_yes] - . \n\t" |
| 47 | _ASM_PTR "%c0 + %c1 - .\n\t" |
| 48 | ".popsection \n\t" |
| 49 | : : "i" (key), "i" (branch) : : l_yes); |
| 50 | |
| 51 | return false; |
| 52 | l_yes: |
| 53 | return true; |
| 54 | } |
| 55 | |
| 56 | #else /* __ASSEMBLY__ */ |
| 57 | |
| 58 | .macro STATIC_JUMP_IF_TRUE target, key, def |
| 59 | .Lstatic_jump_\@: |
| 60 | .if \def |
| 61 | /* Equivalent to "jmp.d32 \target" */ |
| 62 | .byte 0xe9 |
| 63 | .long \target - .Lstatic_jump_after_\@ |
| 64 | .Lstatic_jump_after_\@: |
| 65 | .else |
| 66 | .byte STATIC_KEY_INIT_NOP |
| 67 | .endif |
| 68 | .pushsection __jump_table, "aw" |
| 69 | _ASM_ALIGN |
| 70 | .long .Lstatic_jump_\@ - ., \target - . |
| 71 | _ASM_PTR \key - . |
| 72 | .popsection |
| 73 | .endm |
| 74 | |
| 75 | .macro STATIC_JUMP_IF_FALSE target, key, def |
| 76 | .Lstatic_jump_\@: |
| 77 | .if \def |
| 78 | .byte STATIC_KEY_INIT_NOP |
| 79 | .else |
| 80 | /* Equivalent to "jmp.d32 \target" */ |
| 81 | .byte 0xe9 |
| 82 | .long \target - .Lstatic_jump_after_\@ |
| 83 | .Lstatic_jump_after_\@: |
| 84 | .endif |
| 85 | .pushsection __jump_table, "aw" |
| 86 | _ASM_ALIGN |
| 87 | .long .Lstatic_jump_\@ - ., \target - . |
| 88 | _ASM_PTR \key + 1 - . |
| 89 | .popsection |
| 90 | .endm |
| 91 | |
| 92 | #endif /* __ASSEMBLY__ */ |
| 93 | |
| 94 | #endif |