| xj | b04a402 | 2021-11-25 15:01:52 +0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | #include <linux/export.h> |
| 3 | |
| 4 | #include "libgcc.h" |
| 5 | |
| 6 | /* |
| 7 | * GCC 7 & older can suboptimally generate __multi3 calls for mips64r6, so for |
| 8 | * that specific case only we implement that intrinsic here. |
| 9 | * |
| 10 | * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82981 |
| 11 | */ |
| 12 | #if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) && (__GNUC__ < 8) |
| 13 | |
| 14 | /* multiply 64-bit values, low 64-bits returned */ |
| 15 | static inline long long notrace dmulu(long long a, long long b) |
| 16 | { |
| 17 | long long res; |
| 18 | |
| 19 | asm ("dmulu %0,%1,%2" : "=r" (res) : "r" (a), "r" (b)); |
| 20 | return res; |
| 21 | } |
| 22 | |
| 23 | /* multiply 64-bit unsigned values, high 64-bits of 128-bit result returned */ |
| 24 | static inline long long notrace dmuhu(long long a, long long b) |
| 25 | { |
| 26 | long long res; |
| 27 | |
| 28 | asm ("dmuhu %0,%1,%2" : "=r" (res) : "r" (a), "r" (b)); |
| 29 | return res; |
| 30 | } |
| 31 | |
| 32 | /* multiply 128-bit values, low 128-bits returned */ |
| 33 | ti_type notrace __multi3(ti_type a, ti_type b) |
| 34 | { |
| 35 | TWunion res, aa, bb; |
| 36 | |
| 37 | aa.ti = a; |
| 38 | bb.ti = b; |
| 39 | |
| 40 | /* |
| 41 | * a * b = (a.lo * b.lo) |
| 42 | * + 2^64 * (a.hi * b.lo + a.lo * b.hi) |
| 43 | * [+ 2^128 * (a.hi * b.hi)] |
| 44 | */ |
| 45 | res.s.low = dmulu(aa.s.low, bb.s.low); |
| 46 | res.s.high = dmuhu(aa.s.low, bb.s.low); |
| 47 | res.s.high += dmulu(aa.s.high, bb.s.low); |
| 48 | res.s.high += dmulu(aa.s.low, bb.s.high); |
| 49 | |
| 50 | return res.ti; |
| 51 | } |
| 52 | EXPORT_SYMBOL(__multi3); |
| 53 | |
| 54 | #endif /* 64BIT && CPU_MIPSR6 && GCC7 */ |