yuezonghe | 824eb0c | 2024-06-27 02:32:26 -0700 | [diff] [blame^] | 1 | /* Copy memory to memory until the specified number of bytes |
| 2 | has been copied. Overlap is handled correctly. |
| 3 | Copyright (C) 1991, 1995, 1996, 1997, 2003 Free Software Foundation, Inc. |
| 4 | This file is part of the GNU C Library. |
| 5 | Contributed by Torbjorn Granlund (tege@sics.se). |
| 6 | |
| 7 | The GNU C Library is free software; you can redistribute it and/or |
| 8 | modify it under the terms of the GNU Lesser General Public |
| 9 | License as published by the Free Software Foundation; either |
| 10 | version 2.1 of the License, or (at your option) any later version. |
| 11 | |
| 12 | The GNU C Library is distributed in the hope that it will be useful, |
| 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 15 | Lesser General Public License for more details. |
| 16 | |
| 17 | You should have received a copy of the GNU Lesser General Public |
| 18 | License along with the GNU C Library; if not, write to the Free |
| 19 | Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA |
| 20 | 02111-1307 USA. */ |
| 21 | |
| 22 | #include <string.h> |
| 23 | |
| 24 | #include "memcopy.h" |
| 25 | #include "pagecopy.h" |
| 26 | |
| 27 | #ifdef __ARCH_HAS_BWD_MEMCPY__ |
| 28 | /* generic-opt memmove assumes memcpy does forward copying! */ |
| 29 | #include "_memcpy_fwd.c" |
| 30 | #endif |
| 31 | |
| 32 | |
| 33 | static void _wordcopy_bwd_aligned (long int dstp, long int srcp, size_t len) |
| 34 | { |
| 35 | op_t a0 = 0; |
| 36 | op_t a1 = 0; |
| 37 | |
| 38 | switch (len % 8) |
| 39 | { |
| 40 | case 2: |
| 41 | srcp -= 2 * OPSIZ; |
| 42 | dstp -= 1 * OPSIZ; |
| 43 | a0 = ((op_t *) srcp)[1]; |
| 44 | len += 6; |
| 45 | goto do1; |
| 46 | case 3: |
| 47 | srcp -= 3 * OPSIZ; |
| 48 | dstp -= 2 * OPSIZ; |
| 49 | a1 = ((op_t *) srcp)[2]; |
| 50 | len += 5; |
| 51 | goto do2; |
| 52 | case 4: |
| 53 | srcp -= 4 * OPSIZ; |
| 54 | dstp -= 3 * OPSIZ; |
| 55 | a0 = ((op_t *) srcp)[3]; |
| 56 | len += 4; |
| 57 | goto do3; |
| 58 | case 5: |
| 59 | srcp -= 5 * OPSIZ; |
| 60 | dstp -= 4 * OPSIZ; |
| 61 | a1 = ((op_t *) srcp)[4]; |
| 62 | len += 3; |
| 63 | goto do4; |
| 64 | case 6: |
| 65 | srcp -= 6 * OPSIZ; |
| 66 | dstp -= 5 * OPSIZ; |
| 67 | a0 = ((op_t *) srcp)[5]; |
| 68 | len += 2; |
| 69 | goto do5; |
| 70 | case 7: |
| 71 | srcp -= 7 * OPSIZ; |
| 72 | dstp -= 6 * OPSIZ; |
| 73 | a1 = ((op_t *) srcp)[6]; |
| 74 | len += 1; |
| 75 | goto do6; |
| 76 | |
| 77 | case 0: |
| 78 | if (OP_T_THRES <= 3 * OPSIZ && len == 0) |
| 79 | return; |
| 80 | srcp -= 8 * OPSIZ; |
| 81 | dstp -= 7 * OPSIZ; |
| 82 | a0 = ((op_t *) srcp)[7]; |
| 83 | goto do7; |
| 84 | case 1: |
| 85 | srcp -= 9 * OPSIZ; |
| 86 | dstp -= 8 * OPSIZ; |
| 87 | a1 = ((op_t *) srcp)[8]; |
| 88 | len -= 1; |
| 89 | if (OP_T_THRES <= 3 * OPSIZ && len == 0) |
| 90 | goto do0; |
| 91 | goto do8; /* No-op. */ |
| 92 | } |
| 93 | |
| 94 | do |
| 95 | { |
| 96 | do8: |
| 97 | a0 = ((op_t *) srcp)[7]; |
| 98 | ((op_t *) dstp)[7] = a1; |
| 99 | do7: |
| 100 | a1 = ((op_t *) srcp)[6]; |
| 101 | ((op_t *) dstp)[6] = a0; |
| 102 | do6: |
| 103 | a0 = ((op_t *) srcp)[5]; |
| 104 | ((op_t *) dstp)[5] = a1; |
| 105 | do5: |
| 106 | a1 = ((op_t *) srcp)[4]; |
| 107 | ((op_t *) dstp)[4] = a0; |
| 108 | do4: |
| 109 | a0 = ((op_t *) srcp)[3]; |
| 110 | ((op_t *) dstp)[3] = a1; |
| 111 | do3: |
| 112 | a1 = ((op_t *) srcp)[2]; |
| 113 | ((op_t *) dstp)[2] = a0; |
| 114 | do2: |
| 115 | a0 = ((op_t *) srcp)[1]; |
| 116 | ((op_t *) dstp)[1] = a1; |
| 117 | do1: |
| 118 | a1 = ((op_t *) srcp)[0]; |
| 119 | ((op_t *) dstp)[0] = a0; |
| 120 | |
| 121 | srcp -= 8 * OPSIZ; |
| 122 | dstp -= 8 * OPSIZ; |
| 123 | len -= 8; |
| 124 | } |
| 125 | while (len != 0); |
| 126 | |
| 127 | /* This is the right position for do0. Please don't move |
| 128 | it into the loop. */ |
| 129 | do0: |
| 130 | ((op_t *) dstp)[7] = a1; |
| 131 | } |
| 132 | |
| 133 | /* _wordcopy_bwd_dest_aligned -- Copy block finishing right |
| 134 | before SRCP to block finishing right before DSTP with LEN `op_t' |
| 135 | words (not LEN bytes!). DSTP should be aligned for memory |
| 136 | operations on `op_t', but SRCP must *not* be aligned. */ |
| 137 | |
| 138 | static void _wordcopy_bwd_dest_aligned (long int dstp, long int srcp, size_t len) |
| 139 | { |
| 140 | op_t a0 = 0; |
| 141 | op_t a1 = 0; |
| 142 | op_t a2 = 0; |
| 143 | op_t a3 = 0; |
| 144 | int sh_1, sh_2; |
| 145 | |
| 146 | /* Calculate how to shift a word read at the memory operation |
| 147 | aligned srcp to make it aligned for copy. */ |
| 148 | |
| 149 | sh_1 = 8 * (srcp % OPSIZ); |
| 150 | sh_2 = 8 * OPSIZ - sh_1; |
| 151 | |
| 152 | /* Make srcp aligned by rounding it down to the beginning of the op_t |
| 153 | it points in the middle of. */ |
| 154 | srcp &= -OPSIZ; |
| 155 | srcp += OPSIZ; |
| 156 | |
| 157 | switch (len % 4) |
| 158 | { |
| 159 | case 2: |
| 160 | srcp -= 3 * OPSIZ; |
| 161 | dstp -= 1 * OPSIZ; |
| 162 | a2 = ((op_t *) srcp)[2]; |
| 163 | a1 = ((op_t *) srcp)[1]; |
| 164 | len += 2; |
| 165 | goto do1; |
| 166 | case 3: |
| 167 | srcp -= 4 * OPSIZ; |
| 168 | dstp -= 2 * OPSIZ; |
| 169 | a3 = ((op_t *) srcp)[3]; |
| 170 | a2 = ((op_t *) srcp)[2]; |
| 171 | len += 1; |
| 172 | goto do2; |
| 173 | case 0: |
| 174 | if (OP_T_THRES <= 3 * OPSIZ && len == 0) |
| 175 | return; |
| 176 | srcp -= 5 * OPSIZ; |
| 177 | dstp -= 3 * OPSIZ; |
| 178 | a0 = ((op_t *) srcp)[4]; |
| 179 | a3 = ((op_t *) srcp)[3]; |
| 180 | goto do3; |
| 181 | case 1: |
| 182 | srcp -= 6 * OPSIZ; |
| 183 | dstp -= 4 * OPSIZ; |
| 184 | a1 = ((op_t *) srcp)[5]; |
| 185 | a0 = ((op_t *) srcp)[4]; |
| 186 | len -= 1; |
| 187 | if (OP_T_THRES <= 3 * OPSIZ && len == 0) |
| 188 | goto do0; |
| 189 | goto do4; /* No-op. */ |
| 190 | } |
| 191 | |
| 192 | do |
| 193 | { |
| 194 | do4: |
| 195 | a3 = ((op_t *) srcp)[3]; |
| 196 | ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2); |
| 197 | do3: |
| 198 | a2 = ((op_t *) srcp)[2]; |
| 199 | ((op_t *) dstp)[2] = MERGE (a3, sh_1, a0, sh_2); |
| 200 | do2: |
| 201 | a1 = ((op_t *) srcp)[1]; |
| 202 | ((op_t *) dstp)[1] = MERGE (a2, sh_1, a3, sh_2); |
| 203 | do1: |
| 204 | a0 = ((op_t *) srcp)[0]; |
| 205 | ((op_t *) dstp)[0] = MERGE (a1, sh_1, a2, sh_2); |
| 206 | |
| 207 | srcp -= 4 * OPSIZ; |
| 208 | dstp -= 4 * OPSIZ; |
| 209 | len -= 4; |
| 210 | } |
| 211 | while (len != 0); |
| 212 | |
| 213 | /* This is the right position for do0. Please don't move |
| 214 | it into the loop. */ |
| 215 | do0: |
| 216 | ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2); |
| 217 | } |
| 218 | |
| 219 | void *memmove (void *dest, const void *src, size_t len) |
| 220 | { |
| 221 | unsigned long int dstp = (long int) dest; |
| 222 | unsigned long int srcp = (long int) src; |
| 223 | |
| 224 | /* This test makes the forward copying code be used whenever possible. |
| 225 | Reduces the working set. */ |
| 226 | if (dstp - srcp >= len) /* *Unsigned* compare! */ |
| 227 | { |
| 228 | #ifndef __ARCH_HAS_BWD_MEMCPY__ |
| 229 | /* Backward memcpy implementation cannot be used */ |
| 230 | memcpy(dest, src, len); |
| 231 | #else |
| 232 | /* Copy from the beginning to the end. */ |
| 233 | |
| 234 | /* If there not too few bytes to copy, use word copy. */ |
| 235 | if (len >= OP_T_THRES) |
| 236 | { |
| 237 | /* Copy just a few bytes to make DSTP aligned. */ |
| 238 | len -= (-dstp) % OPSIZ; |
| 239 | BYTE_COPY_FWD (dstp, srcp, (-dstp) % OPSIZ); |
| 240 | |
| 241 | /* Copy whole pages from SRCP to DSTP by virtual address |
| 242 | manipulation, as much as possible. */ |
| 243 | |
| 244 | PAGE_COPY_FWD_MAYBE (dstp, srcp, len, len); |
| 245 | |
| 246 | /* Copy from SRCP to DSTP taking advantage of the known |
| 247 | alignment of DSTP. Number of bytes remaining is put |
| 248 | in the third argument, i.e. in LEN. This number may |
| 249 | vary from machine to machine. */ |
| 250 | |
| 251 | WORD_COPY_FWD (dstp, srcp, len, len); |
| 252 | |
| 253 | /* Fall out and copy the tail. */ |
| 254 | } |
| 255 | |
| 256 | /* There are just a few bytes to copy. Use byte memory operations. */ |
| 257 | BYTE_COPY_FWD (dstp, srcp, len); |
| 258 | #endif |
| 259 | } |
| 260 | else |
| 261 | { |
| 262 | /* Copy from the end to the beginning. */ |
| 263 | srcp += len; |
| 264 | dstp += len; |
| 265 | |
| 266 | /* If there not too few bytes to copy, use word copy. */ |
| 267 | if (len >= OP_T_THRES) |
| 268 | { |
| 269 | /* Copy just a few bytes to make DSTP aligned. */ |
| 270 | len -= dstp % OPSIZ; |
| 271 | BYTE_COPY_BWD (dstp, srcp, dstp % OPSIZ); |
| 272 | |
| 273 | /* Copy from SRCP to DSTP taking advantage of the known |
| 274 | alignment of DSTP. Number of bytes remaining is put |
| 275 | in the third argument, i.e. in LEN. This number may |
| 276 | vary from machine to machine. */ |
| 277 | |
| 278 | WORD_COPY_BWD (dstp, srcp, len, len); |
| 279 | |
| 280 | /* Fall out and copy the tail. */ |
| 281 | } |
| 282 | |
| 283 | /* There are just a few bytes to copy. Use byte memory operations. */ |
| 284 | BYTE_COPY_BWD (dstp, srcp, len); |
| 285 | } |
| 286 | |
| 287 | return (dest); |
| 288 | } |
| 289 | libc_hidden_weak(memmove) |