1 | ! |
---|
2 | ! Fast SH memcpy |
---|
3 | ! |
---|
4 | ! by Toshiyasu Morita (tm@netcom.com) |
---|
5 | ! hacked by J"orn Rernnecke (joern.rennecke@superh.com) ("o for o-umlaut) |
---|
6 | ! SH5 code Copyright 2002 SuperH Ltd. |
---|
7 | ! |
---|
8 | ! Entry: ARG0: destination pointer |
---|
9 | ! ARG1: source pointer |
---|
10 | ! ARG3: byte count |
---|
11 | ! |
---|
12 | ! Exit: RESULT: destination pointer |
---|
13 | ! any other registers in the range r0-r7: trashed |
---|
14 | ! |
---|
15 | ! Notes: Usually one wants to do small reads and write a longword, but |
---|
16 | ! unfortunately it is difficult in some cases to concatanate bytes |
---|
17 | ! into a longword on the SH, so this does a longword read and small |
---|
18 | ! writes. |
---|
19 | ! |
---|
20 | ! This implementation makes two assumptions about how it is called: |
---|
21 | ! |
---|
22 | ! 1.: If the byte count is nonzero, the address of the last byte to be |
---|
23 | ! copied is unsigned greater than the address of the first byte to |
---|
24 | ! be copied. This could be easily swapped for a signed comparison, |
---|
25 | ! but the algorithm used needs some comparison. |
---|
26 | ! |
---|
27 | ! 2.: When there are two or three bytes in the last word of an 11-or-more |
---|
28 | ! bytes memory chunk to b copied, the rest of the word can be read |
---|
29 | ! without side effects. |
---|
30 | ! This could be easily changed by increasing the minumum size of |
---|
31 | ! a fast memcpy and the amount subtracted from r7 before L_2l_loop be 2, |
---|
32 | ! however, this would cost a few extra cyles on average. |
---|
33 | ! For SHmedia, the assumption is that any quadword can be read in its |
---|
34 | ! enirety if at least one byte is included in the copy. |
---|
35 | ! |
---|
36 | |
---|
37 | #include "asm.h" |
---|
38 | |
---|
39 | ENTRY(memcpy) |
---|
40 | |
---|
41 | #if __SHMEDIA__ |
---|
42 | |
---|
43 | #define LDUAQ(P,O,D0,D1) ldlo.q P,O,D0; ldhi.q P,O+7,D1 |
---|
44 | #define STUAQ(P,O,D0,D1) stlo.q P,O,D0; sthi.q P,O+7,D1 |
---|
45 | #define LDUAL(P,O,D0,D1) ldlo.l P,O,D0; ldhi.l P,O+3,D1 |
---|
46 | #define STUAL(P,O,D0,D1) stlo.l P,O,D0; sthi.l P,O+3,D1 |
---|
47 | |
---|
48 | ld.b r3,0,r63 |
---|
49 | pta/l Large,tr0 |
---|
50 | movi 25,r0 |
---|
51 | bgeu/u r4,r0,tr0 |
---|
52 | nsb r4,r0 |
---|
53 | shlli r0,5,r0 |
---|
54 | movi (L1-L0+63*32 + 1) & 0xffff,r1 |
---|
55 | sub r1, r0, r0 |
---|
56 | L0: ptrel r0,tr0 |
---|
57 | add r2,r4,r5 |
---|
58 | ptabs r18,tr1 |
---|
59 | add r3,r4,r6 |
---|
60 | blink tr0,r63 |
---|
61 | |
---|
62 | .balign 8 |
---|
63 | L1: |
---|
64 | /* 0 byte memcpy */ |
---|
65 | blink tr1,r63 |
---|
66 | |
---|
67 | L4_7: /* 4..7 byte memcpy cntd. */ |
---|
68 | stlo.l r2, 0, r0 |
---|
69 | or r6, r7, r6 |
---|
70 | sthi.l r5, -1, r6 |
---|
71 | stlo.l r5, -4, r6 |
---|
72 | blink tr1,r63 |
---|
73 | |
---|
74 | L2_3: /* 2 or 3 byte memcpy cntd. */ |
---|
75 | st.b r5,-1,r6 |
---|
76 | blink tr1,r63 |
---|
77 | |
---|
78 | /* 1 byte memcpy */ |
---|
79 | ld.b r3,0,r0 |
---|
80 | st.b r2,0,r0 |
---|
81 | blink tr1,r63 |
---|
82 | |
---|
83 | L8_15: /* 8..15 byte memcpy cntd. */ |
---|
84 | stlo.q r2, 0, r0 |
---|
85 | or r6, r7, r6 |
---|
86 | sthi.q r5, -1, r6 |
---|
87 | stlo.q r5, -8, r6 |
---|
88 | blink tr1,r63 |
---|
89 | |
---|
90 | /* 2 or 3 byte memcpy */ |
---|
91 | ld.b r3,0,r0 |
---|
92 | ld.b r2,0,r63 |
---|
93 | ld.b r3,1,r1 |
---|
94 | st.b r2,0,r0 |
---|
95 | pta/l L2_3,tr0 |
---|
96 | ld.b r6,-1,r6 |
---|
97 | st.b r2,1,r1 |
---|
98 | blink tr0, r63 |
---|
99 | |
---|
100 | /* 4 .. 7 byte memcpy */ |
---|
101 | LDUAL (r3, 0, r0, r1) |
---|
102 | pta L4_7, tr0 |
---|
103 | ldlo.l r6, -4, r7 |
---|
104 | or r0, r1, r0 |
---|
105 | sthi.l r2, 3, r0 |
---|
106 | ldhi.l r6, -1, r6 |
---|
107 | blink tr0, r63 |
---|
108 | |
---|
109 | /* 8 .. 15 byte memcpy */ |
---|
110 | LDUAQ (r3, 0, r0, r1) |
---|
111 | pta L8_15, tr0 |
---|
112 | ldlo.q r6, -8, r7 |
---|
113 | or r0, r1, r0 |
---|
114 | sthi.q r2, 7, r0 |
---|
115 | ldhi.q r6, -1, r6 |
---|
116 | blink tr0, r63 |
---|
117 | |
---|
118 | /* 16 .. 24 byte memcpy */ |
---|
119 | LDUAQ (r3, 0, r0, r1) |
---|
120 | LDUAQ (r3, 8, r8, r9) |
---|
121 | or r0, r1, r0 |
---|
122 | sthi.q r2, 7, r0 |
---|
123 | or r8, r9, r8 |
---|
124 | sthi.q r2, 15, r8 |
---|
125 | ldlo.q r6, -8, r7 |
---|
126 | ldhi.q r6, -1, r6 |
---|
127 | stlo.q r2, 8, r8 |
---|
128 | stlo.q r2, 0, r0 |
---|
129 | or r6, r7, r6 |
---|
130 | sthi.q r5, -1, r6 |
---|
131 | stlo.q r5, -8, r6 |
---|
132 | blink tr1,r63 |
---|
133 | |
---|
134 | Large: |
---|
135 | ld.b r2, 0, r63 |
---|
136 | pta/l Loop_ua, tr1 |
---|
137 | ori r3, -8, r7 |
---|
138 | sub r2, r7, r22 |
---|
139 | sub r3, r2, r6 |
---|
140 | add r2, r4, r5 |
---|
141 | ldlo.q r3, 0, r0 |
---|
142 | addi r5, -16, r5 |
---|
143 | movi 64+8, r27 // could subtract r7 from that. |
---|
144 | stlo.q r2, 0, r0 |
---|
145 | sthi.q r2, 7, r0 |
---|
146 | ldx.q r22, r6, r0 |
---|
147 | bgtu/l r27, r4, tr1 |
---|
148 | |
---|
149 | addi r5, -48, r27 |
---|
150 | pta/l Loop_line, tr0 |
---|
151 | addi r6, 64, r36 |
---|
152 | addi r6, -24, r19 |
---|
153 | addi r6, -16, r20 |
---|
154 | addi r6, -8, r21 |
---|
155 | |
---|
156 | Loop_line: |
---|
157 | ldx.q r22, r36, r63 |
---|
158 | alloco r22, 32 |
---|
159 | addi r22, 32, r22 |
---|
160 | ldx.q r22, r19, r23 |
---|
161 | sthi.q r22, -25, r0 |
---|
162 | ldx.q r22, r20, r24 |
---|
163 | ldx.q r22, r21, r25 |
---|
164 | stlo.q r22, -32, r0 |
---|
165 | ldx.q r22, r6, r0 |
---|
166 | sthi.q r22, -17, r23 |
---|
167 | sthi.q r22, -9, r24 |
---|
168 | sthi.q r22, -1, r25 |
---|
169 | stlo.q r22, -24, r23 |
---|
170 | stlo.q r22, -16, r24 |
---|
171 | stlo.q r22, -8, r25 |
---|
172 | bgeu r27, r22, tr0 |
---|
173 | |
---|
174 | Loop_ua: |
---|
175 | addi r22, 8, r22 |
---|
176 | sthi.q r22, -1, r0 |
---|
177 | stlo.q r22, -8, r0 |
---|
178 | ldx.q r22, r6, r0 |
---|
179 | bgtu/l r5, r22, tr1 |
---|
180 | |
---|
181 | add r3, r4, r7 |
---|
182 | ldlo.q r7, -8, r1 |
---|
183 | sthi.q r22, 7, r0 |
---|
184 | ldhi.q r7, -1, r7 |
---|
185 | ptabs r18,tr1 |
---|
186 | stlo.q r22, 0, r0 |
---|
187 | or r1, r7, r1 |
---|
188 | sthi.q r5, 15, r1 |
---|
189 | stlo.q r5, 8, r1 |
---|
190 | blink tr1, r63 |
---|
191 | |
---|
192 | #else /* ! SHMEDIA, i.e. SH1 .. SH4 / SHcompact */ |
---|
193 | |
---|
194 | #ifdef __SH5__ |
---|
195 | #define DST r2 |
---|
196 | #define SRC r3 |
---|
197 | #define COUNT r4 |
---|
198 | #define TMP0 r5 |
---|
199 | #define TMP1 r6 |
---|
200 | #define RESULT r2 |
---|
201 | #else |
---|
202 | #define DST r4 |
---|
203 | #define SRC r5 |
---|
204 | #define COUNT r6 |
---|
205 | #define TMP0 r2 |
---|
206 | #define TMP1 r3 |
---|
207 | #define RESULT r0 |
---|
208 | #endif |
---|
209 | |
---|
210 | #ifdef __LITTLE_ENDIAN__ |
---|
211 | ! Little endian version copies with increasing addresses. |
---|
212 | mov DST,TMP1 ! Save return value |
---|
213 | mov #11,r0 ! Check if small number of bytes |
---|
214 | cmp/hs r0,COUNT |
---|
215 | ! COUNT becomes src end address |
---|
216 | SL(bf, L_small, add SRC,COUNT) |
---|
217 | mov #1,r1 |
---|
218 | tst r1,SRC ! check if source even |
---|
219 | SL(bt, L_even, mov COUNT,r7) |
---|
220 | mov.b @SRC+,r0 ! no, make it even. |
---|
221 | mov.b r0,@DST |
---|
222 | add #1,DST |
---|
223 | L_even: tst r1,DST ! check if destination is even |
---|
224 | add #-3,r7 |
---|
225 | SL(bf, L_odddst, mov #2,r1) |
---|
226 | tst r1,DST ! check if destination is 4-byte aligned |
---|
227 | mov DST,r0 |
---|
228 | SL(bt, L_al4dst, sub SRC,r0) |
---|
229 | mov.w @SRC+,TMP0 |
---|
230 | mov.w TMP0,@DST |
---|
231 | ! add #2,DST DST is dead here. |
---|
232 | L_al4dst: |
---|
233 | tst r1,SRC |
---|
234 | bt L_al4both |
---|
235 | mov.w @SRC+,r1 |
---|
236 | swap.w r1,r1 |
---|
237 | add #-6,r0 |
---|
238 | add #-6,r7 ! r7 := src end address minus 9. |
---|
239 | .align 2 |
---|
240 | L_2l_loop: |
---|
241 | mov.l @SRC+,TMP0 ! Read & write two longwords per iteration |
---|
242 | xtrct TMP0,r1 |
---|
243 | mov.l r1,@(r0,SRC) |
---|
244 | cmp/hs r7,SRC |
---|
245 | mov.l @SRC+,r1 |
---|
246 | xtrct r1,TMP0 |
---|
247 | mov.l TMP0,@(r0,SRC) |
---|
248 | bf L_2l_loop |
---|
249 | add #-2,SRC |
---|
250 | bra L_cleanup |
---|
251 | add #5,r0 |
---|
252 | L_al4both: |
---|
253 | add #-4,r0 |
---|
254 | .align 2 |
---|
255 | L_al4both_loop: |
---|
256 | mov.l @SRC+,DST ! Read longword, write longword per iteration |
---|
257 | cmp/hs r7,SRC |
---|
258 | SL(bf, L_al4both_loop, mov.l DST,@(r0,SRC)) |
---|
259 | |
---|
260 | bra L_cleanup |
---|
261 | add #3,r0 |
---|
262 | |
---|
263 | L_odddst: |
---|
264 | tst r1,SRC |
---|
265 | SL(bt, L_al4src, add #-1,DST) |
---|
266 | mov.w @SRC+,r0 |
---|
267 | mov.b r0,@(1,DST) |
---|
268 | shlr8 r0 |
---|
269 | mov.b r0,@(2,DST) |
---|
270 | add #2,DST |
---|
271 | L_al4src: |
---|
272 | .align 2 |
---|
273 | L_odd_loop: |
---|
274 | mov.l @SRC+,r0 ! Read longword, write byte, word, byte per iteration |
---|
275 | cmp/hs r7,SRC |
---|
276 | mov.b r0,@(1,DST) |
---|
277 | shlr8 r0 |
---|
278 | mov.w r0,@(2,DST) |
---|
279 | shlr16 r0 |
---|
280 | mov.b r0,@(4,DST) |
---|
281 | SL(bf, L_odd_loop, add #4,DST) |
---|
282 | .align 2 ! avoid nop in more frequently executed code. |
---|
283 | L_cleanup2: |
---|
284 | mov DST,r0 |
---|
285 | sub SRC,r0 |
---|
286 | L_cleanup: |
---|
287 | cmp/eq COUNT,SRC |
---|
288 | bt L_ready |
---|
289 | .align 2 |
---|
290 | L_cleanup_loop: |
---|
291 | mov.b @SRC+,r1 |
---|
292 | cmp/eq COUNT,SRC |
---|
293 | mov.b r1,@(r0,SRC) |
---|
294 | bf L_cleanup_loop |
---|
295 | L_ready: |
---|
296 | rts |
---|
297 | mov TMP1,RESULT |
---|
298 | L_small: |
---|
299 | bra L_cleanup2 |
---|
300 | add #-1,DST |
---|
301 | #else /* ! __LITTLE_ENDIAN__ */ |
---|
302 | ! Big endian version copies with decreasing addresses. |
---|
303 | mov DST,r0 |
---|
304 | add COUNT,r0 |
---|
305 | sub DST,SRC |
---|
306 | mov #11,r1 |
---|
307 | cmp/hs r1,COUNT |
---|
308 | SL(bf, L_small, add #-1,SRC) |
---|
309 | mov SRC,TMP1 |
---|
310 | add r0,TMP1 |
---|
311 | shlr TMP1 |
---|
312 | SL(bt, L_even, |
---|
313 | mov DST,r7) |
---|
314 | mov.b @(r0,SRC),TMP0 |
---|
315 | add #-1,TMP1 |
---|
316 | mov.b TMP0,@-r0 |
---|
317 | L_even: |
---|
318 | tst #1,r0 |
---|
319 | add #-1,SRC |
---|
320 | SL(bf, L_odddst, add #8,r7) |
---|
321 | tst #2,r0 |
---|
322 | bt L_al4dst |
---|
323 | add #-1,TMP1 |
---|
324 | mov.w @(r0,SRC),r1 |
---|
325 | mov.w r1,@-r0 |
---|
326 | L_al4dst: |
---|
327 | shlr TMP1 |
---|
328 | bt L_al4both |
---|
329 | mov.w @(r0,SRC),r1 |
---|
330 | swap.w r1,r1 |
---|
331 | add #4,r7 |
---|
332 | add #-4,SRC |
---|
333 | .align 2 |
---|
334 | L_2l_loop: |
---|
335 | mov.l @(r0,SRC),TMP0 |
---|
336 | xtrct TMP0,r1 |
---|
337 | mov.l r1,@-r0 |
---|
338 | cmp/hs r7,r0 |
---|
339 | mov.l @(r0,SRC),r1 |
---|
340 | xtrct r1,TMP0 |
---|
341 | mov.l TMP0,@-r0 |
---|
342 | bt L_2l_loop |
---|
343 | bra L_cleanup |
---|
344 | add #5,SRC |
---|
345 | |
---|
346 | nop ! avoid nop in executed code. |
---|
347 | L_al4both: |
---|
348 | add #-2,SRC |
---|
349 | .align 2 |
---|
350 | L_al4both_loop: |
---|
351 | mov.l @(r0,SRC),r1 |
---|
352 | cmp/hs r7,r0 |
---|
353 | SL(bt, L_al4both_loop, |
---|
354 | mov.l r1,@-r0) |
---|
355 | bra L_cleanup |
---|
356 | add #3,SRC |
---|
357 | |
---|
358 | nop ! avoid nop in executed code. |
---|
359 | L_odddst: |
---|
360 | shlr TMP1 |
---|
361 | bt L_al4src |
---|
362 | mov.w @(r0,SRC),r1 |
---|
363 | mov.b r1,@-r0 |
---|
364 | shlr8 r1 |
---|
365 | mov.b r1,@-r0 |
---|
366 | L_al4src: |
---|
367 | add #-2,SRC |
---|
368 | .align 2 |
---|
369 | L_odd_loop: |
---|
370 | mov.l @(r0,SRC),TMP0 |
---|
371 | cmp/hs r7,r0 |
---|
372 | mov.b TMP0,@-r0 |
---|
373 | shlr8 TMP0 |
---|
374 | mov.w TMP0,@-r0 |
---|
375 | shlr16 TMP0 |
---|
376 | mov.b TMP0,@-r0 |
---|
377 | bt L_odd_loop |
---|
378 | |
---|
379 | add #3,SRC |
---|
380 | L_cleanup: |
---|
381 | L_small: |
---|
382 | cmp/eq DST,r0 |
---|
383 | bt L_ready |
---|
384 | add #1,DST |
---|
385 | .align 2 |
---|
386 | L_cleanup_loop: |
---|
387 | mov.b @(r0,SRC),TMP0 |
---|
388 | cmp/eq DST,r0 |
---|
389 | mov.b TMP0,@-r0 |
---|
390 | bf L_cleanup_loop |
---|
391 | L_ready: |
---|
392 | rts |
---|
393 | mov r0,RESULT |
---|
394 | #endif /* ! __LITTLE_ENDIAN__ */ |
---|
395 | #endif /* ! SHMEDIA */ |
---|