/* * Copyright (c) 2002 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * The contents of this file constitute Original Code as defined in and * are subject to the Apple Public Source License Version 1.1 (the * "License"). You may not use this file except in compliance with the * License. Please obtain a copy of the License at * http://www.apple.com/publicsource and read it before using this file. * * This Original Code and all software distributed under the License are * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the * License for the specific language governing rights and limitations * under the License. * * @APPLE_LICENSE_HEADER_END@ */ ; ; Copy bytes of data around. handles overlapped data. ; ; Change this to use Altivec later on, and maybe floating point. ; ; #include <ppc/asm.h> #include <ppc/proc_reg.h> #include <assym.s> ; Use CR5_lt to indicate non-cached #define noncache 20 ; Use CR5_gt to indicate that we need to turn data translation back on #define fixxlate 21 ; Use CR5_eq to indicate that we need to invalidate bats (if 32-bit) or turn off ; 64-bit mode (if 64-bit) before returning to our caller. We overload the ; bit to reduce the number of conditional branches at bcopy exit. #define restorex 22 ; Use CR5_so to indicate that we need to restore real-mode cachability ; Only needed on 64-bit machines #define flipcache 23 ; ; bcopy_nc(from, to, nbytes) ; ; bcopy_nc operates on non-cached memory so we can not use any kind ; of cache instructions. ; .align 5 .globl EXT(bcopy_nc) LEXT(bcopy_nc) crset noncache ; Set non-cached b bcpswap ; ; void bcopy_physvir(from, to, nbytes) ; Attempt to copy physically addressed memory with translation on if conditions are met. ; Otherwise do a normal bcopy_phys. This routine is used because some 32-bit processors ; are very slow doing real-mode (translation off) copies, so we set up temporary BATs ; for the passed phys addrs and do the copy with translation on. ; ; Rules are: neither source nor destination can cross a page. ; ; Interrupts must be disabled throughout the copy when this is called. ; To do this, we build a ; 128 DBAT for both the source and sink. If both are the same, only one is ; loaded. We do not touch the IBATs, so there is no issue if either physical page ; address is the same as the virtual address of the instructions we are executing. ; ; At the end, we invalidate the used DBATs. ; ; Note that the address parameters are long longs. We will transform these to 64-bit ; values. Note that on 32-bit architectures that this will ignore the high half of the ; passed in value. This should be ok since we can not have any bigger than 32 bit addresses ; there anyhow. ; ; Note, this one will not work in user state ; .align 5 .globl EXT(bcopy_physvir) LEXT(bcopy_physvir) crclr flipcache ; (HACK) No cache flip needed mfsprg r8,2 ; get processor feature flags rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg addic. r0,r7,-1 ; Get length - 1 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits add r11,r3,r0 ; Point to last byte of sink rlwinm r4,r5,0,1,0 ; Duplicate high half of long long paddr into top of reg mtcrf 0x02,r8 ; move pf64Bit to cr6 so we can test rlwimi r4,r6,0,0,31 ; Combine bottom of long long to full 64-bits mr r5,r7 ; Get the length into the right register cmplw cr1,r3,r4 ; Does source == sink? bt++ pf64Bitb,bcopy_phys1 ; if 64-bit processor, use standard routine (no BATs) add r12,r4,r0 ; Point to last byte of source bltlr- ; Bail if length is 0 or way too big xor r7,r11,r3 ; See if we went to next page xor r8,r12,r4 ; See if we went to next page or r0,r7,r8 ; Combine wrap // li r9,((PTE_WIMG_CB_CACHED_COHERENT<<3)|2) ; Set default attributes li r9,((2<<3)|2) ; Set default attributes rlwinm. r0,r0,0,0,19 ; Did we overflow a page? li r7,2 ; Set validity flags li r8,2 ; Set validity flags bne- bcopy_phys1 ; Overflowed page, do normal physical copy... crset restorex ; Remember to trash BATs on the way out rlwimi r11,r9,0,15,31 ; Set sink lower DBAT value rlwimi r12,r9,0,15,31 ; Set source lower DBAT value rlwimi r7,r11,0,0,14 ; Set sink upper DBAT value rlwimi r8,r12,0,0,14 ; Set source upper DBAT value cmplw cr1,r11,r12 ; See if sink and source are same block sync mtdbatl 0,r11 ; Set sink lower DBAT mtdbatu 0,r7 ; Set sink upper DBAT beq- cr1,bcpvsame ; Source and sink are in same block mtdbatl 1,r12 ; Set source lower DBAT mtdbatu 1,r8 ; Set source upper DBAT bcpvsame: sync ; wait for BAT to stabilize isync mr r6,r3 ; Set source crclr noncache ; Set cached crclr fixxlate ; Set translation already ok b copyit32 ; Go copy it... ; ; void bcopy_phys(from, to, nbytes) ; Turns off data translation before the copy. Note, this one will ; not work in user state. This routine is used on 32 and 64-bit ; machines. ; ; Note that the address parameters are long longs. We will transform these to 64-bit ; values. Note that on 32-bit architectures that this will ignore the high half of the ; passed in value. This should be ok since we can not have any bigger than 32 bit addresses ; there anyhow. ; ; Also note that you probably will not be happy if either the sink or source spans across the ; boundary between RAM and I/O space. Good chance of hanging the machine and this code ; will not check, so be careful. ; .align 5 .globl EXT(bcopy_phys) LEXT(bcopy_phys) crclr flipcache ; (HACK) No cache flip needed rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg mfsprg r8,2 ; get processor feature flags rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits rlwinm r4,r5,0,1,0 ; Duplicate high half of long long paddr into top of reg mtcrf 0x02,r8 ; move pf64Bit to cr6 so we can test rlwimi r4,r6,0,0,31 ; Combine bottom of long long to full 64-bits mr r5,r7 ; Get the length into the right register bcopy_phys1: ; enter from bcopy_physvir with pf64Bit already in cr6 mfmsr r9 ; Get the MSR crclr noncache ; Set cached bt++ pf64Bitb,bcopy_phys64 ; skip if 64-bit (only they take hint) ; 32-bit CPUs sub. r0,r3,r4 ; to==from? rlwinm r8,r9,0,MSR_DR_BIT,MSR_DR_BIT ; was translation on? cmpwi cr1,r8,0 ; set cr1 beq if translation was off oris r8,r8,hi16(MASK(MSR_VEC)) ; Get vector enable cmplwi cr7,r5,0 ; Check if we have a 0 length beqlr- ; bail if to==from ori r8,r8,lo16(MASK(MSR_FP)) ; Get FP mr r6,r3 ; Set source andc r9,r9,r8 ; Turn off translation if it is on (should be) and FP, VEC beqlr- cr7 ; Bail if length is 0 crclr restorex ; Make sure we do not trash BATs on the way out mtmsr r9 ; Set DR translation off isync ; Wait for it crnot fixxlate,cr1_eq ; Remember to turn on translation if it was b copyit32 ; Go copy it... ; 64-bit: turn DR off and SF on, remember if we need to restore on way out. bcopy_phys64: ; r9 = MSR srdi r2,r3,31 ; (HACK) Get a 1 if source is in I/O memory srdi. r0,r9,63-MSR_SF_BIT ; set cr0 beq on if SF was off when we were called rlwinm r8,r9,MSR_DR_BIT+1,31,31 ; r8 <- DR bit right justified cmpld cr1,r3,r4 ; to==from? li r0,1 ; Note - we use this in a couple places below lis r6,hi16(MASK(MSR_VEC)) ; Get vector enable cmpwi cr7,r5,0 ; length==0 ? ori r6,r6,lo16(MASK(MSR_FP)|MASK(MSR_DR)) ; Add in FP and DR beqlr-- cr1 ; bail if to==from srdi r10,r4,31 ; (HACK) Get a 1 if sink is in I/O memory rldimi r9,r0,63,MSR_SF_BIT ; set SF on beqlr-- cr7 ; bail if length==0 andc r9,r9,r6 ; turn DR, VEC, FP off cmpwi cr1,r8,0 ; was DR on? crmove restorex,cr0_eq ; if SF was off, remember to turn back off before we return mtmsrd r9 ; turn 64-bit addressing on, data translation off cmpldi cr0,r2,1 ; (HACK) Is source in I/O memory? isync ; wait for it to happen mr r6,r3 ; Set source cmpldi cr7,r10,1 ; (HACK) Is sink in I/O memory? crnot fixxlate,cr1_eq ; if DR was on, remember to turn back on before we return cror flipcache,cr0_eq,cr7_eq ; (HACK) See if either source or sink is in I/O area rlwinm r10,r9,MSR_EE_BIT+1,31,31 ; (HACK GLORIOUS HACK) Isolate the EE bit sldi r11,r0,31-MSR_EE_BIT ; (HACK GLORIOUS HACK)) Get a mask for the EE bit sldi r0,r0,32+8 ; (HACK) Get the right bit to turn off caching bf++ flipcache,copyit64 ; (HACK) No need to mess with caching... ; ; HACK GLORIOUS HACK - when we force of caching, we need to also force off ; interruptions. We are out of CR bits, so we need to stash the entry EE ; somewheres. It is in the XER.... We NEED to change this!!!! ; mtxer r10 ; (HACK GLORIOUS HACK) Remember EE andc r9,r9,r11 ; (HACK GLORIOUS HACK) Turn off EE bit mfspr r2,hid4 ; (HACK) Get HID4 crset noncache ; (HACK) Set non-cached mtmsrd r9 ; (HACK GLORIOUS HACK) Force off EE or r2,r2,r0 ; (HACK) Set bit to make real accesses cache-inhibited sync ; (HACK) Sync up li r0,1 mtspr hid4,r2 ; (HACK) Make real accesses cache-inhibited isync ; (HACK) Toss prefetches lis r12,0xE000 ; (HACK) Get the unlikeliest ESID possible srdi r12,r12,1 ; (HACK) Make 0x7FFFFFFFF0000000 slbie r12 ; (HACK) Make sure the ERAT is cleared sync ; (HACK) isync ; (HACK) b copyit64 ; ; void bcopy(from, to, nbytes) ; .align 5 .globl EXT(bcopy) LEXT(bcopy) crclr noncache ; Set cached bcpswap: crclr flipcache ; (HACK) No cache flip needed mfsprg r8,2 ; get processor feature flags sub. r0,r4,r3 ; test for to==from in mode-independent way mtcrf 0x02,r8 ; move pf64Bit to cr6 so we can test cmpwi cr1,r5,0 ; Check if we have a 0 length crclr restorex ; Make sure we do not trash BATs on the way out mr r6,r3 ; Set source crclr fixxlate ; Set translation already ok beqlr- ; Bail if "to" and "from" are the same beqlr- cr1 ; Bail if length is 0 bt++ pf64Bitb,copyit64 ; handle 64-bit processor b copyit32 ; Go copy it... ; ; When we move the memory, forward overlays must be handled. We ; also can not use the cache instructions if we are from bcopy_nc. ; We need to preserve R3 because it needs to be returned for memcpy. ; We can be interrupted and lose control here. ; ; There is no stack, so in order to use vectors, we would ; need to take the vector exception. Any potential gains by using vectors ; would be more than eaten up by this. ; ; NOTE: this code is called in three "modes": ; - on 32-bit processors (32-byte cache line) ; - on 64-bit processors running in 32-bit mode (128-byte cache line) ; - on 64-bit processors running in 64-bit mode (128-byte cache line) ; ; ALSO NOTE: bcopy is called from copyin and copyout etc ; with the "thread_recover" ptr set. This means bcopy must not set up a ; stack frame or touch non-volatile registers, and also means that it ; cannot rely on turning off interrupts, because we expect to get DSIs ; and have execution aborted by a "longjmp" to the thread_recover ; routine. ; .align 5 .globl EXT(memcpy) ; NB: memcpy is only called in 32-bit mode, albeit on both 32- and 64-bit ; processors... LEXT(memcpy) crclr flipcache ; (HACK) No cache flip needed mfsprg r8,2 ; get processor feature flags cmplw cr1,r3,r4 ; "to" and "from" the same? mtcrf 0x02,r8 ; move pf64Bit to cr6 so we can test mr r6,r4 ; Set the "from" mr. r5,r5 ; Length zero? crclr noncache ; Set cached mr r4,r3 ; Set the "to" crclr fixxlate ; Set translation already ok beqlr- cr1 ; "to" and "from" are the same beqlr- ; Length is 0 crclr restorex ; Make sure we do not trash BATs on the way out bt++ pf64Bitb,copyit64 ; handle 64-bit processors copyit32: sub r12,r4,r6 ; Get potential overlap (negative if backward move) lis r8,0x7FFF ; Start up a mask srawi r11,r12,31 ; Propagate the sign bit dcbt br0,r6 ; Touch in the first source line cntlzw r7,r5 ; Get the highest power of 2 factor of the length ori r8,r8,0xFFFF ; Make limit 0x7FFFFFFF xor r9,r12,r11 ; If sink - source was negative, invert bits srw r8,r8,r7 ; Get move length limitation sub r9,r9,r11 ; If sink - source was negative, add 1 and get absolute value cmplw r12,r5 ; See if we actually forward overlap cmplwi cr7,r9,32 ; See if at least a line between source and sink dcbtst br0,r4 ; Touch in the first sink line cmplwi cr1,r5,32 ; Are we moving more than a line? cror noncache,noncache,cr7_lt ; Set to not DCBZ output line if not enough space blt- fwdovrlap ; This is a forward overlapping area, handle it... ; ; R4 = sink ; R5 = length ; R6 = source ; ; ; Here we figure out how much we have to move to get the sink onto a ; cache boundary. If we can, and there are still more that 32 bytes ; left to move, we can really speed things up by DCBZing the sink line. ; We can not do this if noncache is set because we will take an ; alignment exception. G4word: ; enter from 64-bit case with word aligned uncached operands neg r0,r4 ; Get the number of bytes to move to align to a line boundary rlwinm. r0,r0,0,27,31 ; Clean it up and test it and r0,r0,r8 ; limit to the maximum front end move mtcrf 3,r0 ; Make branch mask for partial moves sub r5,r5,r0 ; Set the length left to move beq alline ; Already on a line... bf 31,alhalf ; No single byte to do... lbz r7,0(r6) ; Get the byte addi r6,r6,1 ; Point to the next stb r7,0(r4) ; Save the single addi r4,r4,1 ; Bump sink ; Sink is halfword aligned here alhalf: bf 30,alword ; No halfword to do... lhz r7,0(r6) ; Get the halfword addi r6,r6,2 ; Point to the next sth r7,0(r4) ; Save the halfword addi r4,r4,2 ; Bump sink ; Sink is word aligned here alword: bf 29,aldouble ; No word to do... lwz r7,0(r6) ; Get the word addi r6,r6,4 ; Point to the next stw r7,0(r4) ; Save the word addi r4,r4,4 ; Bump sink ; Sink is double aligned here aldouble: bf 28,alquad ; No double to do... lwz r7,0(r6) ; Get the first word lwz r8,4(r6) ; Get the second word addi r6,r6,8 ; Point to the next stw r7,0(r4) ; Save the first word stw r8,4(r4) ; Save the second word addi r4,r4,8 ; Bump sink ; Sink is quadword aligned here alquad: bf 27,alline ; No quad to do... lwz r7,0(r6) ; Get the first word lwz r8,4(r6) ; Get the second word lwz r9,8(r6) ; Get the third word stw r7,0(r4) ; Save the first word lwz r11,12(r6) ; Get the fourth word addi r6,r6,16 ; Point to the next stw r8,4(r4) ; Save the second word stw r9,8(r4) ; Save the third word stw r11,12(r4) ; Save the fourth word addi r4,r4,16 ; Bump sink ; Sink is line aligned here alline: rlwinm. r0,r5,27,5,31 ; Get the number of full lines to move mtcrf 3,r5 ; Make branch mask for backend partial moves rlwinm r11,r5,0,0,26 ; Get number of bytes we are going to move beq- backend ; No full lines to move sub r5,r5,r11 ; Calculate the residual li r10,96 ; Stride for touch ahead nxtline: subic. r0,r0,1 ; Account for the line now bt- noncache,skipz ; Skip if we are not cached... dcbz br0,r4 ; Blow away the whole line because we are replacing it dcbt r6,r10 ; Touch ahead a bit skipz: lwz r7,0(r6) ; Get the first word lwz r8,4(r6) ; Get the second word lwz r9,8(r6) ; Get the third word stw r7,0(r4) ; Save the first word lwz r11,12(r6) ; Get the fourth word stw r8,4(r4) ; Save the second word lwz r7,16(r6) ; Get the fifth word stw r9,8(r4) ; Save the third word lwz r8,20(r6) ; Get the sixth word stw r11,12(r4) ; Save the fourth word lwz r9,24(r6) ; Get the seventh word stw r7,16(r4) ; Save the fifth word lwz r11,28(r6) ; Get the eighth word addi r6,r6,32 ; Point to the next stw r8,20(r4) ; Save the sixth word stw r9,24(r4) ; Save the seventh word stw r11,28(r4) ; Save the eighth word addi r4,r4,32 ; Bump sink bgt+ nxtline ; Do the next line, if any... ; Move backend quadword backend: bf 27,noquad ; No quad to do... lwz r7,0(r6) ; Get the first word lwz r8,4(r6) ; Get the second word lwz r9,8(r6) ; Get the third word lwz r11,12(r6) ; Get the fourth word stw r7,0(r4) ; Save the first word addi r6,r6,16 ; Point to the next stw r8,4(r4) ; Save the second word stw r9,8(r4) ; Save the third word stw r11,12(r4) ; Save the fourth word addi r4,r4,16 ; Bump sink ; Move backend double noquad: bf 28,nodouble ; No double to do... lwz r7,0(r6) ; Get the first word lwz r8,4(r6) ; Get the second word addi r6,r6,8 ; Point to the next stw r7,0(r4) ; Save the first word stw r8,4(r4) ; Save the second word addi r4,r4,8 ; Bump sink ; Move backend word nodouble: bf 29,noword ; No word to do... lwz r7,0(r6) ; Get the word addi r6,r6,4 ; Point to the next stw r7,0(r4) ; Save the word addi r4,r4,4 ; Bump sink ; Move backend halfword noword: bf 30,nohalf ; No halfword to do... lhz r7,0(r6) ; Get the halfword addi r6,r6,2 ; Point to the next sth r7,0(r4) ; Save the halfword addi r4,r4,2 ; Bump sink ; Move backend byte nohalf: bf 31,bcpydone ; Leave cuz we are all done... lbz r7,0(r6) ; Get the byte stb r7,0(r4) ; Save the single bcpydone: mfmsr r9 ; Get the MSR bf++ flipcache,bcpydone0 ; (HACK) No need to mess with caching... li r0,1 ; (HACK) Get a 1 mfxer r10 ; (HACK GLORIOUS HACK) Get the entry EE sldi r0,r0,32+8 ; (HACK) Get the right bit to turn off caching mfspr r2,hid4 ; (HACK) Get HID4 rlwinm r10,r10,31-MSR_EE_BIT,MSR_EE_BIT,MSR_EE_BIT ; (HACK GLORIOUS HACK) Set the EE bit andc r2,r2,r0 ; (HACK) Clear bit to make real accesses cache-inhibited or r9,r9,r10 ; (HACK GLORIOUS HACK) Set the EE in MSR sync ; (HACK) Sync up mtspr hid4,r2 ; (HACK) Make real accesses not cache-inhibited isync ; (HACK) Toss prefetches lis r12,0xE000 ; (HACK) Get the unlikeliest ESID possible srdi r12,r12,1 ; (HACK) Make 0x7FFFFFFFF0000000 slbie r12 ; (HACK) Make sure the ERAT is cleared mtmsr r9 ; (HACK GLORIOUS HACK) Set EE properly bcpydone0: lis r0,hi16(MASK(MSR_VEC)) ; Get the vector bit ori r0,r0,lo16(MASK(MSR_FP)) ; Get the float bit bf++ fixxlate,bcpydone1 ; skip if we do not need to fix translation... ori r9,r9,lo16(MASK(MSR_DR)) ; Turn data translation on andc r9,r9,r0 ; Make sure that FP and VEC are off mtmsr r9 ; Just do it isync ; Hang in there bcpydone1: bflr++ restorex ; done if we do not have to fix up addressing mfsprg r8,2 ; get the feature flags again mtcrf 0x02,r8 ; put pf64Bit where we can test it bt++ pf64Bitb,bcpydone2 ; skip if 64-bit processor ; 32-bit processor, so clear out the BATs we set up for bcopy_physvir li r0,0 ; Get set to invalidate upper half sync ; Make sure all is well mtdbatu 0,r0 ; Clear sink upper DBAT mtdbatu 1,r0 ; Clear source upper DBAT sync isync blr ; 64-bit processor, so turn off 64-bit mode we turned on to do bcopy_phys bcpydone2: mfmsr r9 ; get MSR again andc r9,r9,r0 ; Make sure that FP and VEC are off rldicl r9,r9,0,MSR_SF_BIT+1 ; clear SF mtmsrd r9 isync blr ; ; 0123456789ABCDEF0123456789ABCDEF ; 0123456789ABCDEF0123456789ABCDEF ; F ; DE ; 9ABC ; 12345678 ; 123456789ABCDEF0 ; 0 ; ; Here is where we handle a forward overlapping move. These will be slow ; because we can not kill the cache of the destination until after we have ; loaded/saved the source area. Also, because reading memory backwards is ; slower when the cache line needs to be loaded because the critical ; doubleword is loaded first, i.e., the last, then it goes back to the first, ; and on in order. That means that when we are at the second to last DW we ; have to wait until the whole line is in cache before we can proceed. ; G4reverseWord: ; here from 64-bit code with word aligned uncached operands fwdovrlap: add r4,r5,r4 ; Point past the last sink byte add r6,r5,r6 ; Point past the last source byte and r0,r4,r8 ; Apply movement limit li r12,-1 ; Make sure we touch in the actual line mtcrf 3,r0 ; Figure out the best way to move backwards dcbt r12,r6 ; Touch in the last line of source rlwinm. r0,r0,0,27,31 ; Calculate the length to adjust to cache boundary dcbtst r12,r4 ; Touch in the last line of the sink beq- balline ; Aready on cache line boundary sub r5,r5,r0 ; Precaculate move length left after alignment bf 31,balhalf ; No single byte to do... lbz r7,-1(r6) ; Get the byte subi r6,r6,1 ; Point to the next stb r7,-1(r4) ; Save the single subi r4,r4,1 ; Bump sink ; Sink is halfword aligned here balhalf: bf 30,balword ; No halfword to do... lhz r7,-2(r6) ; Get the halfword subi r6,r6,2 ; Point to the next sth r7,-2(r4) ; Save the halfword subi r4,r4,2 ; Bump sink ; Sink is word aligned here balword: bf 29,baldouble ; No word to do... lwz r7,-4(r6) ; Get the word subi r6,r6,4 ; Point to the next stw r7,-4(r4) ; Save the word subi r4,r4,4 ; Bump sink ; Sink is double aligned here baldouble: bf 28,balquad ; No double to do... lwz r7,-8(r6) ; Get the first word lwz r8,-4(r6) ; Get the second word subi r6,r6,8 ; Point to the next stw r7,-8(r4) ; Save the first word stw r8,-4(r4) ; Save the second word subi r4,r4,8 ; Bump sink ; Sink is quadword aligned here balquad: bf 27,balline ; No quad to do... lwz r7,-16(r6) ; Get the first word lwz r8,-12(r6) ; Get the second word lwz r9,-8(r6) ; Get the third word lwz r11,-4(r6) ; Get the fourth word stw r7,-16(r4) ; Save the first word subi r6,r6,16 ; Point to the next stw r8,-12(r4) ; Save the second word stw r9,-8(r4) ; Save the third word stw r11,-4(r4) ; Save the fourth word subi r4,r4,16 ; Bump sink ; Sink is line aligned here balline: rlwinm. r0,r5,27,5,31 ; Get the number of full lines to move mtcrf 3,r5 ; Make branch mask for backend partial moves beq- bbackend ; No full lines to move ; Registers in use: R0, R1, R3, R4, R5, R6 ; Registers not in use: R2, R7, R8, R9, R10, R11, R12 - Ok, we can make another free for 8 of them bnxtline: subic. r0,r0,1 ; Account for the line now lwz r7,-32(r6) ; Get the first word lwz r5,-28(r6) ; Get the second word lwz r2,-24(r6) ; Get the third word lwz r12,-20(r6) ; Get the third word lwz r11,-16(r6) ; Get the fifth word lwz r10,-12(r6) ; Get the sixth word lwz r9,-8(r6) ; Get the seventh word lwz r8,-4(r6) ; Get the eighth word subi r6,r6,32 ; Point to the next stw r7,-32(r4) ; Get the first word ble- bnotouch ; Last time, skip touch of source... dcbt br0,r6 ; Touch in next source line bnotouch: stw r5,-28(r4) ; Get the second word stw r2,-24(r4) ; Get the third word stw r12,-20(r4) ; Get the third word stw r11,-16(r4) ; Get the fifth word stw r10,-12(r4) ; Get the sixth word stw r9,-8(r4) ; Get the seventh word stw r8,-4(r4) ; Get the eighth word subi r4,r4,32 ; Bump sink bgt+ bnxtline ; Do the next line, if any... ; ; Note: We touched these lines in at the beginning ; ; Move backend quadword bbackend: bf 27,bnoquad ; No quad to do... lwz r7,-16(r6) ; Get the first word lwz r8,-12(r6) ; Get the second word lwz r9,-8(r6) ; Get the third word lwz r11,-4(r6) ; Get the fourth word stw r7,-16(r4) ; Save the first word subi r6,r6,16 ; Point to the next stw r8,-12(r4) ; Save the second word stw r9,-8(r4) ; Save the third word stw r11,-4(r4) ; Save the fourth word subi r4,r4,16 ; Bump sink ; Move backend double bnoquad: bf 28,bnodouble ; No double to do... lwz r7,-8(r6) ; Get the first word lwz r8,-4(r6) ; Get the second word subi r6,r6,8 ; Point to the next stw r7,-8(r4) ; Save the first word stw r8,-4(r4) ; Save the second word subi r4,r4,8 ; Bump sink ; Move backend word bnodouble: bf 29,bnoword ; No word to do... lwz r7,-4(r6) ; Get the word subi r6,r6,4 ; Point to the next stw r7,-4(r4) ; Save the word subi r4,r4,4 ; Bump sink ; Move backend halfword bnoword: bf 30,bnohalf ; No halfword to do... lhz r7,-2(r6) ; Get the halfword subi r6,r6,2 ; Point to the next sth r7,-2(r4) ; Save the halfword subi r4,r4,2 ; Bump sink ; Move backend byte bnohalf: bf 31,bcpydone ; Leave cuz we are all done... lbz r7,-1(r6) ; Get the byte stb r7,-1(r4) ; Save the single b bcpydone ; Go exit cuz we are all done... // Here on 64-bit processors, which have a 128-byte cache line. This can be // called either in 32 or 64-bit mode, which makes the test for reverse moves // a little tricky. We've already filtered out the (sou==dest) and (len==0) // special cases. // // When entered: // r4 = destination (32 or 64-bit ptr) // r5 = length (always 32 bits) // r6 = source (32 or 64-bit ptr) // cr5 = noncache, fixxlate, flipcache, and restorex flags set .align 5 copyit64: lis r2,0x4000 // r2 = 0x00000000 40000000 neg r12,r4 // start to compute #bytes to align dest bt-- noncache,noncache1 // (HACK) Do not even try anything cached... dcbt 0,r6 // touch in 1st block of source noncache1: add. r2,r2,r2 // if 0x00000000 80000000 < 0, we are in 32-bit mode cntlzw r9,r5 // get highest power-of-2 in length rlwinm r7,r12,0,25,31 // r7 <- bytes to 128-byte align dest bt-- noncache,noncache2 // (HACK) Do not even try anything cached... dcbtst 0,r4 // touch in 1st destination cache block noncache2: sraw r2,r2,r9 // get mask with 1s for leading 0s in length, plus 1 more 1-bit bge copyit64a // skip if we are running in 64-bit mode rlwinm r4,r4,0,0,31 // running in 32-bit mode, so truncate ptrs and lengths to 32 bits rlwinm r5,r5,0,0,31 rlwinm r6,r6,0,0,31 copyit64a: // now we can use 64-bit compares even if running in 32-bit mode sub r8,r4,r6 // get (dest-source) andc r7,r7,r2 // limit bytes to align by operand length cmpld cr1,r8,r5 // if (dest-source)<length, must move reverse bt-- noncache,c64uncached // skip if uncached blt-- cr1,c64rdouble // handle cached reverse moves // Forward, cached or doubleword aligned uncached. This is the common case. // r4-r6 = dest, length, source (as above) // r7 = #bytes 128-byte align dest (limited by copy length) // cr5 = flags, as above c64double: andi. r8,r7,7 // r8 <- #bytes to doubleword align srwi r9,r7,3 // r9 <- #doublewords to 128-byte align sub r5,r5,r7 // adjust length remaining cmpwi cr1,r9,0 // any doublewords to move to cache align? srwi r10,r5,7 // r10 <- 128-byte chunks to xfer after aligning dest cmpwi cr7,r10,0 // set cr7 on chunk count beq c64double2 // dest already doubleword aligned mtctr r8 b c64double1 .align 5 // align inner loops c64double1: // copy bytes until dest is doubleword aligned lbz r0,0(r6) addi r6,r6,1 stb r0,0(r4) addi r4,r4,1 bdnz c64double1 c64double2: // r9/cr1=doublewords, r10=128-byte chunks, cr7=blt if r5==0 beq cr1,c64double4 // no doublewords to xfer in order to cache align mtctr r9 b c64double3 .align 5 // align inner loops c64double3: // copy doublewords until dest is 128-byte aligned ld r7,0(r6) addi r6,r6,8 std r7,0(r4) addi r4,r4,8 bdnz c64double3 // Here to xfer 128-byte chunks, if any. Because the IBM 970 cannot issue two stores/cycle, // we pipeline the inner loop so we can pair loads and stores. Since we only have 8 GPRs for // data (64 bytes), we load/store each twice per 128-byte chunk. c64double4: // r10/cr7=128-byte chunks rlwinm r0,r5,29,28,31 // r0 <- count of leftover doublewords, after moving chunks cmpwi cr1,r0,0 // set cr1 on leftover doublewords beq cr7,c64double7 // no 128-byte chunks sub r8,r6,r4 // r8 <- (source - dest) li r9,128 // start at next cache line (we've already touched in 1st line) cmpldi cr7,r8,128 // if (source-dest)<128, cannot use dcbz128 beacause of overlap cror noncache,cr7_lt,noncache // turn on "noncache" flag if (source-dest)<128 bt-- noncache,noncache3 // (HACK) Skip cache touch if noncachable dcbt128 r9,r6,1 // start forward stream noncache3: mtctr r10 ld r0,0(r6) // start pipe: load 1st half-line ld r2,8(r6) ld r7,16(r6) ld r8,24(r6) ld r9,32(r6) ld r10,40(r6) ld r11,48(r6) ld r12,56(r6) b c64InnerLoopEntryPt .align 5 // align inner loop c64InnerLoop: // loop copying 128-byte cache lines to 128-aligned destination std r0,64(r4) // store 2nd half of chunk n ld r0,0(r6) // load 1st half of chunk n+1 std r2,72(r4) ld r2,8(r6) std r7,80(r4) ld r7,16(r6) std r8,88(r4) ld r8,24(r6) std r9,96(r4) ld r9,32(r6) std r10,104(r4) ld r10,40(r6) std r11,112(r4) ld r11,48(r6) std r12,120(r4) ld r12,56(r6) addi r4,r4,128 // advance to next dest chunk c64InnerLoopEntryPt: // initial entry into loop, with 1st halfline loaded bt noncache,c64InnerLoop1 // skip if uncached or overlap dcbz128 0,r4 // avoid prefetch of next cache line c64InnerLoop1: std r0,0(r4) // store 1st half of chunk n ld r0,64(r6) // load 2nd half of chunk n std r2,8(r4) ld r2,72(r6) std r7,16(r4) ld r7,80(r6) std r8,24(r4) ld r8,88(r6) std r9,32(r4) ld r9,96(r6) std r10,40(r4) ld r10,104(r6) std r11,48(r4) ld r11,112(r6) std r12,56(r4) ld r12,120(r6) addi r6,r6,128 // advance to next source chunk if any bdnz c64InnerLoop // loop if more chunks std r0,64(r4) // store 2nd half of last chunk std r2,72(r4) std r7,80(r4) std r8,88(r4) std r9,96(r4) std r10,104(r4) std r11,112(r4) std r12,120(r4) addi r4,r4,128 // advance to next dest chunk c64double7: // r5 <- leftover bytes, cr1 set on doubleword count rlwinm r0,r5,29,28,31 // r0 <- count of leftover doublewords (0-15) andi. r5,r5,7 // r5/cr0 <- count of leftover bytes (0-7) beq cr1,c64byte // no leftover doublewords mtctr r0 b c64double8 .align 5 // align inner loop c64double8: // loop copying leftover doublewords ld r0,0(r6) addi r6,r6,8 std r0,0(r4) addi r4,r4,8 bdnz c64double8 // Forward byte loop. c64byte: // r5/cr0 <- byte count (can be big if unaligned uncached) beq bcpydone // done if no leftover bytes mtctr r5 b c64byte1 .align 5 // align inner loop c64byte1: lbz r0,0(r6) addi r6,r6,1 stb r0,0(r4) addi r4,r4,1 bdnz c64byte1 b bcpydone // Uncached copies. We must avoid unaligned accesses, since they always take alignment // exceptions on uncached memory on 64-bit processors. This may mean we copy long operands // a byte at a time, but that is still much faster than alignment exceptions. // r4-r6 = dest, length, source (as above) // r2 = mask of 1s for leading 0s in length, plus 1 extra 1 // r7 = #bytes to copy to 128-byte align dest (limited by operand length) // cr1 = blt if reverse move required c64uncached: xor r0,r6,r4 // get relative alignment rlwinm r10,r0,0,29,31 // relatively doubleword aligned? rlwinm r11,r0,0,30,31 // relatively word aligned? not r8,r2 // get mask to limit initial length of copy for G4word blt cr1,c64reverseUncached cmpwi cr0,r10,0 // set cr0 beq if doubleword aligned cmpwi cr1,r11,0 // set cr1 beq if word aligned beq cr0,c64double // doubleword aligned beq cr1,G4word // word aligned, use G3/G4 code cmpwi r5,0 // set cr0 on byte count b c64byte // unaligned operands c64reverseUncached: cmpwi cr0,r10,0 // set cr0 beq if doubleword aligned cmpwi cr1,r11,0 // set cr1 beq if word aligned beq cr0,c64rdouble // doubleword aligned so can use LD/STD beq cr1,G4reverseWord // word aligned, use G3/G4 code add r6,r6,r5 // point to (end+1) of source and dest add r4,r4,r5 cmpwi r5,0 // set cr0 on length b c64rbyte // copy a byte at a time // Reverse doubleword copies. This is used for all cached copies, and doubleword // aligned uncached copies. // r4 = destination (32 or 64-bit ptr) // r5 = length (always 32 bits) // r6 = source (32 or 64-bit ptr) // cr5 = noncache, fixxlate, and restorex flags set c64rdouble: add r6,r6,r5 // point to (end+1) of source and dest add r4,r4,r5 rlwinm. r7,r4,0,29,31 // r7 <- #bytes to doubleword align dest cmplw cr1,r7,r5 // operand long enough to doubleword align? blt cr1,c64rd0 // yes mr r7,r5 // no c64rd0: sub r5,r5,r7 // adjust length srwi r8,r5,6 // r8 <- 64-byte chunks to xfer cmpwi cr1,r8,0 // any chunks? beq c64rd2 // source already doubleword aligned mtctr r7 c64rd1: // copy bytes until source doublword aligned lbzu r0,-1(r6) stbu r0,-1(r4) bdnz c64rd1 c64rd2: // r8/cr1 <- count of 64-byte chunks rlwinm r0,r5,29,29,31 // r0 <- count of leftover doublewords andi. r5,r5,7 // r5/cr0 <- count of leftover bytes cmpwi cr7,r0,0 // leftover doublewords? beq cr1,c64rd4 // no chunks to xfer li r9,-128 // start at next cache line mtctr r8 bt noncache,c64rd3 // (HACK) Do not start a stream if noncachable... dcbt128 r9,r6,3 // start reverse stream b c64rd3 .align 5 // align inner loop c64rd3: // loop copying 64-byte chunks ld r7,-8(r6) ld r8,-16(r6) ld r9,-24(r6) ld r10,-32(r6) ld r11,-40(r6) ld r12,-48(r6) std r7,-8(r4) std r8,-16(r4) ld r7,-56(r6) ldu r8,-64(r6) std r9,-24(r4) std r10,-32(r4) std r11,-40(r4) std r12,-48(r4) std r7,-56(r4) stdu r8,-64(r4) bdnz c64rd3 c64rd4: // r0/cr7 = leftover doublewords r5/cr0 = leftover bytes beq cr7,c64rbyte // no leftover doublewords mtctr r0 c64rd5: // loop copying leftover doublewords ldu r0,-8(r6) stdu r0,-8(r4) bdnz c64rd5 // Reverse byte loop. c64rbyte: // r5/cr0 <- byte count (can be big if unaligned uncached) beq bcpydone // done if no leftover bytes mtctr r5 c64rbyte1: lbzu r0,-1(r6) stbu r0,-1(r4) bdnz c64rbyte1 b bcpydone