/*
* Copyright (c) 2009 Apple Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this
* file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
#include <arm/arch.h>
#if defined _ARM_ARCH_7 && !defined VARIANT_DYLD
/*****************************************************************************
* Cortex-A8 implementation *
*****************************************************************************/
// Cortex-A8 implementations of memcpy( ), memmove( ) and bcopy( ).
//
// Our tests have shown that NEON is always a performance win for memcpy( ).
// However, for the specific case of copies from a warm source to a cold
// destination when the buffer size is between 1k and 32k, it is not enough
// of a performance win to offset the increased power footprint, resulting
// in an energy usage regression. Thus, we detect that particular case, and
// pass those copies through the ARM core registers. All other copies larger
// than 8 bytes are handled on NEON.
//
// Stephen Canon, August 2009
.text
.code 16
.syntax unified
// void bcopy(const void * source,
// void * destination,
// size_t length)// void *memmove(void * destination,
// const void * source,
// size_t n)// void *memcpy(void * restrict destination,
// const void * restrict source,
// size_t n)// all copy n successive bytes from source to destination. memmove and memcpy
// returns destination, whereas bcopy has no return value. copying takes place
// as if it were through a temporary buffer -- after return destination contains
// exactly the bytes from source, even if the buffers overlap.
.thumb_func _bcopy$VARIANT$CortexA8
.thumb_func _memmove$VARIANT$CortexA8
.thumb_func _memcpy$VARIANT$CortexA8
.globl _bcopy$VARIANT$CortexA8
.globl _memmove$VARIANT$CortexA8
.globl _memcpy$VARIANT$CortexA8
#define SAVE_REGISTERS {r4,r5,r6,r8,r10,r11}
#define COPY_REGISTERS {r3,r4,r5,r6,r8,r9,r10,r11}
/*****************************************************************************
* entry points *
*****************************************************************************/
.align 2
_bcopy$VARIANT$CortexA8:
// bcopy has the first and second arguments in the opposite order as the C
// library functions memmove and memcpy. If bcopy is called, we swap these
// two arguments and then fall into memmove.
mov r3, r0
mov r0, r1
mov r1, r3
.align 2
_memmove$VARIANT$CortexA8:
_memcpy$VARIANT$CortexA8:
// At entry to memmove/memcpy, registers contain the following values:
//
// r0 pointer to the first byte of the destination buffer
// r1 pointer to the first byte of the source buffer
// r2 number of bytes to copy
//
// Our preference is to use a (faster and easier to understand) front-to-back
// copy of the buffer. However, memmove requires that copies take place as
// though through a temporary buffer. This means that if the buffers overlap,
// it may be necessary to copy the buffer in reverse order.
//
// To properly detect such overlap, we begin by computing the offset between
// the source and destination pointers. If the offset happens to be zero,
// then there is no work to be done, so we can early out.
subs r3, r0, r1
it eq
bxeq lr
// r3 now contains the offset between the buffers, (destination - source). If
// 0 < offset < length, then the high-addressed bits of the source alias the
// low addressed bytes of the destination. Thus, if we were to perform the
// copy in ascending address order, we would overwrite the high-addressed
// source bytes before we had a chance to copy them, and the data would be lost.
//
// Thus, we can use the front-to-back copy only if offset is negative or
// greater than the length. This is the case precisely if offset compares
// unsigned higher than length.
cmp r3, r2
bhs L_copyFrontToBack
/*****************************************************************************
* back to front copy *
*****************************************************************************/
// Here we have fallen through into the back-to-front copy. We preserve the
// original destination pointer in r0 because it is the return value for the
// routine, and update the other registers as follows:
//
// r1 one byte beyond the end of the destination buffer
// r2 number of bytes to copy
// ip one byte beyond the end of the destination buffer
mov ip, r0
add r1, r2
add ip, r2
// Subtract 8 from the buffer length
subs r2, $8
blt L_scalarReverseCopy
// If the destination pointer is 8-byte aligned we can use 8-byte NEON copies
// to move the data.
tst ip, $7
beq L_vectorReverseCopy
// Otherwise, we copy a single byte at a time, in order of descending memory
// address, until the destination is 8 byte aligned. Within this loop,
// registers are used as follows:
//
// r0 original destination pointer
// r1 pointer to one byte past the next element to be copied
// r2 (bytes remaining to be copied) - 8
// r3 temporary to hold the byte that is being copied
// ip pointer one byte past the destination of the next byte to be copied
//
// byte that will be copied in this iteration
// | byte that was copied in the previous iteration
// Source buffer: v v
// ------------------------+---+---+-------------------------
// bytes still to copy ... | | | ... bytes already copied
// ------------------------+---+---+-------------------------
// ^
// r1 holds the address of this byte
0: ldrb r3, [r1, $-1]!
sub r2, $1
strb r3, [ip, $-1]!
tst ip, $7
bne 0b
// At this point, the destination pointer is 8 byte aligned. Check again that
// there are at least 8 bytes remaining to copy by comparing the remaining
// length minus 8 to zero. If fewer than 8 bytes remain, jump to the cleanup
// path.
cmp r2, $0
blt L_scalarReverseCopy
/*****************************************************************************
* destination is 8 byte aligned *
*****************************************************************************/
L_vectorReverseCopy:
// At this point, registers contain the following values:
//
// r0 original destination pointer
// r1 pointer to one byte past the next element to be copied
// r2 (bytes remaining to copy) - 8
// ip pointer one byte past the destination of the next byte to be copied
//
// Furthermore, it is known that ip is 8 byte aligned, and that r2 is positive.
// NEON has really excellent alignment handling in hardware, so we would like
// to use that to handle cases where the source is not similarly aligned to the
// destination (it supports even single-byte misalignment at speed). However,
// on some SoC designs, not all of the DMA busses support such access. Thus,
// we must unfortunately use a software workaround in those cases.
//
// Fortunately, 4-byte aligned loads are supported even on the DMA busses, so
// we only need to handle the different possible source alignments modulo 4.
// Here we have a dispatch table to jump to the correct copy implementation
// for the given source alignment.
//
// The tbh instruction loads the address offset of the correct implementation
// from the data table that immediately follows it and adds it to the pc to
// jump to the correct branch.
ands r3, r1, $3
tbh [pc, r3, lsl $1]
0:
.short (L_reverseAligned0-0b)/2
.short (L_reverseAligned1-0b)/2
.short (L_reverseAligned2-0b)/2
.short (L_reverseAligned3-0b)/2
/*****************************************************************************
* source is also at least word aligned *
*****************************************************************************/
L_reverseAligned0:
// Subtract 56 from r2, so that it contains the number of bytes remaining to
// copy minus 64. If this result is negative, then we jump into a loop that
// copies 8 bytes at a time.
subs r2, $0x38
blt L_reverseVectorCleanup
// Check if the destination pointer is 64-byte aligned. If so, jump to a loop
// that copies whole cachelines.
tst ip, $0x38
beq L_reverseCachelineAligned
// Otherwise, we copy a 8 bytes at a time, in order of descending memory
// address, until the destination is 64 byte aligned. Within this loop,
// registers are used as follows:
//
// r0 original destination pointer
// r1 pointer to one byte past the next element to be copied
// r2 (bytes remaining to be copied) - 64
// ip pointer one byte past the destination of the next byte to be copied
// d0 temporary storage for copy
//
// bytes that will be copied after this iteration
// | 8 byte block that will be copied in this iteration
// v v
// --------------+-------------------------------+---------------------
// | 0 1 2 3 4 5 6 7 | bytes already copied
// --------------+-------------------------------+---------------------
// ^
// r1 points here
0: sub r1, $8
vld1.32 {d0}, [r1]
sub ip, $8
sub r2, $8
tst ip, $0x38
vst1.64 {d0}, [ip,:64]
bne 0b
// At this point, the destination pointer is 64 byte aligned. Check again that
// there are at least 64 bytes remaining to copy by comparing the remaining
// length minus 64 to zero. If fewer than 64 bytes remain, skip over the main
// copy loop.
cmp r2, $0
blt L_reverseVectorCleanup
/*****************************************************************************
* destination is cacheline aligned *
*****************************************************************************/
L_reverseCachelineAligned:
// In the special case that we are copying a buffer of between 1k and 32k bytes
// we do not use a NEON copy for the main loop. This is because if we happen
// to be doing a copy from a source in cache to a destination that is not in
// cache, this will result in an increase in energy usage. In all other cases,
// NEON gives superior energy conservation.
sub r3, r2, $0x3c0
cmp r3, $0x7c00
blo L_useSTMDB
// Pre-decrement the source (r1) and destination (ip) pointers so that they
// point to the first byte of the trailing 32-byte window of each buffer.
// Additionally, load the address increment of -32 into r3.
sub r1, $32
sub ip, $32
mov r3, $-32
// The destination pointer is known to be 64-byte aligned, so we can use the
// maximal alignment hint (:256) for our vector stores. Detect if the source
// is also at least 32-byte aligned and jump to a loop that uses maximal
// alignment hints for the loads as well if possible.
tst r1, $0x1f
beq L_reverseSourceAligned
// This loop copies 64 bytes per iteration, from a 4-byte aligned source to a
// 64-byte aligned destination, in order of descending memory address. Within
// this loop, registers are used as follows:
//
// r0 original destination pointer (unmodified)
// r1 pointer to the next 32-byte block to load
// r2 (number of bytes remaining to copy) - 64
// r3 address increment of -32.
// ip pointer to which the next 32-byte block is to be stored
// q0-q3 temporary registers used for copies
//
// Note that the loop is arrange in such a way that a single cleanup store is
// necessary after the final loop iteration. This occurs at label (1), and is
// shared between the unaligned and aligned loops.
vld1.32 {q2,q3}, [r1], r3
vld1.32 {q0,q1}, [r1], r3
subs r2, $64
vst1.64 {q2,q3}, [ip,:256], r3
blt 1f
.align 3
0: vld1.32 {q2,q3}, [r1], r3
vst1.64 {q0,q1}, [ip,:256], r3
vld1.32 {q0,q1}, [r1], r3
subs r2, $64
vst1.64 {q2,q3}, [ip,:256], r3
bge 0b
b 1f
L_reverseSourceAligned:
// This loop is identical to the immediately preceeding loop, except that it
// uses the additional alignment hint that the source pointer (r1) is 32-byte
// aligned. The two loops share cleanup code for the final iteration.
vld1.64 {q2,q3}, [r1,:256], r3
vld1.64 {q0,q1}, [r1,:256], r3
subs r2, $64
vst1.64 {q2,q3}, [ip,:256], r3
blt 1f
.align 3
0: vld1.64 {q2,q3}, [r1,:256], r3
vst1.64 {q0,q1}, [ip,:256], r3
vld1.64 {q0,q1}, [r1,:256], r3
subs r2, $64
vst1.64 {q2,q3}, [ip,:256], r3
bge 0b
// Final vector store for both of the above loops.
1: vst1.64 {q0,q1}, [ip,:256], r3
// Adjust the source and destination pointers so that they once again point to
// the last byte that we used (which is one byte higher than the address that
// we will use next for any required cleanup).
add r1, $32
add ip, $32
L_reverseVectorCleanup:
// Add 56 to r2, so that it contains the number of bytes remaing to copy minus
// 8. A comparison of this value with zero tells us if any more whole 8-byte
// blocks need to be copied.
adds r2, r2, $0x38
blt L_scalarReverseCopy
// This loop copies 8 bytes at a time in order of descending memory address,
// until fewer than 8 bytes remain to be copied. Within this loop, registers
// are used as follows:
//
// r0 original destination pointer
// r1 pointer to one byte past the next element to be copied
// r2 (bytes remaining to be copied) - 64
// ip pointer one byte past the destination of the next byte to be copied
// d0 temporary storage for copy
0: sub r1, $8
vld1.32 {d0}, [r1]
sub ip, $8
subs r2, $8
vst1.64 {d0}, [ip,:64]
bge 0b
/*****************************************************************************
* sub-doubleword cleanup copies *
*****************************************************************************/
L_scalarReverseCopy:
// Add 8 to r2, so that it contains the number of bytes remaining to copy, and
// return to the calling routine if zero bytes remain.
adds r2, $8
it eq
bxeq lr
// Copy one byte at a time in descending address order until we reach the front
// of the buffer. Within this loop, registers are used as follows:
//
// r0 original destination pointer
// r1 pointer to one byte past the next element to be copied
// r2 (bytes remaining to be copied) - 8
// r3 temporary to hold the byte that is being copied
// ip pointer one byte past the destination of the next byte to be copied
0: ldrb r3, [r1, $-1]!
subs r2, $1
strb r3, [ip, $-1]!
bne 0b
bx lr
/*****************************************************************************
* STMDB loop for 1k-32k buffers *
*****************************************************************************/
// This loop copies 64 bytes each iteration in order of descending memory
// address, using the GPRs instead of NEON.
//
// r0 original destination pointer
// r1 pointer to one byte past the next element to be copied
// r2 (bytes remaining to be copied) - 64
// r3-6,r8-11 (COPY_REGISTERS) temporary registers used for moving data
// ip pointer to one byte past the next location to store to
L_useSTMDB:
push SAVE_REGISTERS
.align 3
0: ldmdb r1!, COPY_REGISTERS
subs r2, r2, $64
stmdb ip!, COPY_REGISTERS
ldmdb r1!, COPY_REGISTERS
pld [r1, $-64]
stmdb ip!, COPY_REGISTERS
bge 0b
pop SAVE_REGISTERS
b L_reverseVectorCleanup
/*****************************************************************************
* Misaligned reverse vld1 loop *
*****************************************************************************/
// Software alignment fixup to handle source and dest that are relatively
// misaligned mod 4 bytes.
//
// The basic idea is to use 4-byte aligned loads to load 8 bytes per iteration,
// which we combine with the 8 bytes loaded in the previous iteration to get a
// 16 byte field//
// | 8 bytes from this iteration | 8 bytes from last iteration |
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
// | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f |
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
// ^8 bytes to store this iteration^ |
// could be a page boundary
//
// We need to be a little bit careful, however. Because the loads only have 4
// byte alignment, the very first load could slop over into a page that is not
// mapped readable. In order to prevent this scenario, we copy eight bytes
// using byte-by-byte before beginning the main loop.
//
// At the beginning of each iteration through this loop, registers are used
// as follows:
//
// r0 original destination pointer
// r1 pointer to the next block of 8 bytes to load
// r2 (bytes remaining to copy) - 8
// ip pointer to the next block of 8 bytes to store
// d0 next 8 bytes to store
// d2 8 bytes loaded in the previous iteration
// d3 8 bytes loaded two iterations ago
#define RCOPY_UNALIGNED(offset) \
0: ldrb r3, [r1,$-1]! subs r2, $1 tst ip, $7 bic r1, $3 sub ip, $8 vld1.32 {d2,d3}, [r1], r3 blt 1f vmov d3, d2 subs r2, $8 bge 0b add r1, $8 2: add r1, $(offset)
L_reverseAligned1:
RCOPY_UNALIGNED(1)
L_reverseAligned2:
RCOPY_UNALIGNED(2)
L_reverseAligned3:
RCOPY_UNALIGNED(3)
/*****************************************************************************
* front to back copy *
*****************************************************************************/
L_copyFrontToBack:
// Here the pointers are laid out such that we can use our preferred
// front-to-back copy. We preserve original destination pointer in r0 because
// it is the return value for the routine, and copy it to ip to use in this
// routine.
mov ip, r0
// Subtract 8 from the buffer length
subs r2, $8
blt L_scalarCopy
// If the destination pointer is 8-byte aligned we can use 8-byte NEON copies
// to move the data.
tst ip, $7
beq L_vectorCopy
// Otherwise, we copy a single byte at a time, in order of ascending memory
// address, until the destination is 8 byte aligned. Within this loop,
// registers are used as follows:
//
// r0 original destination pointer
// r1 pointer to the next byte to copy
// r2 (bytes remaining to be copied) - 8
// r3 temporary to hold the byte that is being copied
// ip pointer to the next byte to store to
0: ldrb r3, [r1], $1
sub r2, $1
strb r3, [ip], $1
tst ip, $7
bne 0b
// At this point, the destination pointer is 8 byte aligned. Check again that
// there are at least 8 bytes remaining to copy by comparing the remaining
// length minus 8 to zero. If fewer than 8 bytes remain, jump to the cleanup
// path.
cmp r2, $0
blt L_scalarCopy
/*****************************************************************************
* destination is doubleword aligned *
*****************************************************************************/
L_vectorCopy:
// At this point, registers contain the following values:
//
// r0 original destination pointer
// r1 pointer to the next element to be copied
// r2 (bytes remaining to copy) - 8
// ip pointer to the destination of the next byte to be copied
//
// Furthermore, it is known that ip is 8 byte aligned, and that r2 is positive.
// NEON has really excellent alignment handling in hardware, so we would like
// to use that to handle cases where the source is not similarly aligned to the
// destination (it supports even single-byte misalignment at speed). However,
// on some SoC designs, not all of the DMA busses support such access. Thus,
// we must unfortunately use a software workaround in those cases.
//
// Fortunately, 4-byte aligned loads are supported even on the DMA busses, so
// we only need to handle the different possible source alignments modulo 4.
// Here we have a dispatch table to jump to the correct copy implementation
// for the given source alignment.
//
// The tbh instruction loads the address offset of the correct implementation
// from the data table that immediately follows it and adds it to the pc to
// jump to the correct branch.
ands r3, r1, $3
bic r1, $3
tbh [pc, r3, lsl $1]
0:
.short (L_sourceAligned0-0b)/2
.short (L_sourceAligned1-0b)/2
.short (L_sourceAligned2-0b)/2
.short (L_sourceAligned3-0b)/2
/*****************************************************************************
* source is also at least word aligned *
*****************************************************************************/
L_sourceAligned0:
// Subtract 56 from r2, so that it contains the number of bytes remaining to
// copy minus 64. If this result is negative, then we jump into a loop that
// copies 8 bytes at a time.
subs r2, $0x38
blt L_vectorCleanup
// Check if the destination pointer is 64-byte aligned. If so, jump to a loop
// that copies whole cachelines.
tst ip, $0x38
beq L_cachelineAligned
// Otherwise, we copy a 8 bytes at a time, in order of ascending memory
// address, until the destination is 64 byte aligned. Within this loop,
// registers are used as follows:
//
// r0 original destination pointer
// r1 pointer to the next element to be copied
// r2 (bytes remaining to be copied) - 64
// ip pointer to the destination of the next byte to be copied
// d0 temporary storage for copy
0: vld1.32 {d0}, [r1]!
sub r2, $8
vst1.64 {d0}, [ip,:64]!
tst ip, $0x38
bne 0b
// At this point, the destination pointer is 64 byte aligned. Check again that
// there are at least 64 bytes remaining to copy by comparing the remaining
// length minus 64 to zero. If fewer than 64 bytes remain, skip over the main
// copy loop.
cmp r2, $0
blt L_vectorCleanup
/*****************************************************************************
* destination is cacheline aligned *
*****************************************************************************/
// In the special case that we are copying a buffer of between 1k and 32k bytes
// we do not use a NEON copy for the main loop. This is because if we happen
// to be doing a copy from a source in cache to a destination that is not in
// cache, this will result in an increase in energy usage. In all other cases,
// NEON gives superior energy conservation.
L_cachelineAligned:
sub r3, r2, $0x3c0
cmp r3, $0x7c00
blo L_useSTMIA
// The destination pointer is known to be 64-byte aligned, so we can use the
// maximal alignment hint (:256) for our vector stores. Detect if the source
// is also at least 32-byte aligned and jump to a loop that uses maximal
// alignment hints for the loads as well if possible.
tst r1, $0x1f
beq L_sourceAligned32
// This loop copies 64 bytes per iteration, from a 4-byte aligned source to a
// 64-byte aligned destination, in order of ascending memory address. Within
// this loop, registers are used as follows:
//
// r0 original destination pointer (unmodified)
// r1 pointer to the next 32-byte block to load
// r2 (number of bytes remaining to copy) - 64
// ip pointer to which the next 32-byte block is to be stored
// q0-q3 temporary registers used for copies
//
// Note that the loop is arrange in such a way that a single cleanup store is
// necessary after the final loop iteration. This occurs at label (1), and is
// shared between the unaligned and aligned loops.
vld1.32 {q2,q3}, [r1]!
vld1.32 {q0,q1}, [r1]!
subs r2, $64
vst1.64 {q2,q3}, [ip,:256]!
blt 1f
.align 3
0: vld1.32 {q2,q3}, [r1]!
vst1.64 {q0,q1}, [ip,:256]!
vld1.32 {q0,q1}, [r1]!
subs r2, $64
vst1.64 {q2,q3}, [ip,:256]!
bge 0b
b 1f
L_sourceAligned32:
// This loop is identical to the immediately preceeding loop, except that it
// uses the additional alignment hint that the source pointer (r1) is 32-byte
// aligned. The two loops share cleanup code for the final iteration.
vld1.64 {q2,q3}, [r1,:256]!
vld1.64 {q0,q1}, [r1,:256]!
subs r2, $64
vst1.64 {q2,q3}, [ip,:256]!
blt 1f
.align 3
0: vld1.64 {q2,q3}, [r1,:256]!
vst1.64 {q0,q1}, [ip,:256]!
vld1.64 {q0,q1}, [r1,:256]!
subs r2, $64
vst1.64 {q2,q3}, [ip,:256]!
bge 0b
// Final vector store for both of the above loops.
1: vst1.64 {q0,q1}, [ip,:256]!
L_vectorCleanup:
// Add 56 to r2, so that it contains the number of bytes remaing to copy minus
// 8. A comparison of this value with zero tells us if any more whole 8-byte
// blocks need to be copied.
adds r2, $0x38
blt L_scalarCopy
// This loop copies 8 bytes at a time in order of descending memory address,
// until fewer than 8 bytes remain to be copied. Within this loop, registers
// are used as follows:
//
// r0 original destination pointer
// r1 pointer to the next element to be copied
// r2 (bytes remaining to be copied) - 64
// ip pointer to the destination of the next byte to be copied
// d0 temporary storage for copy
0: vld1.32 {d0}, [r1]!
subs r2, $8
vst1.64 {d0}, [ip,:64]!
bge 0b
/*****************************************************************************
* sub-doubleword cleanup copies *
*****************************************************************************/
L_scalarCopy:
// Add 8 to r2, so that it contains the number of bytes remaining to copy, and
// return to the calling routine if zero bytes remain.
adds r2, $8
it eq
bxeq lr
// Copy one byte at a time in descending address order until we reach the front
// of the buffer. Within this loop, registers are used as follows:
//
// r0 original destination pointer
// r1 pointer to one byte past the next element to be copied
// r2 (bytes remaining to be copied) - 8
// r3 temporary to hold the byte that is being copied
// ip pointer one byte past the destination of the next byte to be copied
0: ldrb r3, [r1], $1
strb r3, [ip], $1
subs r2, $1
bne 0b
bx lr
/*****************************************************************************
* STMIA loop for 1k-32k buffers *
*****************************************************************************/
// This loop copies 64 bytes each iteration in order of ascending memory
// address, using the GPRs instead of NEON.
//
// r0 original destination pointer
// r1 pointer to the next element to be copied
// r2 (bytes remaining to be copied) - 64
// r3-6,r8-11 (COPY_REGISTERS) temporary registers used for moving data
// ip pointer to the next location to store to
L_useSTMIA:
push SAVE_REGISTERS
.align 3
0: ldmia r1!, COPY_REGISTERS
subs r2, r2, $64
stmia ip!, COPY_REGISTERS
ldmia r1!, COPY_REGISTERS
pld [r1, $64]
stmia ip!, COPY_REGISTERS
bge 0b
pop SAVE_REGISTERS
b L_vectorCleanup
/*****************************************************************************
* Misaligned forward vld1 loop *
*****************************************************************************/
// Software alignment fixup to handle source and dest that are relatively
// misaligned mod 4 bytes.
//
// The basic idea is to use 4-byte aligned loads to load 8 bytes per iteration,
// which we combine with the 8 bytes loaded in the previous iteration to get a
// 16 byte field//
// | 8 bytes from last iteration | 8 bytes from this iteration |
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
// | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f |
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
// ^8 bytes to store this iteration^ |
// could be a page boundary
//
// We need to be a little bit careful, however. Because the loads only have 4
// byte alignment, if we used this approach all the way to the end of the
// buffer, the very last 8 byte load might slop over onto a new page by 4
// bytes, and that new page might not be mapped into our process. Thus, we
// terminate this copy loop when fewer than 12 bytes remain to be copied,
// instead of the more natural-seeming termination condition of "8 bytes
// remaining" (the illustration above shows the worst case and demonstrates
// why 12 is a sufficiently safe condition).
//
// At the beginning of each iteration through this loop, registers are used
// as follows:
//
// r0 original destination pointer
// r1 pointer to the next block of 8 bytes to load
// r2 (bytes remaining to copy) - 12
// ip pointer to the next block of 8 bytes to store
// d0 next 8 bytes to store
// d2 8 bytes loaded in the previous iteration
// d3 8 bytes loaded two iterations ago
#define COPY_UNALIGNED(offset) \
subs r2, $4 vld1.32 {d2,d3}, [r1]! blt 1f vmov d2, d3 subs r2, $8 bge 0b sub r1, $8 2: add r1, $(offset) b L_scalarCopy
L_sourceAligned1:
COPY_UNALIGNED(1)
L_sourceAligned2:
COPY_UNALIGNED(2)
L_sourceAligned3:
COPY_UNALIGNED(3)
#endif // defined _ARM_ARCH_7 && !defined VARIANT_DYLD