/* * Copyright (c) 2011 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this * file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_LICENSE_HEADER_END@ * * This file implements the following functions for the Swift micro-arch: * * void memset_pattern4(void *b, const void *pattern4, size_t len); * void memset_pattern8(void *b, const void *pattern8, size_t len); * void memset_pattern16(void *b, const void *pattern16, size_t len); * * The implementation of all three functions is fundamentally the same. * memset_pattern4 is extensively commented to explain, reference that * if you have any questions about the other two. */ #include <arm/arch.h> #if defined _ARM_ARCH_7 && !defined VARIANT_DYLD .syntax unified .code 32 .text .globl _memset_pattern4$VARIANT$Swift .globl _memset_pattern8$VARIANT$Swift .globl _memset_pattern16$VARIANT$Swift /******************************************************************************/ .align 4 _memset_pattern4$VARIANT$Swift: push {r7,lr} mov r7, sp // Load the pattern and splat it to q0, then check if the buffer is at least // 64 bytes long. If not, branch to a short-buffer implementation. ldr r1, [r1] vdup.32 q0, r1 subs r3, r2, #64 blo L_short4 // We want to use aligned vector stores to fill the bulk of the buffer. In // order to make that work, we need to rotate the pattern as necessary to // match up with aligned locations, and we also need to extract the alignment // of the destination pointer mod 16. lsl ip, r0, #3 and lr, r0, #0xf // alignment of destination pointer mod 16 rsb ip, ip, #32 // low five bits contain 32 - 8*(address%4). // Before we start the aligned stores, we do a single unaligned store of // 16 bytes of the pattern to the start of the buffer. Since the buffer is // at least 64 bytes long, this store is known to lie entirely inside the // buffer: // first aligned address in buffer // v // ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- // ... | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f | 0 | 1 | 2 | ... // ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- // ^ // unaligned store starts here: // [ 0 1 2 3 0 1 2 3 0 1 2 3 0 1 2 3 ] vst1.8 {q0}, [r0]! // Subsequent stores will be aligned, and will start at the first aligned // address in the buffer. We apply the rotation that we calculated before // the vector store (in the low five bits of ip) to get the pattern that // is to be stored starting at the aligned location. For example, in the // picture above, the buffer had alignment of 3 mod 4, so the rotation to // be applied is 32 - 8*3 = 8. Rotating the pattern right by 8 bits gives // us [ 1 2 3 0 ] (remember, we're little-endian), which we see is what // needs to be stored starting at the first aligned location. // // Besides rotating the pattern, we also need to adjust the length (by // subtracting 16 - alignment mod 16), and to advance the pointer to the // first aligned location. ror r1, ip // Pattern to use for aligned memory add r3, lr bic r0, #0xf // destination for first aligned store subs r3, #16 // updated length blo 1f // Splat the rotated value across q1 and q2 vdup.32 q1, r1 vmov q2, q1 // Main store loop. We write the splatted aligned pattern across 64 bytes // per iteration, terminating the loop when the remaining length of the // buffer is 64 bytes or less. 0: subs r3, #64 vst1.32 {q1,q2}, [r0,:128]! vst1.32 {q1,q2}, [r0,:128]! bhi 0b // The remaining length of the buffer is 64 bytes or less (but the total // length of the buffer is at least 64 bytes; otherwise we would have // branched to the "short" path). Thus, we can handle the entirety of the // remaining buffer with two 32-byte unaligned stores. // // Again, we need to rotate the pattern to match the alignment, this time // by 8*(length%4), and we also need to back up the destination pointer // so that it points to precisely 64 bytes before the end of the buffer. // We accomplish this by adding r3, which contains the remaining length of // the buffer minus 64. 1: lsl ip, r3, #3 ror r1, ip vdup.32 q8, r1 vmov q9, q8 add r0, r3 vst1.32 {q8,q9}, [r0]! vst1.32 {q8,q9}, [r0] pop {r7,pc} L_short4: // If we branch here, the buffer is less than 64 bytes long. At this point, // register contents are as follows: // // r0 pointer to the buffer // r1 pattern // r2 buffer length // q0 splatted pattern // // To begin, we store eight bytes at a time until the remaining length is // less than eight bytes. subs r3, r2, #8 blo 1f 0: subs r3, #8 vst1.32 {d0}, [r0]! bhs 0b // Then we store one byte at a time, rotating the pattern to get the next // byte, until we reach the end of the buffer. add r2, r3, #8 1: subs r2, #1 strbhs r1, [r0],#1 ror r1, #8 bhi 1b pop {r7,pc} /******************************************************************************/ .align 4 _memset_pattern8$VARIANT$Swift: // The implementation of this function is substantially identical to that of // memset_pattern4. The only differences are in how we rotate the pattern for // the purposes of extracting the bytes to store. For clarity, only those // differences are commented here; consult memset_pattern4 (above) for // a detailed description of the algorithm used. push {r7,lr} mov r7, sp vld1.8 {d0}, [r1] vmov d1, d0 subs r3, r2, #64 blo L_short8 bic sp, #0xf // Align stack to 16 bytes and write 32 bytes sub sp, #16 // of pattern to the stack. We will use vst1.8 {q0}, [sp,:128] // unaligned loads from this scratch buffer sub sp, #16 // to get rotated forms of the pattern. vst1.8 {q0}, [sp,:128] and ip, r0, #0x7 // Now generate an unaligned pointer to the rsb ip, ip, #8 // rotated pattern that we need to use for add ip, sp // aligned stores in the main loop. and lr, r0, #0xf vst1.8 {q0}, [r0]! add r3, lr bic r0, #0xf subs r3, #16 blo 1f vld1.8 {q1}, [ip] vmov q2, q1 0: subs r3, #64 vst1.32 {q1,q2}, [r0,:128]! vst1.32 {q1,q2}, [r0,:128]! bhi 0b 1: and lr, r3, #0x7 // Generate an unaligned pointer to the add ip, lr // rotated pattern to use for cleanup. vld1.8 {q8}, [ip] vmov q9, q8 add r0, r3 vst1.32 {q8,q9}, [r0]! vst1.32 {q8,q9}, [r0] mov sp, r7 // Restore stack pointer pop {r7,pc} L_short8: subs r2, #8 blo 1f 0: subs r2, #8 vst1.32 {d0}, [r0]! bhs 0b 1: adds r2, #8 beq 3f 2: vst1.8 {d0[0]}, [r0]! // Store one byte from NEON vext.8 d0, d0, d0, #1 // Use VEXT to rotate pattern subs r2, #1 bhi 2b 3: pop {r7,pc} /******************************************************************************/ .align 4 _memset_pattern16$VARIANT$Swift: // The implementation of this function is substantially identical to that of // memset_pattern4. The only differences are in how we rotate the pattern for // the purposes of extracting the bytes to store. For clarity, only those // differences are commented here; consult memset_pattern4 (above) for // a detailed description of the algorithm used. push {r7,lr} mov r7, sp vld1.8 {q0}, [r1] subs r3, r2, #64 blo L_short16 bic sp, #0xf // Align stack to 16 bytes and write 48 bytes sub sp, #16 // of pattern to the stack. We will use vst1.8 {q0}, [sp,:128] // unaligned loads from this scratch buffer sub sp, #16 // to get rotated forms of the pattern. vst1.8 {q0}, [sp,:128] sub sp, #16 vst1.8 {q0}, [sp,:128] and lr, r0, #0xf // Now generate an unaligned pointer to the rsb ip, lr, #16 // rotated pattern that we need to use for add ip, sp // aligned stores in the main loop. vst1.8 {q0}, [r0]! add r3, lr bic r0, #0xf subs r3, #16 blo 1f vld1.8 {q1}, [ip] vmov q2, q1 0: subs r3, #64 vst1.32 {q1,q2}, [r0,:128]! vst1.32 {q1,q2}, [r0,:128]! bhi 0b 1: and lr, r3, #0xf // Generate an unaligned pointer to the add ip, lr // rotated pattern to use for cleanup. vld1.8 {q8}, [ip] vmov q9, q8 add r0, r3 vst1.32 {q8,q9}, [r0]! vst1.32 {q8,q9}, [r0] mov sp, r7 // Restore stack pointer pop {r7,pc} L_short16: subs r2, #16 blo 1f 0: subs r2, #16 vst1.32 {q0}, [r0]! bhs 0b 1: adds r2, #16 beq 3f 2: vst1.8 {d0[0]}, [r0]! // Store one byte from NEON vext.8 q0, q0, q0, #1 // Use VEXT to rotate pattern subs r2, #1 bhi 2b 3: pop {r7,pc} #endif // defined _ARM_ARCH_7 && !defined VARIANT_DYLD