diff mbox series

[v3,3/7] x86: Improve svml_s_atanhf4_core_sse4.S

Message ID 20220609165835.2527760-3-goldstein.w.n@gmail.com
State New
Headers show
Series [v3,1/7] x86: Improve svml_s_atanhf16_core_avx512.S | expand

Commit Message

Noah Goldstein June 9, 2022, 4:58 p.m. UTC
Improvements are:
    1. Reduce code size (-62 bytes).
    2. Remove redundant move instructions.
    3. Slightly improve instruction selection/scheduling where
       possible.
    4. Prefer registers which get short instruction encoding.
    5. Reduce rodata usage (-16 bytes).

The throughput improvement is not significant as the port 0 bottleneck
is unavoidable.

       Function, New Time, Old Time, New / Old
_ZGVbN4v_atanhf,    8.821,    8.903,     0.991
---
 .../fpu/multiarch/svml_s_atanhf4_core_sse4.S  | 378 ++++++++----------
 1 file changed, 169 insertions(+), 209 deletions(-)

Comments

H.J. Lu June 9, 2022, 5:07 p.m. UTC | #1
On Thu, Jun 9, 2022 at 9:58 AM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> Improvements are:
>     1. Reduce code size (-62 bytes).
>     2. Remove redundant move instructions.
>     3. Slightly improve instruction selection/scheduling where
>        possible.
>     4. Prefer registers which get short instruction encoding.
>     5. Reduce rodata usage (-16 bytes).
>
> The throughput improvement is not significant as the port 0 bottleneck
> is unavoidable.
>
>        Function, New Time, Old Time, New / Old
> _ZGVbN4v_atanhf,    8.821,    8.903,     0.991
> ---
>  .../fpu/multiarch/svml_s_atanhf4_core_sse4.S  | 378 ++++++++----------
>  1 file changed, 169 insertions(+), 209 deletions(-)
>
> diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S b/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S
> index 2d3ad2617f..37200b3601 100644
> --- a/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S
> +++ b/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S
> @@ -30,96 +30,80 @@
>   *
>   */
>
> -/* Offsets for data table __svml_satanh_data_internal
> - */
> -#define SgnMask                                0
> -#define sOne                           16
> -#define sPoly                          32
> -#define iBrkValue                      160
> -#define iOffExpoMask                   176
> -#define sHalf                          192
> -#define sSign                          208
> -#define sTopMask12                     224
> -#define TinyRange                      240
> -#define sLn2                           256
> +/* Offsets for data table __svml_satanh_data_internal_avx512. Ordered
> +   by use in the function. On cold-starts this might help the
> +   prefetcher. Possibly a better idea is to interleave start/end so
> +   that the prefetcher is less likely to detect a stream and pull
> +   irrelivant lines into cache.  */
> +#define sOne                           0
> +#define SgnMask                                16
> +#define sTopMask12                     32
> +#define iBrkValue                      48
> +#define iOffExpoMask                   64
> +#define sPoly                          80
> +#define sLn2                           208
> +#define TinyRange                      224
>
>  #include <sysdep.h>
> +#define ATANHF_DATA(x)                 ((x)+__svml_satanh_data_internal)
>
>         .section .text.sse4, "ax", @progbits
>  ENTRY(_ZGVbN4v_atanhf_sse4)
> -       subq    $72, %rsp
> -       cfi_def_cfa_offset(80)
>         movaps  %xmm0, %xmm5
>
>         /* Load constants including One = 1 */
> -       movups  sOne+__svml_satanh_data_internal(%rip), %xmm4
> +       movups  ATANHF_DATA(sOne)(%rip), %xmm4
>         movaps  %xmm5, %xmm3
>
>         /* Strip off the sign, so treat X as positive until right at the end */
> -       movups  SgnMask+__svml_satanh_data_internal(%rip), %xmm7
> -       movaps  %xmm4, %xmm8
> -       andps   %xmm5, %xmm7
> +       movups  ATANHF_DATA(SgnMask)(%rip), %xmm1
> +       movaps  %xmm4, %xmm2
> +       andps   %xmm1, %xmm0
>         movaps  %xmm4, %xmm10
> -       movups  sTopMask12+__svml_satanh_data_internal(%rip), %xmm11
> +       movups  ATANHF_DATA(sTopMask12)(%rip), %xmm11
>         movaps  %xmm4, %xmm14
>         movaps  %xmm11, %xmm9
>
> +
>         /*
>          * Compute V = 2 * X trivially, and UHi + U_lo = 1 - X in two pieces,
>          * the upper part UHi being <= 12 bits long. Then we have
>          * atanh(X) = 1/2 * log((1 + X) / (1 - X)) = 1/2 * log1p(V / (UHi + ULo)).
>          */
> -       movaps  %xmm7, %xmm12
> +       movaps  %xmm0, %xmm6
> +       mulps   %xmm5, %xmm3
> +       subps   %xmm0, %xmm2
> +       addps   %xmm0, %xmm6
> +       subps   %xmm2, %xmm10
> +       addps   %xmm5, %xmm3
> +       subps   %xmm0, %xmm10
> +       andps   %xmm2, %xmm9
> +
>
>         /*
>          * Check whether |X| < 1, in which case we use the main function.
>          * Otherwise set the rangemask so that the callout will get used.
>          * Note that this will also use the callout for NaNs since not(NaN < 1).
>          */
> -       movaps  %xmm7, %xmm6
> -       movaps  %xmm7, %xmm2
> -       cmpnltps %xmm4, %xmm6
> -       cmpltps TinyRange+__svml_satanh_data_internal(%rip), %xmm2
> -       mulps   %xmm5, %xmm3
> -       subps   %xmm7, %xmm8
> -       addps   %xmm7, %xmm12
> -       movmskps %xmm6, %edx
> -       subps   %xmm8, %xmm10
> -       addps   %xmm5, %xmm3
> -       subps   %xmm7, %xmm10
> -       andps   %xmm8, %xmm9
> +       rcpps   %xmm9, %xmm7
> +       subps   %xmm9, %xmm2
> +       andps   %xmm11, %xmm7
>
> -       /*
> -        * Now we feed into the log1p code, using H in place of _VARG1 and
> -        * later incorporating L into the reduced argument.
> -        * compute 1+x as high, low parts
> -        */
> -       movaps  %xmm4, %xmm7
> -
> -       /*
> -        * Now compute R = 1/(UHi+ULo) * (1 - E) and the error term E
> -        * The first FMR is exact (we force R to 12 bits just in case it
> -        * isn't already, to make absolutely sure), and since E is ~ 2^-12,
> -        * the rounding error in the other one is acceptable.
> -        */
> -       rcpps   %xmm9, %xmm15
> -       subps   %xmm9, %xmm8
> -       andps   %xmm11, %xmm15
>
>         /*
>          * Split V as well into upper 12 bits and lower part, so that we can get
>          * a preliminary quotient estimate without rounding error.
>          */
> -       andps   %xmm12, %xmm11
> -       mulps   %xmm15, %xmm9
> -       addps   %xmm8, %xmm10
> -       subps   %xmm11, %xmm12
> +       andps   %xmm6, %xmm11
> +       mulps   %xmm7, %xmm9
> +       addps   %xmm2, %xmm10
> +       subps   %xmm11, %xmm6
>
>         /* Hence get initial quotient estimate QHi + QLo = R * VHi + R * VLo */
> -       mulps   %xmm15, %xmm11
> -       mulps   %xmm15, %xmm10
> +       mulps   %xmm7, %xmm11
> +       mulps   %xmm7, %xmm10
>         subps   %xmm9, %xmm14
> -       mulps   %xmm12, %xmm15
> +       mulps   %xmm6, %xmm7
>         subps   %xmm10, %xmm14
>
>         /* Compute D = E + E^2 */
> @@ -127,8 +111,8 @@ ENTRY(_ZGVbN4v_atanhf_sse4)
>         movaps  %xmm4, %xmm8
>         mulps   %xmm14, %xmm13
>
> -       /* reduction: compute r, n */
> -       movdqu  iBrkValue+__svml_satanh_data_internal(%rip), %xmm9
> +       /* reduction: compute r,n */
> +       movdqu  ATANHF_DATA(iBrkValue)(%rip), %xmm9
>         addps   %xmm13, %xmm14
>
>         /*
> @@ -136,168 +120,149 @@ ENTRY(_ZGVbN4v_atanhf_sse4)
>          * = R *  (VHi + VLo) * (1 + D)
>          * = QHi + (QHi * D + QLo + QLo * D)
>          */
> -       movaps  %xmm14, %xmm0
> -       mulps   %xmm15, %xmm14
> -       mulps   %xmm11, %xmm0
> -       addps   %xmm14, %xmm15
> -       movdqu  iOffExpoMask+__svml_satanh_data_internal(%rip), %xmm12
> +       movaps  %xmm14, %xmm2
> +       mulps   %xmm7, %xmm14
> +       mulps   %xmm11, %xmm2
> +       addps   %xmm14, %xmm7
> +       movdqu  ATANHF_DATA(iOffExpoMask)(%rip), %xmm12
>         movaps  %xmm4, %xmm14
>
>         /* Record the sign for eventual reincorporation. */
> -       movups  sSign+__svml_satanh_data_internal(%rip), %xmm1
> -       addps   %xmm15, %xmm0
> +       addps   %xmm7, %xmm2
> +
>
>         /*
>          * Now finally accumulate the high and low parts of the
>          * argument to log1p, H + L, with a final compensated summation.
>          */
> -       movaps  %xmm0, %xmm6
> -       andps   %xmm5, %xmm1
> -
> +       movaps  %xmm2, %xmm6
> +       andnps  %xmm5, %xmm1
> +       movaps  %xmm4, %xmm7
>         /* Or the sign bit in with the tiny result to handle atanh(-0) correctly */
> -       orps    %xmm1, %xmm3
>         addps   %xmm11, %xmm6
>         maxps   %xmm6, %xmm7
>         minps   %xmm6, %xmm8
>         subps   %xmm6, %xmm11
>         movaps  %xmm7, %xmm10
> -       andps   %xmm2, %xmm3
>         addps   %xmm8, %xmm10
> -       addps   %xmm11, %xmm0
> +       addps   %xmm11, %xmm2
>         subps   %xmm10, %xmm7
>         psubd   %xmm9, %xmm10
> -       addps   %xmm7, %xmm8
> +       addps   %xmm8, %xmm7
>         pand    %xmm10, %xmm12
>         psrad   $23, %xmm10
>         cvtdq2ps %xmm10, %xmm13
> -       addps   %xmm8, %xmm0
> +       addps   %xmm7, %xmm2
>
>         /* final reconstruction */
> -       mulps   sLn2+__svml_satanh_data_internal(%rip), %xmm13
>         pslld   $23, %xmm10
>         paddd   %xmm9, %xmm12
>         psubd   %xmm10, %xmm14
>
>         /* polynomial evaluation */
>         subps   %xmm4, %xmm12
> -       mulps   %xmm0, %xmm14
> -       movups  sPoly+112+__svml_satanh_data_internal(%rip), %xmm0
> -       addps   %xmm12, %xmm14
> -       mulps   %xmm14, %xmm0
> +       mulps   %xmm14, %xmm2
> +       movups  ATANHF_DATA(sPoly+0)(%rip), %xmm7
> +       addps   %xmm12, %xmm2
> +       mulps   %xmm2, %xmm7
> +
>
>         /* Finally, halve the result and reincorporate the sign */
> -       movups  sHalf+__svml_satanh_data_internal(%rip), %xmm4
> -       pxor    %xmm1, %xmm4
> -       addps   sPoly+96+__svml_satanh_data_internal(%rip), %xmm0
> -       mulps   %xmm14, %xmm0
> -       addps   sPoly+80+__svml_satanh_data_internal(%rip), %xmm0
> -       mulps   %xmm14, %xmm0
> -       addps   sPoly+64+__svml_satanh_data_internal(%rip), %xmm0
> -       mulps   %xmm14, %xmm0
> -       addps   sPoly+48+__svml_satanh_data_internal(%rip), %xmm0
> -       mulps   %xmm14, %xmm0
> -       addps   sPoly+32+__svml_satanh_data_internal(%rip), %xmm0
> -       mulps   %xmm14, %xmm0
> -       addps   sPoly+16+__svml_satanh_data_internal(%rip), %xmm0
> -       mulps   %xmm14, %xmm0
> -       addps   sPoly+__svml_satanh_data_internal(%rip), %xmm0
> -       mulps   %xmm14, %xmm0
> -       mulps   %xmm14, %xmm0
> -       addps   %xmm0, %xmm14
> -       movaps  %xmm2, %xmm0
> -       addps   %xmm13, %xmm14
> -       mulps   %xmm14, %xmm4
> -       andnps  %xmm4, %xmm0
> -       orps    %xmm3, %xmm0
> -       testl   %edx, %edx
> +       addps   ATANHF_DATA(sPoly+16)(%rip), %xmm7
> +       mulps   %xmm2, %xmm7
> +       addps   ATANHF_DATA(sPoly+32)(%rip), %xmm7
> +       mulps   %xmm2, %xmm7
> +       addps   ATANHF_DATA(sPoly+48)(%rip), %xmm7
> +       mulps   %xmm2, %xmm7
> +       addps   ATANHF_DATA(sPoly+64)(%rip), %xmm7
> +       mulps   %xmm2, %xmm7
> +       addps   ATANHF_DATA(sPoly+80)(%rip), %xmm7
> +       mulps   %xmm2, %xmm7
> +       addps   ATANHF_DATA(sPoly+96)(%rip), %xmm7
> +       mulps   %xmm2, %xmm7
> +       movaps  ATANHF_DATA(sPoly+112)(%rip), %xmm6
> +       addps   %xmm6, %xmm7
> +       mulps   %xmm2, %xmm7
> +       mulps   %xmm2, %xmm7
> +       mulps   ATANHF_DATA(sLn2)(%rip), %xmm13
> +       /* We can build `sHalf` with `sPoly & sOne`.  */
> +       andps   %xmm4, %xmm6
> +       orps    %xmm1, %xmm3
> +       xorps   %xmm6, %xmm1
>
> -       /* Go to special inputs processing branch */
> -       jne     L(SPECIAL_VALUES_BRANCH)
> -       # LOE rbx rbp r12 r13 r14 r15 edx xmm0 xmm5
> +       addps   %xmm2, %xmm7
> +       addps   %xmm13, %xmm7
> +       mulps   %xmm7, %xmm1
>
> -       /* Restore registers
> -        * and exit the function
> -        */
> +       /* Finish check of NaNs.  */
> +       cmpleps %xmm0, %xmm4
> +       movmskps %xmm4, %edx
> +       cmpltps ATANHF_DATA(TinyRange)(%rip), %xmm0
>
> -L(EXIT):
> -       addq    $72, %rsp
> -       cfi_def_cfa_offset(8)
> +       andps   %xmm0, %xmm3
> +       andnps  %xmm1, %xmm0
> +       orps    %xmm3, %xmm0
> +
> +       testl   %edx, %edx
> +       /* Go to special inputs processing branch.  */
> +       jne     L(SPECIAL_VALUES_BRANCH)
> +       # LOE rbx rbp r12 r13 r14 r15 xmm0
> +       /* No registers to restore on fast path.  */
>         ret
> -       cfi_def_cfa_offset(80)
>
> -       /* Branch to process
> -        * special inputs
> -        */
>
> +       /* Cold case. edx has 1s where there was a special value that
> +          needs to be handled by a atanhf call. Optimize for code size
> +          more so than speed here. */
>  L(SPECIAL_VALUES_BRANCH):
> -       movups  %xmm5, 32(%rsp)
> -       movups  %xmm0, 48(%rsp)
> -       # LOE rbx rbp r12 r13 r14 r15 edx
> -
> -       xorl    %eax, %eax
> -       movq    %r12, 16(%rsp)
> -       cfi_offset(12, -64)
> -       movl    %eax, %r12d
> -       movq    %r13, 8(%rsp)
> -       cfi_offset(13, -72)
> -       movl    %edx, %r13d
> -       movq    %r14, (%rsp)
> -       cfi_offset(14, -80)
> -       # LOE rbx rbp r15 r12d r13d
> -
> -       /* Range mask
> -        * bits check
> -        */
> -
> -L(RANGEMASK_CHECK):
> -       btl     %r12d, %r13d
> -
> -       /* Call scalar math function */
> -       jc      L(SCALAR_MATH_CALL)
> -       # LOE rbx rbp r15 r12d r13d
> -
> -       /* Special inputs
> -        * processing loop
> -        */
> -
> +       # LOE rbx rdx rbp r12 r13 r14 r15 xmm0 xmm5
> +       /* Stack coming in 16-byte aligned. Set 8-byte misaligned so on
> +       call entry will be 16-byte aligned. */
> +       subq    $56, %rsp
> +       cfi_def_cfa_offset(64)
> +       movups  %xmm0, 24(%rsp)
> +       movups  %xmm5, 40(%rsp)
> +
> +       /* Use rbx/rbp for callee save registers as they get short
> +       encoding for many instructions (as compared with r12/r13). */
> +       movq    %rbx, (%rsp)
> +       cfi_offset(rbx, -64)
> +       movq    %rbp, 8(%rsp)
> +       cfi_offset(rbp, -56)
> +       /* edx has 1s where there was a special value that needs to be handled
> +          by a tanhf call.  */
> +       movl    %edx, %ebx
>  L(SPECIAL_VALUES_LOOP):
> -       incl    %r12d
> -       cmpl    $4, %r12d
> -
> -       /* Check bits in range mask */
> -       jl      L(RANGEMASK_CHECK)
> -       # LOE rbx rbp r15 r12d r13d
> -
> -       movq    16(%rsp), %r12
> -       cfi_restore(12)
> -       movq    8(%rsp), %r13
> -       cfi_restore(13)
> -       movq    (%rsp), %r14
> -       cfi_restore(14)
> -       movups  48(%rsp), %xmm0
> -
> -       /* Go to exit */
> -       jmp     L(EXIT)
> -       cfi_offset(12, -64)
> -       cfi_offset(13, -72)
> -       cfi_offset(14, -80)
> -       # LOE rbx rbp r12 r13 r14 r15 xmm0
> -
> -       /* Scalar math fucntion call
> -        * to process special input
> -        */
> -
> -L(SCALAR_MATH_CALL):
> -       movl    %r12d, %r14d
> -       movss   32(%rsp, %r14, 4), %xmm0
> +       # LOE rbx rbp r12 r13 r14 r15
> +       /* use rbp as index for special value that is saved across calls to
> +          tanhf. We technically don't need a callee save register here as offset
> +          to rsp is always [0, 12] so we can restore rsp by realigning to 64.
> +          Essentially the tradeoff is 1 extra save/restore vs 2 extra instructions
> +          in the loop.  */
> +       xorl    %ebp, %ebp
> +       bsfl    %ebx, %ebp
> +
> +       /* Scalar math fucntion call to process special input.  */
> +       movss   40(%rsp, %rbp, 4), %xmm0
>         call    atanhf@PLT
> -       # LOE rbx rbp r14 r15 r12d r13d xmm0
> -
> -       movss   %xmm0, 48(%rsp, %r14, 4)
> -
> -       /* Process special inputs in loop */
> -       jmp     L(SPECIAL_VALUES_LOOP)
> -       # LOE rbx rbp r15 r12d r13d
> +       /* No good way to avoid the store-forwarding fault this will cause on
> +          return. `lfence` avoids the SF fault but at greater cost as it
> +          serialized stack/callee save restoration.  */
> +       movss   %xmm0, 24(%rsp, %rbp, 4)
> +
> +       leal    -1(%rbx), %eax
> +       andl    %eax, %ebx
> +       jnz     L(SPECIAL_VALUES_LOOP)
> +       # LOE r12 r13 r14 r15
> +       /* All results have been written to 24(%rsp).  */
> +       movups  24(%rsp), %xmm0
> +       movq    (%rsp), %rbx
> +       cfi_restore(rbx)
> +       movq    8(%rsp), %rbp
> +       cfi_restore(rbp)
> +       addq    $56, %rsp
> +       cfi_def_cfa_offset(8)
> +       ret
>  END(_ZGVbN4v_atanhf_sse4)
>
>         .section .rodata, "a"
> @@ -305,56 +270,51 @@ END(_ZGVbN4v_atanhf_sse4)
>
>  #ifdef __svml_satanh_data_internal_typedef
>  typedef unsigned int VUINT32;
> -typedef struct {
> -       __declspec(align(16)) VUINT32 SgnMask[4][1];
> +typedef struct{
>         __declspec(align(16)) VUINT32 sOne[4][1];
> -       __declspec(align(16)) VUINT32 sPoly[8][4][1];
> +       __declspec(align(16)) VUINT32 SgnMask[4][1];
> +       __declspec(align(16)) VUINT32 sTopMask12[4][1];
>         __declspec(align(16)) VUINT32 iBrkValue[4][1];
>         __declspec(align(16)) VUINT32 iOffExpoMask[4][1];
> -       __declspec(align(16)) VUINT32 sHalf[4][1];
> -       __declspec(align(16)) VUINT32 sSign[4][1];
> -       __declspec(align(16)) VUINT32 sTopMask12[4][1];
> -       __declspec(align(16)) VUINT32 TinyRange[4][1];
> +       __declspec(align(16)) VUINT32 sPoly[8][4][1];
>         __declspec(align(16)) VUINT32 sLn2[4][1];
> +       __declspec(align(16)) VUINT32 TinyRange[4][1];
>  } __svml_satanh_data_internal;
>  #endif
> +
>  __svml_satanh_data_internal:
> -       /* SgnMask */
> -       .long   0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff
>         /* sOne = SP 1.0 */
>         .align  16
>         .long   0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000
> -       /* sPoly[] = SP polynomial */
> +       /* SgnMask */
> +       .long   0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff
> +       /* sTopMask12 */
>         .align  16
> -       .long   0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 /* -5.0000000000000000000000000e-01 P0 */
> -       .long   0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94 /* 3.3333265781402587890625000e-01 P1 */
> -       .long   0xbe80058e, 0xbe80058e, 0xbe80058e, 0xbe80058e /* -2.5004237890243530273437500e-01 P2 */
> -       .long   0x3e4ce190, 0x3e4ce190, 0x3e4ce190, 0x3e4ce190 /* 2.0007920265197753906250000e-01 P3 */
> -       .long   0xbe28ad37, 0xbe28ad37, 0xbe28ad37, 0xbe28ad37 /* -1.6472326219081878662109375e-01 P4 */
> -       .long   0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12 /* 1.4042308926582336425781250e-01 P5 */
> -       .long   0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3 /* -1.5122179687023162841796875e-01 P6 */
> -       .long   0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed /* 1.3820238411426544189453125e-01 P7 */
> +       .long   0xFFFFF000, 0xFFFFF000, 0xFFFFF000, 0xFFFFF000
>         /* iBrkValue = SP 2/3 */
>         .align  16
>         .long   0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab
> -       /* iOffExpoMask = SP significand mask */
> +       /* iOffExpoMask = SP significand mask ==*/
>         .align  16
>         .long   0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff
> -       /* sHalf */
> -       .align  16
> -       .long   0x3F000000, 0x3F000000, 0x3F000000, 0x3F000000
> -       /* sSign */
> +
> +       /* sPoly[] = SP polynomial */
>         .align  16
> -       .long   0x80000000, 0x80000000, 0x80000000, 0x80000000
> -       /* sTopMask12 */
> +       .long   0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed /* 1.3820238411426544189453125e-01 P7 */
> +       .long   0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3 /* -1.5122179687023162841796875e-01 P6 */
> +       .long   0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12 /* 1.4042308926582336425781250e-01 P5 */
> +       .long   0xbe28ad37, 0xbe28ad37, 0xbe28ad37, 0xbe28ad37 /* -1.6472326219081878662109375e-01 P4 */
> +       .long   0x3e4ce190, 0x3e4ce190, 0x3e4ce190, 0x3e4ce190 /* 2.0007920265197753906250000e-01 P3 */
> +       .long   0xbe80058e, 0xbe80058e, 0xbe80058e, 0xbe80058e /* -2.5004237890243530273437500e-01 P2 */
> +       .long   0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94 /* 3.3333265781402587890625000e-01 P1 */
> +       .long   0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 /* -5.0000000000000000000000000e-01 P0 */
> +
> +       /* sLn2 = SP ln(2) */
>         .align  16
> -       .long   0xFFFFF000, 0xFFFFF000, 0xFFFFF000, 0xFFFFF000
> +       .long   0x3f317218, 0x3f317218, 0x3f317218, 0x3f317218
>         /* TinyRange */
>         .align  16
>         .long   0x0C000000, 0x0C000000, 0x0C000000, 0x0C000000
> -       /* sLn2 = SP ln(2) */
> -       .align  16
> -       .long   0x3f317218, 0x3f317218, 0x3f317218, 0x3f317218
>         .align  16
>         .type   __svml_satanh_data_internal, @object
>         .size   __svml_satanh_data_internal, .-__svml_satanh_data_internal
> --
> 2.34.1
>

LGTM.

Thanks.
diff mbox series

Patch

diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S b/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S
index 2d3ad2617f..37200b3601 100644
--- a/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S
+++ b/sysdeps/x86_64/fpu/multiarch/svml_s_atanhf4_core_sse4.S
@@ -30,96 +30,80 @@ 
  *
  */
 
-/* Offsets for data table __svml_satanh_data_internal
- */
-#define SgnMask				0
-#define sOne				16
-#define sPoly				32
-#define iBrkValue			160
-#define iOffExpoMask			176
-#define sHalf				192
-#define sSign				208
-#define sTopMask12			224
-#define TinyRange			240
-#define sLn2				256
+/* Offsets for data table __svml_satanh_data_internal_avx512. Ordered
+   by use in the function. On cold-starts this might help the
+   prefetcher. Possibly a better idea is to interleave start/end so
+   that the prefetcher is less likely to detect a stream and pull
+   irrelivant lines into cache.  */
+#define sOne				0
+#define SgnMask				16
+#define sTopMask12			32
+#define iBrkValue			48
+#define iOffExpoMask			64
+#define sPoly				80
+#define sLn2				208
+#define TinyRange			224
 
 #include <sysdep.h>
+#define ATANHF_DATA(x)			((x)+__svml_satanh_data_internal)
 
 	.section .text.sse4, "ax", @progbits
 ENTRY(_ZGVbN4v_atanhf_sse4)
-	subq	$72, %rsp
-	cfi_def_cfa_offset(80)
 	movaps	%xmm0, %xmm5
 
 	/* Load constants including One = 1 */
-	movups	sOne+__svml_satanh_data_internal(%rip), %xmm4
+	movups	ATANHF_DATA(sOne)(%rip), %xmm4
 	movaps	%xmm5, %xmm3
 
 	/* Strip off the sign, so treat X as positive until right at the end */
-	movups	SgnMask+__svml_satanh_data_internal(%rip), %xmm7
-	movaps	%xmm4, %xmm8
-	andps	%xmm5, %xmm7
+	movups	ATANHF_DATA(SgnMask)(%rip), %xmm1
+	movaps	%xmm4, %xmm2
+	andps	%xmm1, %xmm0
 	movaps	%xmm4, %xmm10
-	movups	sTopMask12+__svml_satanh_data_internal(%rip), %xmm11
+	movups	ATANHF_DATA(sTopMask12)(%rip), %xmm11
 	movaps	%xmm4, %xmm14
 	movaps	%xmm11, %xmm9
 
+
 	/*
 	 * Compute V = 2 * X trivially, and UHi + U_lo = 1 - X in two pieces,
 	 * the upper part UHi being <= 12 bits long. Then we have
 	 * atanh(X) = 1/2 * log((1 + X) / (1 - X)) = 1/2 * log1p(V / (UHi + ULo)).
 	 */
-	movaps	%xmm7, %xmm12
+	movaps	%xmm0, %xmm6
+	mulps	%xmm5, %xmm3
+	subps	%xmm0, %xmm2
+	addps	%xmm0, %xmm6
+	subps	%xmm2, %xmm10
+	addps	%xmm5, %xmm3
+	subps	%xmm0, %xmm10
+	andps	%xmm2, %xmm9
+
 
 	/*
 	 * Check whether |X| < 1, in which case we use the main function.
 	 * Otherwise set the rangemask so that the callout will get used.
 	 * Note that this will also use the callout for NaNs since not(NaN < 1).
 	 */
-	movaps	%xmm7, %xmm6
-	movaps	%xmm7, %xmm2
-	cmpnltps %xmm4, %xmm6
-	cmpltps	TinyRange+__svml_satanh_data_internal(%rip), %xmm2
-	mulps	%xmm5, %xmm3
-	subps	%xmm7, %xmm8
-	addps	%xmm7, %xmm12
-	movmskps %xmm6, %edx
-	subps	%xmm8, %xmm10
-	addps	%xmm5, %xmm3
-	subps	%xmm7, %xmm10
-	andps	%xmm8, %xmm9
+	rcpps	%xmm9, %xmm7
+	subps	%xmm9, %xmm2
+	andps	%xmm11, %xmm7
 
-	/*
-	 * Now we feed into the log1p code, using H in place of _VARG1 and
-	 * later incorporating L into the reduced argument.
-	 * compute 1+x as high, low parts
-	 */
-	movaps	%xmm4, %xmm7
-
-	/*
-	 * Now compute R = 1/(UHi+ULo) * (1 - E) and the error term E
-	 * The first FMR is exact (we force R to 12 bits just in case it
-	 * isn't already, to make absolutely sure), and since E is ~ 2^-12,
-	 * the rounding error in the other one is acceptable.
-	 */
-	rcpps	%xmm9, %xmm15
-	subps	%xmm9, %xmm8
-	andps	%xmm11, %xmm15
 
 	/*
 	 * Split V as well into upper 12 bits and lower part, so that we can get
 	 * a preliminary quotient estimate without rounding error.
 	 */
-	andps	%xmm12, %xmm11
-	mulps	%xmm15, %xmm9
-	addps	%xmm8, %xmm10
-	subps	%xmm11, %xmm12
+	andps	%xmm6, %xmm11
+	mulps	%xmm7, %xmm9
+	addps	%xmm2, %xmm10
+	subps	%xmm11, %xmm6
 
 	/* Hence get initial quotient estimate QHi + QLo = R * VHi + R * VLo */
-	mulps	%xmm15, %xmm11
-	mulps	%xmm15, %xmm10
+	mulps	%xmm7, %xmm11
+	mulps	%xmm7, %xmm10
 	subps	%xmm9, %xmm14
-	mulps	%xmm12, %xmm15
+	mulps	%xmm6, %xmm7
 	subps	%xmm10, %xmm14
 
 	/* Compute D = E + E^2 */
@@ -127,8 +111,8 @@  ENTRY(_ZGVbN4v_atanhf_sse4)
 	movaps	%xmm4, %xmm8
 	mulps	%xmm14, %xmm13
 
-	/* reduction: compute r, n */
-	movdqu	iBrkValue+__svml_satanh_data_internal(%rip), %xmm9
+	/* reduction: compute r,n */
+	movdqu	ATANHF_DATA(iBrkValue)(%rip), %xmm9
 	addps	%xmm13, %xmm14
 
 	/*
@@ -136,168 +120,149 @@  ENTRY(_ZGVbN4v_atanhf_sse4)
 	 * = R *  (VHi + VLo) * (1 + D)
 	 * = QHi + (QHi * D + QLo + QLo * D)
 	 */
-	movaps	%xmm14, %xmm0
-	mulps	%xmm15, %xmm14
-	mulps	%xmm11, %xmm0
-	addps	%xmm14, %xmm15
-	movdqu	iOffExpoMask+__svml_satanh_data_internal(%rip), %xmm12
+	movaps	%xmm14, %xmm2
+	mulps	%xmm7, %xmm14
+	mulps	%xmm11, %xmm2
+	addps	%xmm14, %xmm7
+	movdqu	ATANHF_DATA(iOffExpoMask)(%rip), %xmm12
 	movaps	%xmm4, %xmm14
 
 	/* Record the sign for eventual reincorporation. */
-	movups	sSign+__svml_satanh_data_internal(%rip), %xmm1
-	addps	%xmm15, %xmm0
+	addps	%xmm7, %xmm2
+
 
 	/*
 	 * Now finally accumulate the high and low parts of the
 	 * argument to log1p, H + L, with a final compensated summation.
 	 */
-	movaps	%xmm0, %xmm6
-	andps	%xmm5, %xmm1
-
+	movaps	%xmm2, %xmm6
+	andnps	%xmm5, %xmm1
+	movaps	%xmm4, %xmm7
 	/* Or the sign bit in with the tiny result to handle atanh(-0) correctly */
-	orps	%xmm1, %xmm3
 	addps	%xmm11, %xmm6
 	maxps	%xmm6, %xmm7
 	minps	%xmm6, %xmm8
 	subps	%xmm6, %xmm11
 	movaps	%xmm7, %xmm10
-	andps	%xmm2, %xmm3
 	addps	%xmm8, %xmm10
-	addps	%xmm11, %xmm0
+	addps	%xmm11, %xmm2
 	subps	%xmm10, %xmm7
 	psubd	%xmm9, %xmm10
-	addps	%xmm7, %xmm8
+	addps	%xmm8, %xmm7
 	pand	%xmm10, %xmm12
 	psrad	$23, %xmm10
 	cvtdq2ps %xmm10, %xmm13
-	addps	%xmm8, %xmm0
+	addps	%xmm7, %xmm2
 
 	/* final reconstruction */
-	mulps	sLn2+__svml_satanh_data_internal(%rip), %xmm13
 	pslld	$23, %xmm10
 	paddd	%xmm9, %xmm12
 	psubd	%xmm10, %xmm14
 
 	/* polynomial evaluation */
 	subps	%xmm4, %xmm12
-	mulps	%xmm0, %xmm14
-	movups	sPoly+112+__svml_satanh_data_internal(%rip), %xmm0
-	addps	%xmm12, %xmm14
-	mulps	%xmm14, %xmm0
+	mulps	%xmm14, %xmm2
+	movups	ATANHF_DATA(sPoly+0)(%rip), %xmm7
+	addps	%xmm12, %xmm2
+	mulps	%xmm2, %xmm7
+
 
 	/* Finally, halve the result and reincorporate the sign */
-	movups	sHalf+__svml_satanh_data_internal(%rip), %xmm4
-	pxor	%xmm1, %xmm4
-	addps	sPoly+96+__svml_satanh_data_internal(%rip), %xmm0
-	mulps	%xmm14, %xmm0
-	addps	sPoly+80+__svml_satanh_data_internal(%rip), %xmm0
-	mulps	%xmm14, %xmm0
-	addps	sPoly+64+__svml_satanh_data_internal(%rip), %xmm0
-	mulps	%xmm14, %xmm0
-	addps	sPoly+48+__svml_satanh_data_internal(%rip), %xmm0
-	mulps	%xmm14, %xmm0
-	addps	sPoly+32+__svml_satanh_data_internal(%rip), %xmm0
-	mulps	%xmm14, %xmm0
-	addps	sPoly+16+__svml_satanh_data_internal(%rip), %xmm0
-	mulps	%xmm14, %xmm0
-	addps	sPoly+__svml_satanh_data_internal(%rip), %xmm0
-	mulps	%xmm14, %xmm0
-	mulps	%xmm14, %xmm0
-	addps	%xmm0, %xmm14
-	movaps	%xmm2, %xmm0
-	addps	%xmm13, %xmm14
-	mulps	%xmm14, %xmm4
-	andnps	%xmm4, %xmm0
-	orps	%xmm3, %xmm0
-	testl	%edx, %edx
+	addps	ATANHF_DATA(sPoly+16)(%rip), %xmm7
+	mulps	%xmm2, %xmm7
+	addps	ATANHF_DATA(sPoly+32)(%rip), %xmm7
+	mulps	%xmm2, %xmm7
+	addps	ATANHF_DATA(sPoly+48)(%rip), %xmm7
+	mulps	%xmm2, %xmm7
+	addps	ATANHF_DATA(sPoly+64)(%rip), %xmm7
+	mulps	%xmm2, %xmm7
+	addps	ATANHF_DATA(sPoly+80)(%rip), %xmm7
+	mulps	%xmm2, %xmm7
+	addps	ATANHF_DATA(sPoly+96)(%rip), %xmm7
+	mulps	%xmm2, %xmm7
+	movaps	ATANHF_DATA(sPoly+112)(%rip), %xmm6
+	addps	%xmm6, %xmm7
+	mulps	%xmm2, %xmm7
+	mulps	%xmm2, %xmm7
+	mulps	ATANHF_DATA(sLn2)(%rip), %xmm13
+	/* We can build `sHalf` with `sPoly & sOne`.  */
+	andps	%xmm4, %xmm6
+	orps	%xmm1, %xmm3
+	xorps	%xmm6, %xmm1
 
-	/* Go to special inputs processing branch */
-	jne	L(SPECIAL_VALUES_BRANCH)
-	# LOE rbx rbp r12 r13 r14 r15 edx xmm0 xmm5
+	addps	%xmm2, %xmm7
+	addps	%xmm13, %xmm7
+	mulps	%xmm7, %xmm1
 
-	/* Restore registers
-	 * and exit the function
-	 */
+	/* Finish check of NaNs.  */
+	cmpleps	%xmm0, %xmm4
+	movmskps %xmm4, %edx
+	cmpltps	ATANHF_DATA(TinyRange)(%rip), %xmm0
 
-L(EXIT):
-	addq	$72, %rsp
-	cfi_def_cfa_offset(8)
+	andps	%xmm0, %xmm3
+	andnps	%xmm1, %xmm0
+	orps	%xmm3, %xmm0
+
+	testl	%edx, %edx
+	/* Go to special inputs processing branch.  */
+	jne	L(SPECIAL_VALUES_BRANCH)
+	# LOE rbx rbp r12 r13 r14 r15 xmm0
+	/* No registers to restore on fast path.  */
 	ret
-	cfi_def_cfa_offset(80)
 
-	/* Branch to process
-	 * special inputs
-	 */
 
+	/* Cold case. edx has 1s where there was a special value that
+	   needs to be handled by a atanhf call. Optimize for code size
+	   more so than speed here. */
 L(SPECIAL_VALUES_BRANCH):
-	movups	%xmm5, 32(%rsp)
-	movups	%xmm0, 48(%rsp)
-	# LOE rbx rbp r12 r13 r14 r15 edx
-
-	xorl	%eax, %eax
-	movq	%r12, 16(%rsp)
-	cfi_offset(12, -64)
-	movl	%eax, %r12d
-	movq	%r13, 8(%rsp)
-	cfi_offset(13, -72)
-	movl	%edx, %r13d
-	movq	%r14, (%rsp)
-	cfi_offset(14, -80)
-	# LOE rbx rbp r15 r12d r13d
-
-	/* Range mask
-	 * bits check
-	 */
-
-L(RANGEMASK_CHECK):
-	btl	%r12d, %r13d
-
-	/* Call scalar math function */
-	jc	L(SCALAR_MATH_CALL)
-	# LOE rbx rbp r15 r12d r13d
-
-	/* Special inputs
-	 * processing loop
-	 */
-
+	# LOE rbx rdx rbp r12 r13 r14 r15 xmm0 xmm5
+	/* Stack coming in 16-byte aligned. Set 8-byte misaligned so on
+       call entry will be 16-byte aligned. */
+	subq	$56, %rsp
+	cfi_def_cfa_offset(64)
+	movups	%xmm0, 24(%rsp)
+	movups	%xmm5, 40(%rsp)
+
+	/* Use rbx/rbp for callee save registers as they get short
+       encoding for many instructions (as compared with r12/r13). */
+	movq	%rbx, (%rsp)
+	cfi_offset(rbx, -64)
+	movq	%rbp, 8(%rsp)
+	cfi_offset(rbp, -56)
+	/* edx has 1s where there was a special value that needs to be handled
+	   by a tanhf call.  */
+	movl	%edx, %ebx
 L(SPECIAL_VALUES_LOOP):
-	incl	%r12d
-	cmpl	$4, %r12d
-
-	/* Check bits in range mask */
-	jl	L(RANGEMASK_CHECK)
-	# LOE rbx rbp r15 r12d r13d
-
-	movq	16(%rsp), %r12
-	cfi_restore(12)
-	movq	8(%rsp), %r13
-	cfi_restore(13)
-	movq	(%rsp), %r14
-	cfi_restore(14)
-	movups	48(%rsp), %xmm0
-
-	/* Go to exit */
-	jmp	L(EXIT)
-	cfi_offset(12, -64)
-	cfi_offset(13, -72)
-	cfi_offset(14, -80)
-	# LOE rbx rbp r12 r13 r14 r15 xmm0
-
-	/* Scalar math fucntion call
-	 * to process special input
-	 */
-
-L(SCALAR_MATH_CALL):
-	movl	%r12d, %r14d
-	movss	32(%rsp, %r14, 4), %xmm0
+	# LOE rbx rbp r12 r13 r14 r15
+	/* use rbp as index for special value that is saved across calls to
+	   tanhf. We technically don't need a callee save register here as offset
+	   to rsp is always [0, 12] so we can restore rsp by realigning to 64.
+	   Essentially the tradeoff is 1 extra save/restore vs 2 extra instructions
+	   in the loop.  */
+	xorl	%ebp, %ebp
+	bsfl	%ebx, %ebp
+
+	/* Scalar math fucntion call to process special input.  */
+	movss	40(%rsp, %rbp, 4), %xmm0
 	call	atanhf@PLT
-	# LOE rbx rbp r14 r15 r12d r13d xmm0
-
-	movss	%xmm0, 48(%rsp, %r14, 4)
-
-	/* Process special inputs in loop */
-	jmp	L(SPECIAL_VALUES_LOOP)
-	# LOE rbx rbp r15 r12d r13d
+	/* No good way to avoid the store-forwarding fault this will cause on
+	   return. `lfence` avoids the SF fault but at greater cost as it
+	   serialized stack/callee save restoration.  */
+	movss	%xmm0, 24(%rsp, %rbp, 4)
+
+	leal	-1(%rbx), %eax
+	andl	%eax, %ebx
+	jnz	L(SPECIAL_VALUES_LOOP)
+	# LOE r12 r13 r14 r15
+	/* All results have been written to 24(%rsp).  */
+	movups	24(%rsp), %xmm0
+	movq	(%rsp), %rbx
+	cfi_restore(rbx)
+	movq	8(%rsp), %rbp
+	cfi_restore(rbp)
+	addq	$56, %rsp
+	cfi_def_cfa_offset(8)
+	ret
 END(_ZGVbN4v_atanhf_sse4)
 
 	.section .rodata, "a"
@@ -305,56 +270,51 @@  END(_ZGVbN4v_atanhf_sse4)
 
 #ifdef __svml_satanh_data_internal_typedef
 typedef unsigned int VUINT32;
-typedef struct {
-	__declspec(align(16)) VUINT32 SgnMask[4][1];
+typedef struct{
 	__declspec(align(16)) VUINT32 sOne[4][1];
-	__declspec(align(16)) VUINT32 sPoly[8][4][1];
+	__declspec(align(16)) VUINT32 SgnMask[4][1];
+	__declspec(align(16)) VUINT32 sTopMask12[4][1];
 	__declspec(align(16)) VUINT32 iBrkValue[4][1];
 	__declspec(align(16)) VUINT32 iOffExpoMask[4][1];
-	__declspec(align(16)) VUINT32 sHalf[4][1];
-	__declspec(align(16)) VUINT32 sSign[4][1];
-	__declspec(align(16)) VUINT32 sTopMask12[4][1];
-	__declspec(align(16)) VUINT32 TinyRange[4][1];
+	__declspec(align(16)) VUINT32 sPoly[8][4][1];
 	__declspec(align(16)) VUINT32 sLn2[4][1];
+	__declspec(align(16)) VUINT32 TinyRange[4][1];
 } __svml_satanh_data_internal;
 #endif
+
 __svml_satanh_data_internal:
-	/* SgnMask */
-	.long	0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff
 	/* sOne = SP 1.0 */
 	.align	16
 	.long	0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000
-	/* sPoly[] = SP polynomial */
+	/* SgnMask */
+	.long	0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff
+	/* sTopMask12 */
 	.align	16
-	.long	0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 /* -5.0000000000000000000000000e-01 P0 */
-	.long	0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94 /* 3.3333265781402587890625000e-01 P1 */
-	.long	0xbe80058e, 0xbe80058e, 0xbe80058e, 0xbe80058e /* -2.5004237890243530273437500e-01 P2 */
-	.long	0x3e4ce190, 0x3e4ce190, 0x3e4ce190, 0x3e4ce190 /* 2.0007920265197753906250000e-01 P3 */
-	.long	0xbe28ad37, 0xbe28ad37, 0xbe28ad37, 0xbe28ad37 /* -1.6472326219081878662109375e-01 P4 */
-	.long	0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12 /* 1.4042308926582336425781250e-01 P5 */
-	.long	0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3 /* -1.5122179687023162841796875e-01 P6 */
-	.long	0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed /* 1.3820238411426544189453125e-01 P7 */
+	.long	0xFFFFF000, 0xFFFFF000, 0xFFFFF000, 0xFFFFF000
 	/* iBrkValue = SP 2/3 */
 	.align	16
 	.long	0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab
-	/* iOffExpoMask = SP significand mask */
+	/* iOffExpoMask = SP significand mask ==*/
 	.align	16
 	.long	0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff
-	/* sHalf */
-	.align	16
-	.long	0x3F000000, 0x3F000000, 0x3F000000, 0x3F000000
-	/* sSign */
+
+	/* sPoly[] = SP polynomial */
 	.align	16
-	.long	0x80000000, 0x80000000, 0x80000000, 0x80000000
-	/* sTopMask12 */
+	.long	0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed, 0x3e0d84ed /* 1.3820238411426544189453125e-01 P7 */
+	.long	0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3, 0xbe1ad9e3 /* -1.5122179687023162841796875e-01 P6 */
+	.long	0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12, 0x3e0fcb12 /* 1.4042308926582336425781250e-01 P5 */
+	.long	0xbe28ad37, 0xbe28ad37, 0xbe28ad37, 0xbe28ad37 /* -1.6472326219081878662109375e-01 P4 */
+	.long	0x3e4ce190, 0x3e4ce190, 0x3e4ce190, 0x3e4ce190 /* 2.0007920265197753906250000e-01 P3 */
+	.long	0xbe80058e, 0xbe80058e, 0xbe80058e, 0xbe80058e /* -2.5004237890243530273437500e-01 P2 */
+	.long	0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94, 0x3eaaaa94 /* 3.3333265781402587890625000e-01 P1 */
+	.long	0xbf000000, 0xbf000000, 0xbf000000, 0xbf000000 /* -5.0000000000000000000000000e-01 P0 */
+
+	/* sLn2 = SP ln(2) */
 	.align	16
-	.long	0xFFFFF000, 0xFFFFF000, 0xFFFFF000, 0xFFFFF000
+	.long	0x3f317218, 0x3f317218, 0x3f317218, 0x3f317218
 	/* TinyRange */
 	.align	16
 	.long	0x0C000000, 0x0C000000, 0x0C000000, 0x0C000000
-	/* sLn2 = SP ln(2) */
-	.align	16
-	.long	0x3f317218, 0x3f317218, 0x3f317218, 0x3f317218
 	.align	16
 	.type	__svml_satanh_data_internal, @object
 	.size	__svml_satanh_data_internal, .-__svml_satanh_data_internal