lib/crypto/arm64/poly1305-armv8.pl - third_party/linux - Git at Google


            
            #!/usr/bin/env perl

            
            # SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause

            
            #

            
            # ====================================================================

            
            # Written by Andy Polyakov, @dot-asm, initially for the OpenSSL

            
            # project.

            
            # ====================================================================

            
            #

            
            # This module implements Poly1305 hash for ARMv8.

            
            #

            
            # June 2015

            
            #

            
            # Numbers are cycles per processed byte with poly1305_blocks alone.

            
            #

            
            #		IALU/gcc-4.9	NEON

            
            #

            
            # Apple A7	1.86/+5%	0.72

            
            # Cortex-A53	2.69/+58%	1.47

            
            # Cortex-A57	2.70/+7%	1.14

            
            # Denver	1.64/+50%	1.18(*)

            
            # X-Gene	2.13/+68%	2.27

            
            # Mongoose	1.77/+75%	1.12

            
            # Kryo		2.70/+55%	1.13

            
            # ThunderX2	1.17/+95%	1.36

            
            #

            
            # (*)	estimate based on resources availability is less than 1.0,

            
            #	i.e. measured result is worse than expected, presumably binary

            
            #	translator is not almighty;


            $flavour=shift;

            
            $output=shift;


            if ($flavour && $flavour ne "void") {

            
                $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

            
                ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or

            
                ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or

            
                die "can't locate arm-xlate.pl";


                open STDOUT,"| \"$^X\" $xlate $flavour $output";

            
            } else {

            
                open STDOUT,">$output";

            
            }


            my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));

            
            my ($mac,$nonce)=($inp,$len);


            my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));


            $code.=<<___;

            
            #ifndef __KERNEL__

            
            # include "arm_arch.h"

            
            .extern	OPENSSL_armcap_P

            
            #else

            
            # define poly1305_init   poly1305_block_init

            
            # define poly1305_blocks poly1305_blocks_arm64

            
            #endif


            .text


            // forward "declarations" are required for Apple

            
            .globl	poly1305_blocks

            
            .globl	poly1305_emit


            .globl	poly1305_init

            
            .type	poly1305_init,%function

            
            .align	5

            
            poly1305_init:

            
            	cmp	$inp,xzr

            
            	stp	xzr,xzr,[$ctx]		// zero hash value

            
            	stp	xzr,xzr,[$ctx,#16]	// [along with is_base2_26]


            	csel	x0,xzr,x0,eq

            
            	b.eq	.Lno_key


            #ifndef	__KERNEL__

            
            	adrp	x17,OPENSSL_armcap_P

            
            	ldr	w17,[x17,#:lo12:OPENSSL_armcap_P]

            
            #endif


            	ldp	$r0,$r1,[$inp]		// load key

            
            	mov	$s1,#0xfffffffc0fffffff

            
            	movk	$s1,#0x0fff,lsl#48

            
            #ifdef	__AARCH64EB__

            
            	rev	$r0,$r0			// flip bytes

            
            	rev	$r1,$r1

            
            #endif

            
            	and	$r0,$r0,$s1		// &=0ffffffc0fffffff

            
            	and	$s1,$s1,#-4

            
            	and	$r1,$r1,$s1		// &=0ffffffc0ffffffc

            
            	mov	w#$s1,#-1

            
            	stp	$r0,$r1,[$ctx,#32]	// save key value

            
            	str	w#$s1,[$ctx,#48]	// impossible key power value


            #ifndef	__KERNEL__

            
            	tst	w17,#ARMV7_NEON


            	adr	$d0,.Lpoly1305_blocks

            
            	adr	$r0,.Lpoly1305_blocks_neon

            
            	adr	$d1,.Lpoly1305_emit


            	csel	$d0,$d0,$r0,eq


            # ifdef	__ILP32__

            
            	stp	w#$d0,w#$d1,[$len]

            
            # else

            
            	stp	$d0,$d1,[$len]

            
            # endif

            
            #endif

            
            	mov	x0,#1

            
            .Lno_key:

            
            	ret

            
            .size	poly1305_init,.-poly1305_init


            .type	poly1305_blocks,%function

            
            .align	5

            
            poly1305_blocks:

            
            .Lpoly1305_blocks:

            
            	ands	$len,$len,#-16

            
            	b.eq	.Lno_data


            	ldp	$h0,$h1,[$ctx]		// load hash value

            
            	ldp	$h2,x17,[$ctx,#16]	// [along with is_base2_26]

            
            	ldp	$r0,$r1,[$ctx,#32]	// load key value


            #ifdef	__AARCH64EB__

            
            	lsr	$d0,$h0,#32

            
            	mov	w#$d1,w#$h0

            
            	lsr	$d2,$h1,#32

            
            	mov	w15,w#$h1

            
            	lsr	x16,$h2,#32

            
            #else

            
            	mov	w#$d0,w#$h0

            
            	lsr	$d1,$h0,#32

            
            	mov	w#$d2,w#$h1

            
            	lsr	x15,$h1,#32

            
            	mov	w16,w#$h2

            
            #endif


            	add	$d0,$d0,$d1,lsl#26	// base 2^26 -> base 2^64

            
            	lsr	$d1,$d2,#12

            
            	adds	$d0,$d0,$d2,lsl#52

            
            	add	$d1,$d1,x15,lsl#14

            
            	adc	$d1,$d1,xzr

            
            	lsr	$d2,x16,#24

            
            	adds	$d1,$d1,x16,lsl#40

            
            	adc	$d2,$d2,xzr


            	cmp	x17,#0			// is_base2_26?

            
            	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)

            
            	csel	$h0,$h0,$d0,eq		// choose between radixes

            
            	csel	$h1,$h1,$d1,eq

            
            	csel	$h2,$h2,$d2,eq


            .Loop:

            
            	ldp	$t0,$t1,[$inp],#16	// load input

            
            	sub	$len,$len,#16

            
            #ifdef	__AARCH64EB__

            
            	rev	$t0,$t0

            
            	rev	$t1,$t1

            
            #endif

            
            	adds	$h0,$h0,$t0		// accumulate input

            
            	adcs	$h1,$h1,$t1


            	mul	$d0,$h0,$r0		// h0*r0

            
            	adc	$h2,$h2,$padbit

            
            	umulh	$d1,$h0,$r0


            	mul	$t0,$h1,$s1		// h1*5*r1

            
            	umulh	$t1,$h1,$s1


            	adds	$d0,$d0,$t0

            
            	mul	$t0,$h0,$r1		// h0*r1

            
            	adc	$d1,$d1,$t1

            
            	umulh	$d2,$h0,$r1


            	adds	$d1,$d1,$t0

            
            	mul	$t0,$h1,$r0		// h1*r0

            
            	adc	$d2,$d2,xzr

            
            	umulh	$t1,$h1,$r0


            	adds	$d1,$d1,$t0

            
            	mul	$t0,$h2,$s1		// h2*5*r1

            
            	adc	$d2,$d2,$t1

            
            	mul	$t1,$h2,$r0		// h2*r0


            	adds	$d1,$d1,$t0

            
            	adc	$d2,$d2,$t1


            	and	$t0,$d2,#-4		// final reduction

            
            	and	$h2,$d2,#3

            
            	add	$t0,$t0,$d2,lsr#2

            
            	adds	$h0,$d0,$t0

            
            	adcs	$h1,$d1,xzr

            
            	adc	$h2,$h2,xzr


            	cbnz	$len,.Loop


            	stp	$h0,$h1,[$ctx]		// store hash value

            
            	stp	$h2,xzr,[$ctx,#16]	// [and clear is_base2_26]


            .Lno_data:

            
            	ret

            
            .size	poly1305_blocks,.-poly1305_blocks


            .type	poly1305_emit,%function

            
            .align	5

            
            poly1305_emit:

            
            .Lpoly1305_emit:

            
            	ldp	$h0,$h1,[$ctx]		// load hash base 2^64

            
            	ldp	$h2,$r0,[$ctx,#16]	// [along with is_base2_26]

            
            	ldp	$t0,$t1,[$nonce]	// load nonce


            #ifdef	__AARCH64EB__

            
            	lsr	$d0,$h0,#32

            
            	mov	w#$d1,w#$h0

            
            	lsr	$d2,$h1,#32

            
            	mov	w15,w#$h1

            
            	lsr	x16,$h2,#32

            
            #else

            
            	mov	w#$d0,w#$h0

            
            	lsr	$d1,$h0,#32

            
            	mov	w#$d2,w#$h1

            
            	lsr	x15,$h1,#32

            
            	mov	w16,w#$h2

            
            #endif


            	add	$d0,$d0,$d1,lsl#26	// base 2^26 -> base 2^64

            
            	lsr	$d1,$d2,#12

            
            	adds	$d0,$d0,$d2,lsl#52

            
            	add	$d1,$d1,x15,lsl#14

            
            	adc	$d1,$d1,xzr

            
            	lsr	$d2,x16,#24

            
            	adds	$d1,$d1,x16,lsl#40

            
            	adc	$d2,$d2,xzr


            	cmp	$r0,#0			// is_base2_26?

            
            	csel	$h0,$h0,$d0,eq		// choose between radixes

            
            	csel	$h1,$h1,$d1,eq

            
            	csel	$h2,$h2,$d2,eq


            	adds	$d0,$h0,#5		// compare to modulus

            
            	adcs	$d1,$h1,xzr

            
            	adc	$d2,$h2,xzr


            	tst	$d2,#-4			// see if it's carried/borrowed


            	csel	$h0,$h0,$d0,eq

            
            	csel	$h1,$h1,$d1,eq


            #ifdef	__AARCH64EB__

            
            	ror	$t0,$t0,#32		// flip nonce words

            
            	ror	$t1,$t1,#32

            
            #endif

            
            	adds	$h0,$h0,$t0		// accumulate nonce

            
            	adc	$h1,$h1,$t1

            
            #ifdef	__AARCH64EB__

            
            	rev	$h0,$h0			// flip output bytes

            
            	rev	$h1,$h1

            
            #endif

            
            	stp	$h0,$h1,[$mac]		// write result


            	ret

            
            .size	poly1305_emit,.-poly1305_emit

            
            ___

            
            my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));

            
            my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));

            
            my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));

            
            my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));

            
            my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));

            
            my ($T0,$T1,$MASK) = map("v$_",(29..31));


            my ($in2,$zeros)=("x16","x17");

            
            my $is_base2_26 = $zeros;		# borrow


            $code.=<<___;

            
            .type	poly1305_mult,%function

            
            .align	5

            
            poly1305_mult:

            
            	mul	$d0,$h0,$r0		// h0*r0

            
            	umulh	$d1,$h0,$r0


            	mul	$t0,$h1,$s1		// h1*5*r1

            
            	umulh	$t1,$h1,$s1


            	adds	$d0,$d0,$t0

            
            	mul	$t0,$h0,$r1		// h0*r1

            
            	adc	$d1,$d1,$t1

            
            	umulh	$d2,$h0,$r1


            	adds	$d1,$d1,$t0

            
            	mul	$t0,$h1,$r0		// h1*r0

            
            	adc	$d2,$d2,xzr

            
            	umulh	$t1,$h1,$r0


            	adds	$d1,$d1,$t0

            
            	mul	$t0,$h2,$s1		// h2*5*r1

            
            	adc	$d2,$d2,$t1

            
            	mul	$t1,$h2,$r0		// h2*r0


            	adds	$d1,$d1,$t0

            
            	adc	$d2,$d2,$t1


            	and	$t0,$d2,#-4		// final reduction

            
            	and	$h2,$d2,#3

            
            	add	$t0,$t0,$d2,lsr#2

            
            	adds	$h0,$d0,$t0

            
            	adcs	$h1,$d1,xzr

            
            	adc	$h2,$h2,xzr


            	ret

            
            .size	poly1305_mult,.-poly1305_mult


            .type	poly1305_splat,%function

            
            .align	4

            
            poly1305_splat:

            
            	and	x12,$h0,#0x03ffffff	// base 2^64 -> base 2^26

            
            	ubfx	x13,$h0,#26,#26

            
            	extr	x14,$h1,$h0,#52

            
            	and	x14,x14,#0x03ffffff

            
            	ubfx	x15,$h1,#14,#26

            
            	extr	x16,$h2,$h1,#40


            	str	w12,[$ctx,#16*0]	// r0

            
            	add	w12,w13,w13,lsl#2	// r1*5

            
            	str	w13,[$ctx,#16*1]	// r1

            
            	add	w13,w14,w14,lsl#2	// r2*5

            
            	str	w12,[$ctx,#16*2]	// s1

            
            	str	w14,[$ctx,#16*3]	// r2

            
            	add	w14,w15,w15,lsl#2	// r3*5

            
            	str	w13,[$ctx,#16*4]	// s2

            
            	str	w15,[$ctx,#16*5]	// r3

            
            	add	w15,w16,w16,lsl#2	// r4*5

            
            	str	w14,[$ctx,#16*6]	// s3

            
            	str	w16,[$ctx,#16*7]	// r4

            
            	str	w15,[$ctx,#16*8]	// s4


            	ret

            
            .size	poly1305_splat,.-poly1305_splat


            #ifdef	__KERNEL__

            
            .globl	poly1305_blocks_neon

            
            #endif

            
            .type	poly1305_blocks_neon,%function

            
            .align	5

            
            poly1305_blocks_neon:

            
            .Lpoly1305_blocks_neon:

            
            	ldr	$is_base2_26,[$ctx,#24]

            
            	cmp	$len,#128

            
            	b.lo	.Lpoly1305_blocks


            	.inst	0xd503233f		// paciasp

            
            	stp	x29,x30,[sp,#-80]!

            
            	add	x29,sp,#0


            	stp	d8,d9,[sp,#16]		// meet ABI requirements

            
            	stp	d10,d11,[sp,#32]

            
            	stp	d12,d13,[sp,#48]

            
            	stp	d14,d15,[sp,#64]


            	cbz	$is_base2_26,.Lbase2_64_neon


            	ldp	w10,w11,[$ctx]		// load hash value base 2^26

            
            	ldp	w12,w13,[$ctx,#8]

            
            	ldr	w14,[$ctx,#16]


            	tst	$len,#31

            
            	b.eq	.Leven_neon


            	ldp	$r0,$r1,[$ctx,#32]	// load key value


            	add	$h0,x10,x11,lsl#26	// base 2^26 -> base 2^64

            
            	lsr	$h1,x12,#12

            
            	adds	$h0,$h0,x12,lsl#52

            
            	add	$h1,$h1,x13,lsl#14

            
            	adc	$h1,$h1,xzr

            
            	lsr	$h2,x14,#24

            
            	adds	$h1,$h1,x14,lsl#40

            
            	adc	$h2,$h2,xzr		// preserve carry into top limb


            	ldp	$d0,$d1,[$inp],#16	// load input

            
            	sub	$len,$len,#16

            
            	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)


            #ifdef	__AARCH64EB__

            
            	rev	$d0,$d0

            
            	rev	$d1,$d1

            
            #endif

            
            	adds	$h0,$h0,$d0		// accumulate input

            
            	adcs	$h1,$h1,$d1

            
            	adc	$h2,$h2,$padbit


            	bl	poly1305_mult


            	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26

            
            	ubfx	x11,$h0,#26,#26

            
            	extr	x12,$h1,$h0,#52

            
            	and	x12,x12,#0x03ffffff

            
            	ubfx	x13,$h1,#14,#26

            
            	extr	x14,$h2,$h1,#40


            	b	.Leven_neon


            .align	4

            
            .Lbase2_64_neon:

            
            	ldp	$r0,$r1,[$ctx,#32]	// load key value


            	ldp	$h0,$h1,[$ctx]		// load hash value base 2^64

            
            	ldr	$h2,[$ctx,#16]


            	tst	$len,#31

            
            	b.eq	.Linit_neon


            	ldp	$d0,$d1,[$inp],#16	// load input

            
            	sub	$len,$len,#16

            
            	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)

            
            #ifdef	__AARCH64EB__

            
            	rev	$d0,$d0

            
            	rev	$d1,$d1

            
            #endif

            
            	adds	$h0,$h0,$d0		// accumulate input

            
            	adcs	$h1,$h1,$d1

            
            	adc	$h2,$h2,$padbit


            	bl	poly1305_mult


            .Linit_neon:

            
            	ldr	w17,[$ctx,#48]		// first table element

            
            	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26

            
            	ubfx	x11,$h0,#26,#26

            
            	extr	x12,$h1,$h0,#52

            
            	and	x12,x12,#0x03ffffff

            
            	ubfx	x13,$h1,#14,#26

            
            	extr	x14,$h2,$h1,#40


            	cmp	w17,#-1			// is value impossible?

            
            	b.ne	.Leven_neon


            	fmov	${H0},x10

            
            	fmov	${H1},x11

            
            	fmov	${H2},x12

            
            	fmov	${H3},x13

            
            	fmov	${H4},x14


            	////////////////////////////////// initialize r^n table

            
            	mov	$h0,$r0			// r^1

            
            	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)

            
            	mov	$h1,$r1

            
            	mov	$h2,xzr

            
            	add	$ctx,$ctx,#48+12

            
            	bl	poly1305_splat


            	bl	poly1305_mult		// r^2

            
            	sub	$ctx,$ctx,#4

            
            	bl	poly1305_splat


            	bl	poly1305_mult		// r^3

            
            	sub	$ctx,$ctx,#4

            
            	bl	poly1305_splat


            	bl	poly1305_mult		// r^4

            
            	sub	$ctx,$ctx,#4

            
            	bl	poly1305_splat

            
            	sub	$ctx,$ctx,#48		// restore original $ctx

            
            	b	.Ldo_neon


            .align	4

            
            .Leven_neon:

            
            	fmov	${H0},x10

            
            	fmov	${H1},x11

            
            	fmov	${H2},x12

            
            	fmov	${H3},x13

            
            	fmov	${H4},x14


            .Ldo_neon:

            
            	ldp	x8,x12,[$inp,#32]	// inp[2:3]

            
            	subs	$len,$len,#64

            
            	ldp	x9,x13,[$inp,#48]

            
            	add	$in2,$inp,#96

            
            	adrp	$zeros,.Lzeros

            
            	add	$zeros,$zeros,#:lo12:.Lzeros


            	lsl	$padbit,$padbit,#24

            
            	add	x15,$ctx,#48


            #ifdef	__AARCH64EB__

            
            	rev	x8,x8

            
            	rev	x12,x12

            
            	rev	x9,x9

            
            	rev	x13,x13

            
            #endif

            
            	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26

            
            	and	x5,x9,#0x03ffffff

            
            	ubfx	x6,x8,#26,#26

            
            	ubfx	x7,x9,#26,#26

            
            	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32

            
            	extr	x8,x12,x8,#52

            
            	extr	x9,x13,x9,#52

            
            	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32

            
            	fmov	$IN23_0,x4

            
            	and	x8,x8,#0x03ffffff

            
            	and	x9,x9,#0x03ffffff

            
            	ubfx	x10,x12,#14,#26

            
            	ubfx	x11,x13,#14,#26

            
            	add	x12,$padbit,x12,lsr#40

            
            	add	x13,$padbit,x13,lsr#40

            
            	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32

            
            	fmov	$IN23_1,x6

            
            	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32

            
            	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32

            
            	fmov	$IN23_2,x8

            
            	fmov	$IN23_3,x10

            
            	fmov	$IN23_4,x12


            	ldp	x8,x12,[$inp],#16	// inp[0:1]

            
            	ldp	x9,x13,[$inp],#48


            	ld1	{$R0,$R1,$S1,$R2},[x15],#64

            
            	ld1	{$S2,$R3,$S3,$R4},[x15],#64

            
            	ld1	{$S4},[x15]


            #ifdef	__AARCH64EB__

            
            	rev	x8,x8

            
            	rev	x12,x12

            
            	rev	x9,x9

            
            	rev	x13,x13

            
            #endif

            
            	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26

            
            	and	x5,x9,#0x03ffffff

            
            	ubfx	x6,x8,#26,#26

            
            	ubfx	x7,x9,#26,#26

            
            	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32

            
            	extr	x8,x12,x8,#52

            
            	extr	x9,x13,x9,#52

            
            	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32

            
            	fmov	$IN01_0,x4

            
            	and	x8,x8,#0x03ffffff

            
            	and	x9,x9,#0x03ffffff

            
            	ubfx	x10,x12,#14,#26

            
            	ubfx	x11,x13,#14,#26

            
            	add	x12,$padbit,x12,lsr#40

            
            	add	x13,$padbit,x13,lsr#40

            
            	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32

            
            	fmov	$IN01_1,x6

            
            	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32

            
            	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32

            
            	movi	$MASK.2d,#-1

            
            	fmov	$IN01_2,x8

            
            	fmov	$IN01_3,x10

            
            	fmov	$IN01_4,x12

            
            	ushr	$MASK.2d,$MASK.2d,#38


            	b.ls	.Lskip_loop


            .align	4

            
            .Loop_neon:

            
            	////////////////////////////////////////////////////////////////

            
            	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2

            
            	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r

            
            	//   \___________________/

            
            	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2

            
            	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r

            
            	//   \___________________/ \____________________/

            
            	//

            
            	// Note that we start with inp[2:3]*r^2. This is because it

            
            	// doesn't depend on reduction in previous iteration.

            
            	////////////////////////////////////////////////////////////////

            
            	// d4 = h0*r4 + h1*r3   + h2*r2   + h3*r1   + h4*r0

            
            	// d3 = h0*r3 + h1*r2   + h2*r1   + h3*r0   + h4*5*r4

            
            	// d2 = h0*r2 + h1*r1   + h2*r0   + h3*5*r4 + h4*5*r3

            
            	// d1 = h0*r1 + h1*r0   + h2*5*r4 + h3*5*r3 + h4*5*r2

            
            	// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1


            	subs	$len,$len,#64

            
            	umull	$ACC4,$IN23_0,${R4}[2]

            
            	csel	$in2,$zeros,$in2,lo

            
            	umull	$ACC3,$IN23_0,${R3}[2]

            
            	umull	$ACC2,$IN23_0,${R2}[2]

            
            	 ldp	x8,x12,[$in2],#16	// inp[2:3] (or zero)

            
            	umull	$ACC1,$IN23_0,${R1}[2]

            
            	 ldp	x9,x13,[$in2],#48

            
            	umull	$ACC0,$IN23_0,${R0}[2]

            
            #ifdef	__AARCH64EB__

            
            	 rev	x8,x8

            
            	 rev	x12,x12

            
            	 rev	x9,x9

            
            	 rev	x13,x13

            
            #endif


            	umlal	$ACC4,$IN23_1,${R3}[2]

            
            	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26

            
            	umlal	$ACC3,$IN23_1,${R2}[2]

            
            	 and	x5,x9,#0x03ffffff

            
            	umlal	$ACC2,$IN23_1,${R1}[2]

            
            	 ubfx	x6,x8,#26,#26

            
            	umlal	$ACC1,$IN23_1,${R0}[2]

            
            	 ubfx	x7,x9,#26,#26

            
            	umlal	$ACC0,$IN23_1,${S4}[2]

            
            	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32


            	umlal	$ACC4,$IN23_2,${R2}[2]

            
            	 extr	x8,x12,x8,#52

            
            	umlal	$ACC3,$IN23_2,${R1}[2]

            
            	 extr	x9,x13,x9,#52

            
            	umlal	$ACC2,$IN23_2,${R0}[2]

            
            	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32

            
            	umlal	$ACC1,$IN23_2,${S4}[2]

            
            	 fmov	$IN23_0,x4

            
            	umlal	$ACC0,$IN23_2,${S3}[2]

            
            	 and	x8,x8,#0x03ffffff


            	umlal	$ACC4,$IN23_3,${R1}[2]

            
            	 and	x9,x9,#0x03ffffff

            
            	umlal	$ACC3,$IN23_3,${R0}[2]

            
            	 ubfx	x10,x12,#14,#26

            
            	umlal	$ACC2,$IN23_3,${S4}[2]

            
            	 ubfx	x11,x13,#14,#26

            
            	umlal	$ACC1,$IN23_3,${S3}[2]

            
            	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32

            
            	umlal	$ACC0,$IN23_3,${S2}[2]

            
            	 fmov	$IN23_1,x6


            	add	$IN01_2,$IN01_2,$H2

            
            	 add	x12,$padbit,x12,lsr#40

            
            	umlal	$ACC4,$IN23_4,${R0}[2]

            
            	 add	x13,$padbit,x13,lsr#40

            
            	umlal	$ACC3,$IN23_4,${S4}[2]

            
            	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32

            
            	umlal	$ACC2,$IN23_4,${S3}[2]

            
            	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32

            
            	umlal	$ACC1,$IN23_4,${S2}[2]

            
            	 fmov	$IN23_2,x8

            
            	umlal	$ACC0,$IN23_4,${S1}[2]

            
            	 fmov	$IN23_3,x10


            	////////////////////////////////////////////////////////////////

            
            	// (hash+inp[0:1])*r^4 and accumulate


            	add	$IN01_0,$IN01_0,$H0

            
            	 fmov	$IN23_4,x12

            
            	umlal	$ACC3,$IN01_2,${R1}[0]

            
            	 ldp	x8,x12,[$inp],#16	// inp[0:1]

            
            	umlal	$ACC0,$IN01_2,${S3}[0]

            
            	 ldp	x9,x13,[$inp],#48

            
            	umlal	$ACC4,$IN01_2,${R2}[0]

            
            	umlal	$ACC1,$IN01_2,${S4}[0]

            
            	umlal	$ACC2,$IN01_2,${R0}[0]

            
            #ifdef	__AARCH64EB__

            
            	 rev	x8,x8

            
            	 rev	x12,x12

            
            	 rev	x9,x9

            
            	 rev	x13,x13

            
            #endif


            	add	$IN01_1,$IN01_1,$H1

            
            	umlal	$ACC3,$IN01_0,${R3}[0]

            
            	umlal	$ACC4,$IN01_0,${R4}[0]

            
            	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26

            
            	umlal	$ACC2,$IN01_0,${R2}[0]

            
            	 and	x5,x9,#0x03ffffff

            
            	umlal	$ACC0,$IN01_0,${R0}[0]

            
            	 ubfx	x6,x8,#26,#26

            
            	umlal	$ACC1,$IN01_0,${R1}[0]

            
            	 ubfx	x7,x9,#26,#26


            	add	$IN01_3,$IN01_3,$H3

            
            	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32

            
            	umlal	$ACC3,$IN01_1,${R2}[0]

            
            	 extr	x8,x12,x8,#52

            
            	umlal	$ACC4,$IN01_1,${R3}[0]

            
            	 extr	x9,x13,x9,#52

            
            	umlal	$ACC0,$IN01_1,${S4}[0]

            
            	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32

            
            	umlal	$ACC2,$IN01_1,${R1}[0]

            
            	 fmov	$IN01_0,x4

            
            	umlal	$ACC1,$IN01_1,${R0}[0]

            
            	 and	x8,x8,#0x03ffffff


            	add	$IN01_4,$IN01_4,$H4

            
            	 and	x9,x9,#0x03ffffff

            
            	umlal	$ACC3,$IN01_3,${R0}[0]

            
            	 ubfx	x10,x12,#14,#26

            
            	umlal	$ACC0,$IN01_3,${S2}[0]

            
            	 ubfx	x11,x13,#14,#26

            
            	umlal	$ACC4,$IN01_3,${R1}[0]

            
            	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32

            
            	umlal	$ACC1,$IN01_3,${S3}[0]

            
            	 fmov	$IN01_1,x6

            
            	umlal	$ACC2,$IN01_3,${S4}[0]

            
            	 add	x12,$padbit,x12,lsr#40


            	umlal	$ACC3,$IN01_4,${S4}[0]

            
            	 add	x13,$padbit,x13,lsr#40

            
            	umlal	$ACC0,$IN01_4,${S1}[0]

            
            	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32

            
            	umlal	$ACC4,$IN01_4,${R0}[0]

            
            	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32

            
            	umlal	$ACC1,$IN01_4,${S2}[0]

            
            	 fmov	$IN01_2,x8

            
            	umlal	$ACC2,$IN01_4,${S3}[0]

            
            	 fmov	$IN01_3,x10

            
            	 fmov	$IN01_4,x12


            	/////////////////////////////////////////////////////////////////

            
            	// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein

            
            	// and P. Schwabe

            
            	//

            
            	// [see discussion in poly1305-armv4 module]


            	ushr	$T0.2d,$ACC3,#26

            
            	xtn	$H3,$ACC3

            
            	 ushr	$T1.2d,$ACC0,#26

            
            	 and	$ACC0,$ACC0,$MASK.2d

            
            	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4

            
            	bic	$H3,#0xfc,lsl#24	// &=0x03ffffff

            
            	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1


            	ushr	$T0.2d,$ACC4,#26

            
            	xtn	$H4,$ACC4

            
            	 ushr	$T1.2d,$ACC1,#26

            
            	 xtn	$H1,$ACC1

            
            	bic	$H4,#0xfc,lsl#24

            
            	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2


            	add	$ACC0,$ACC0,$T0.2d

            
            	shl	$T0.2d,$T0.2d,#2

            
            	 shrn	$T1.2s,$ACC2,#26

            
            	 xtn	$H2,$ACC2

            
            	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0

            
            	 bic	$H1,#0xfc,lsl#24

            
            	 add	$H3,$H3,$T1.2s		// h2 -> h3

            
            	 bic	$H2,#0xfc,lsl#24


            	shrn	$T0.2s,$ACC0,#26

            
            	xtn	$H0,$ACC0

            
            	 ushr	$T1.2s,$H3,#26

            
            	 bic	$H3,#0xfc,lsl#24

            
            	 bic	$H0,#0xfc,lsl#24

            
            	add	$H1,$H1,$T0.2s		// h0 -> h1

            
            	 add	$H4,$H4,$T1.2s		// h3 -> h4


            	b.hi	.Loop_neon


            .Lskip_loop:

            
            	dup	$IN23_2,${IN23_2}[0]

            
            	add	$IN01_2,$IN01_2,$H2


            	////////////////////////////////////////////////////////////////

            
            	// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1


            	adds	$len,$len,#32

            
            	b.ne	.Long_tail


            	dup	$IN23_2,${IN01_2}[0]

            
            	add	$IN23_0,$IN01_0,$H0

            
            	add	$IN23_3,$IN01_3,$H3

            
            	add	$IN23_1,$IN01_1,$H1

            
            	add	$IN23_4,$IN01_4,$H4


            .Long_tail:

            
            	dup	$IN23_0,${IN23_0}[0]

            
            	umull2	$ACC0,$IN23_2,${S3}

            
            	umull2	$ACC3,$IN23_2,${R1}

            
            	umull2	$ACC4,$IN23_2,${R2}

            
            	umull2	$ACC2,$IN23_2,${R0}

            
            	umull2	$ACC1,$IN23_2,${S4}


            	dup	$IN23_1,${IN23_1}[0]

            
            	umlal2	$ACC0,$IN23_0,${R0}

            
            	umlal2	$ACC2,$IN23_0,${R2}

            
            	umlal2	$ACC3,$IN23_0,${R3}

            
            	umlal2	$ACC4,$IN23_0,${R4}

            
            	umlal2	$ACC1,$IN23_0,${R1}


            	dup	$IN23_3,${IN23_3}[0]

            
            	umlal2	$ACC0,$IN23_1,${S4}

            
            	umlal2	$ACC3,$IN23_1,${R2}

            
            	umlal2	$ACC2,$IN23_1,${R1}

            
            	umlal2	$ACC4,$IN23_1,${R3}

            
            	umlal2	$ACC1,$IN23_1,${R0}


            	dup	$IN23_4,${IN23_4}[0]

            
            	umlal2	$ACC3,$IN23_3,${R0}

            
            	umlal2	$ACC4,$IN23_3,${R1}

            
            	umlal2	$ACC0,$IN23_3,${S2}

            
            	umlal2	$ACC1,$IN23_3,${S3}

            
            	umlal2	$ACC2,$IN23_3,${S4}


            	umlal2	$ACC3,$IN23_4,${S4}

            
            	umlal2	$ACC0,$IN23_4,${S1}

            
            	umlal2	$ACC4,$IN23_4,${R0}

            
            	umlal2	$ACC1,$IN23_4,${S2}

            
            	umlal2	$ACC2,$IN23_4,${S3}


            	b.eq	.Lshort_tail


            	////////////////////////////////////////////////////////////////

            
            	// (hash+inp[0:1])*r^4:r^3 and accumulate


            	add	$IN01_0,$IN01_0,$H0

            
            	umlal	$ACC3,$IN01_2,${R1}

            
            	umlal	$ACC0,$IN01_2,${S3}

            
            	umlal	$ACC4,$IN01_2,${R2}

            
            	umlal	$ACC1,$IN01_2,${S4}

            
            	umlal	$ACC2,$IN01_2,${R0}


            	add	$IN01_1,$IN01_1,$H1

            
            	umlal	$ACC3,$IN01_0,${R3}

            
            	umlal	$ACC0,$IN01_0,${R0}

            
            	umlal	$ACC4,$IN01_0,${R4}

            
            	umlal	$ACC1,$IN01_0,${R1}

            
            	umlal	$ACC2,$IN01_0,${R2}


            	add	$IN01_3,$IN01_3,$H3

            
            	umlal	$ACC3,$IN01_1,${R2}

            
            	umlal	$ACC0,$IN01_1,${S4}

            
            	umlal	$ACC4,$IN01_1,${R3}

            
            	umlal	$ACC1,$IN01_1,${R0}

            
            	umlal	$ACC2,$IN01_1,${R1}


            	add	$IN01_4,$IN01_4,$H4

            
            	umlal	$ACC3,$IN01_3,${R0}

            
            	umlal	$ACC0,$IN01_3,${S2}

            
            	umlal	$ACC4,$IN01_3,${R1}

            
            	umlal	$ACC1,$IN01_3,${S3}

            
            	umlal	$ACC2,$IN01_3,${S4}


            	umlal	$ACC3,$IN01_4,${S4}

            
            	umlal	$ACC0,$IN01_4,${S1}

            
            	umlal	$ACC4,$IN01_4,${R0}

            
            	umlal	$ACC1,$IN01_4,${S2}

            
            	umlal	$ACC2,$IN01_4,${S3}


            .Lshort_tail:

            
            	////////////////////////////////////////////////////////////////

            
            	// horizontal add


            	addp	$ACC3,$ACC3,$ACC3

            
            	 ldp	d8,d9,[sp,#16]		// meet ABI requirements

            
            	addp	$ACC0,$ACC0,$ACC0

            
            	 ldp	d10,d11,[sp,#32]

            
            	addp	$ACC4,$ACC4,$ACC4

            
            	 ldp	d12,d13,[sp,#48]

            
            	addp	$ACC1,$ACC1,$ACC1

            
            	 ldp	d14,d15,[sp,#64]

            
            	addp	$ACC2,$ACC2,$ACC2

            
            	 ldr	x30,[sp,#8]


            	////////////////////////////////////////////////////////////////

            
            	// lazy reduction, but without narrowing


            	ushr	$T0.2d,$ACC3,#26

            
            	and	$ACC3,$ACC3,$MASK.2d

            
            	 ushr	$T1.2d,$ACC0,#26

            
            	 and	$ACC0,$ACC0,$MASK.2d


            	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4

            
            	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1


            	ushr	$T0.2d,$ACC4,#26

            
            	and	$ACC4,$ACC4,$MASK.2d

            
            	 ushr	$T1.2d,$ACC1,#26

            
            	 and	$ACC1,$ACC1,$MASK.2d

            
            	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2


            	add	$ACC0,$ACC0,$T0.2d

            
            	shl	$T0.2d,$T0.2d,#2

            
            	 ushr	$T1.2d,$ACC2,#26

            
            	 and	$ACC2,$ACC2,$MASK.2d

            
            	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0

            
            	 add	$ACC3,$ACC3,$T1.2d	// h2 -> h3


            	ushr	$T0.2d,$ACC0,#26

            
            	and	$ACC0,$ACC0,$MASK.2d

            
            	 ushr	$T1.2d,$ACC3,#26

            
            	 and	$ACC3,$ACC3,$MASK.2d

            
            	add	$ACC1,$ACC1,$T0.2d	// h0 -> h1

            
            	 add	$ACC4,$ACC4,$T1.2d	// h3 -> h4


            	////////////////////////////////////////////////////////////////

            
            	// write the result, can be partially reduced


            	st4	{$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16

            
            	mov	x4,#1

            
            	st1	{$ACC4}[0],[$ctx]

            
            	str	x4,[$ctx,#8]		// set is_base2_26


            	ldr	x29,[sp],#80

            
            	 .inst	0xd50323bf		// autiasp

            
            	ret

            
            .size	poly1305_blocks_neon,.-poly1305_blocks_neon


            .pushsection .rodata

            
            .align	5

            
            .Lzeros:

            
            .long	0,0,0,0,0,0,0,0

            
            .asciz	"Poly1305 for ARMv8, CRYPTOGAMS by \@dot-asm"

            
            .popsection


            .align	2

            
            #if !defined(__KERNEL__) && !defined(_WIN64)

            
            .comm	OPENSSL_armcap_P,4,4

            
            .hidden	OPENSSL_armcap_P

            
            #endif

            
            ___


            foreach (split("\n",$code)) {

            
            	s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/			or

            
            	s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/	or

            
            	(m/\bdup\b/ and (s/\.[24]s/.2d/g or 1))			or

            
            	(m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1))	or

            
            	(m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1))		or

            
            	(m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1))		or

            
            	(m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));


            	s/\.[124]([sd])\[/.$1\[/;

            
            	s/w#x([0-9]+)/w$1/g;


            	print $_,"\n";

            
            }

            
            close STDOUT;
	#!/usr/bin/env perl
	# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause
	#
	# ====================================================================
	# Written by Andy Polyakov, @dot-asm, initially for the OpenSSL
	# project.
	# ====================================================================
	#
	# This module implements Poly1305 hash for ARMv8.
	#
	# June 2015
	#
	# Numbers are cycles per processed byte with poly1305_blocks alone.
	#
	# IALU/gcc-4.9 NEON
	#
	# Apple A7 1.86/+5% 0.72
	# Cortex-A53 2.69/+58% 1.47
	# Cortex-A57 2.70/+7% 1.14
	# Denver 1.64/+50% 1.18(*)
	# X-Gene 2.13/+68% 2.27
	# Mongoose 1.77/+75% 1.12
	# Kryo 2.70/+55% 1.13
	# ThunderX2 1.17/+95% 1.36
	#
	# (*) estimate based on resources availability is less than 1.0,
	# i.e. measured result is worse than expected, presumably binary
	# translator is not almighty;

	$flavour=shift;
	$output=shift;

	if ($flavour && $flavour ne "void") {
	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
	( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
	( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
	die "can't locate arm-xlate.pl";

	open STDOUT,"\| \"$^X\" $xlate $flavour $output";
	} else {
	open STDOUT,">$output";
	}

	my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
	my ($mac,$nonce)=($inp,$len);

	my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));

	$code.=<<___;
	#ifndef __KERNEL__
	# include "arm_arch.h"
	.extern OPENSSL_armcap_P
	#else
	# define poly1305_init poly1305_block_init
	# define poly1305_blocks poly1305_blocks_arm64
	#endif

	.text

	// forward "declarations" are required for Apple
	.globl poly1305_blocks
	.globl poly1305_emit

	.globl poly1305_init
	.type poly1305_init,%function
	.align 5
	poly1305_init:
	cmp $inp,xzr
	stp xzr,xzr,[$ctx] // zero hash value
	stp xzr,xzr,[$ctx,#16] // [along with is_base2_26]

	csel x0,xzr,x0,eq
	b.eq .Lno_key

	#ifndef __KERNEL__
	adrp x17,OPENSSL_armcap_P
	ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
	#endif

	ldp $r0,$r1,[$inp] // load key
	mov $s1,#0xfffffffc0fffffff
	movk $s1,#0x0fff,lsl#48
	#ifdef __AARCH64EB__
	rev $r0,$r0 // flip bytes
	rev $r1,$r1
	#endif
	and $r0,$r0,$s1 // &=0ffffffc0fffffff
	and $s1,$s1,#-4
	and $r1,$r1,$s1 // &=0ffffffc0ffffffc
	mov w#$s1,#-1
	stp $r0,$r1,[$ctx,#32] // save key value
	str w#$s1,[$ctx,#48] // impossible key power value

	#ifndef __KERNEL__
	tst w17,#ARMV7_NEON

	adr $d0,.Lpoly1305_blocks
	adr $r0,.Lpoly1305_blocks_neon
	adr $d1,.Lpoly1305_emit

	csel $d0,$d0,$r0,eq

	# ifdef __ILP32__
	stp w#$d0,w#$d1,[$len]
	# else
	stp $d0,$d1,[$len]
	# endif
	#endif
	mov x0,#1
	.Lno_key:
	ret
	.size poly1305_init,.-poly1305_init

	.type poly1305_blocks,%function
	.align 5
	poly1305_blocks:
	.Lpoly1305_blocks:
	ands $len,$len,#-16
	b.eq .Lno_data

	ldp $h0,$h1,[$ctx] // load hash value
	ldp $h2,x17,[$ctx,#16] // [along with is_base2_26]
	ldp $r0,$r1,[$ctx,#32] // load key value

	#ifdef __AARCH64EB__
	lsr $d0,$h0,#32
	mov w#$d1,w#$h0
	lsr $d2,$h1,#32
	mov w15,w#$h1
	lsr x16,$h2,#32
	#else
	mov w#$d0,w#$h0
	lsr $d1,$h0,#32
	mov w#$d2,w#$h1
	lsr x15,$h1,#32
	mov w16,w#$h2
	#endif

	add $d0,$d0,$d1,lsl#26 // base 2^26 -> base 2^64
	lsr $d1,$d2,#12
	adds $d0,$d0,$d2,lsl#52
	add $d1,$d1,x15,lsl#14
	adc $d1,$d1,xzr
	lsr $d2,x16,#24
	adds $d1,$d1,x16,lsl#40
	adc $d2,$d2,xzr

	cmp x17,#0 // is_base2_26?
	add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
	csel $h0,$h0,$d0,eq // choose between radixes
	csel $h1,$h1,$d1,eq
	csel $h2,$h2,$d2,eq

	.Loop:
	ldp $t0,$t1,[$inp],#16 // load input
	sub $len,$len,#16
	#ifdef __AARCH64EB__
	rev $t0,$t0
	rev $t1,$t1
	#endif
	adds $h0,$h0,$t0 // accumulate input
	adcs $h1,$h1,$t1

	mul $d0,$h0,$r0 // h0*r0
	adc $h2,$h2,$padbit
	umulh $d1,$h0,$r0

	mul $t0,$h1,$s1 // h15r1
	umulh $t1,$h1,$s1

	adds $d0,$d0,$t0
	mul $t0,$h0,$r1 // h0*r1
	adc $d1,$d1,$t1
	umulh $d2,$h0,$r1

	adds $d1,$d1,$t0
	mul $t0,$h1,$r0 // h1*r0
	adc $d2,$d2,xzr
	umulh $t1,$h1,$r0

	adds $d1,$d1,$t0
	mul $t0,$h2,$s1 // h25r1
	adc $d2,$d2,$t1
	mul $t1,$h2,$r0 // h2*r0

	adds $d1,$d1,$t0
	adc $d2,$d2,$t1

	and $t0,$d2,#-4 // final reduction
	and $h2,$d2,#3
	add $t0,$t0,$d2,lsr#2
	adds $h0,$d0,$t0
	adcs $h1,$d1,xzr
	adc $h2,$h2,xzr

	cbnz $len,.Loop

	stp $h0,$h1,[$ctx] // store hash value
	stp $h2,xzr,[$ctx,#16] // [and clear is_base2_26]

	.Lno_data:
	ret
	.size poly1305_blocks,.-poly1305_blocks

	.type poly1305_emit,%function
	.align 5
	poly1305_emit:
	.Lpoly1305_emit:
	ldp $h0,$h1,[$ctx] // load hash base 2^64
	ldp $h2,$r0,[$ctx,#16] // [along with is_base2_26]
	ldp $t0,$t1,[$nonce] // load nonce

	#ifdef __AARCH64EB__
	lsr $d0,$h0,#32
	mov w#$d1,w#$h0
	lsr $d2,$h1,#32
	mov w15,w#$h1
	lsr x16,$h2,#32
	#else
	mov w#$d0,w#$h0
	lsr $d1,$h0,#32
	mov w#$d2,w#$h1
	lsr x15,$h1,#32
	mov w16,w#$h2
	#endif

	add $d0,$d0,$d1,lsl#26 // base 2^26 -> base 2^64
	lsr $d1,$d2,#12
	adds $d0,$d0,$d2,lsl#52
	add $d1,$d1,x15,lsl#14
	adc $d1,$d1,xzr
	lsr $d2,x16,#24
	adds $d1,$d1,x16,lsl#40
	adc $d2,$d2,xzr

	cmp $r0,#0 // is_base2_26?
	csel $h0,$h0,$d0,eq // choose between radixes
	csel $h1,$h1,$d1,eq
	csel $h2,$h2,$d2,eq

	adds $d0,$h0,#5 // compare to modulus
	adcs $d1,$h1,xzr
	adc $d2,$h2,xzr

	tst $d2,#-4 // see if it's carried/borrowed

	csel $h0,$h0,$d0,eq
	csel $h1,$h1,$d1,eq

	#ifdef __AARCH64EB__
	ror $t0,$t0,#32 // flip nonce words
	ror $t1,$t1,#32
	#endif
	adds $h0,$h0,$t0 // accumulate nonce
	adc $h1,$h1,$t1
	#ifdef __AARCH64EB__
	rev $h0,$h0 // flip output bytes
	rev $h1,$h1
	#endif
	stp $h0,$h1,[$mac] // write result

	ret
	.size poly1305_emit,.-poly1305_emit
	___
	my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
	my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
	my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
	my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
	my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
	my ($T0,$T1,$MASK) = map("v$_",(29..31));

	my ($in2,$zeros)=("x16","x17");
	my $is_base2_26 = $zeros; # borrow

	$code.=<<___;
	.type poly1305_mult,%function
	.align 5
	poly1305_mult:
	mul $d0,$h0,$r0 // h0*r0
	umulh $d1,$h0,$r0

	mul $t0,$h1,$s1 // h15r1
	umulh $t1,$h1,$s1

	adds $d0,$d0,$t0
	mul $t0,$h0,$r1 // h0*r1
	adc $d1,$d1,$t1
	umulh $d2,$h0,$r1

	adds $d1,$d1,$t0
	mul $t0,$h1,$r0 // h1*r0
	adc $d2,$d2,xzr
	umulh $t1,$h1,$r0

	adds $d1,$d1,$t0
	mul $t0,$h2,$s1 // h25r1
	adc $d2,$d2,$t1
	mul $t1,$h2,$r0 // h2*r0

	adds $d1,$d1,$t0
	adc $d2,$d2,$t1

	and $t0,$d2,#-4 // final reduction
	and $h2,$d2,#3
	add $t0,$t0,$d2,lsr#2
	adds $h0,$d0,$t0
	adcs $h1,$d1,xzr
	adc $h2,$h2,xzr

	ret
	.size poly1305_mult,.-poly1305_mult

	.type poly1305_splat,%function
	.align 4
	poly1305_splat:
	and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26
	ubfx x13,$h0,#26,#26
	extr x14,$h1,$h0,#52
	and x14,x14,#0x03ffffff
	ubfx x15,$h1,#14,#26
	extr x16,$h2,$h1,#40

	str w12,[$ctx,#16*0] // r0
	add w12,w13,w13,lsl#2 // r1*5
	str w13,[$ctx,#16*1] // r1
	add w13,w14,w14,lsl#2 // r2*5
	str w12,[$ctx,#16*2] // s1
	str w14,[$ctx,#16*3] // r2
	add w14,w15,w15,lsl#2 // r3*5
	str w13,[$ctx,#16*4] // s2
	str w15,[$ctx,#16*5] // r3
	add w15,w16,w16,lsl#2 // r4*5
	str w14,[$ctx,#16*6] // s3
	str w16,[$ctx,#16*7] // r4
	str w15,[$ctx,#16*8] // s4

	ret
	.size poly1305_splat,.-poly1305_splat

	#ifdef __KERNEL__
	.globl poly1305_blocks_neon
	#endif
	.type poly1305_blocks_neon,%function
	.align 5
	poly1305_blocks_neon:
	.Lpoly1305_blocks_neon:
	ldr $is_base2_26,[$ctx,#24]
	cmp $len,#128
	b.lo .Lpoly1305_blocks

	.inst 0xd503233f // paciasp
	stp x29,x30,[sp,#-80]!
	add x29,sp,#0

	stp d8,d9,[sp,#16] // meet ABI requirements
	stp d10,d11,[sp,#32]
	stp d12,d13,[sp,#48]
	stp d14,d15,[sp,#64]

	cbz $is_base2_26,.Lbase2_64_neon

	ldp w10,w11,[$ctx] // load hash value base 2^26
	ldp w12,w13,[$ctx,#8]
	ldr w14,[$ctx,#16]

	tst $len,#31
	b.eq .Leven_neon

	ldp $r0,$r1,[$ctx,#32] // load key value

	add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
	lsr $h1,x12,#12
	adds $h0,$h0,x12,lsl#52
	add $h1,$h1,x13,lsl#14
	adc $h1,$h1,xzr
	lsr $h2,x14,#24
	adds $h1,$h1,x14,lsl#40
	adc $h2,$h2,xzr // preserve carry into top limb

	ldp $d0,$d1,[$inp],#16 // load input
	sub $len,$len,#16
	add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)

	#ifdef __AARCH64EB__
	rev $d0,$d0
	rev $d1,$d1
	#endif
	adds $h0,$h0,$d0 // accumulate input
	adcs $h1,$h1,$d1
	adc $h2,$h2,$padbit

	bl poly1305_mult

	and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
	ubfx x11,$h0,#26,#26
	extr x12,$h1,$h0,#52
	and x12,x12,#0x03ffffff
	ubfx x13,$h1,#14,#26
	extr x14,$h2,$h1,#40

	b .Leven_neon

	.align 4
	.Lbase2_64_neon:
	ldp $r0,$r1,[$ctx,#32] // load key value

	ldp $h0,$h1,[$ctx] // load hash value base 2^64
	ldr $h2,[$ctx,#16]

	tst $len,#31
	b.eq .Linit_neon

	ldp $d0,$d1,[$inp],#16 // load input
	sub $len,$len,#16
	add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
	#ifdef __AARCH64EB__
	rev $d0,$d0
	rev $d1,$d1
	#endif
	adds $h0,$h0,$d0 // accumulate input
	adcs $h1,$h1,$d1
	adc $h2,$h2,$padbit

	bl poly1305_mult

	.Linit_neon:
	ldr w17,[$ctx,#48] // first table element
	and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
	ubfx x11,$h0,#26,#26
	extr x12,$h1,$h0,#52
	and x12,x12,#0x03ffffff
	ubfx x13,$h1,#14,#26
	extr x14,$h2,$h1,#40

	cmp w17,#-1 // is value impossible?
	b.ne .Leven_neon

	fmov ${H0},x10
	fmov ${H1},x11
	fmov ${H2},x12
	fmov ${H3},x13
	fmov ${H4},x14

	////////////////////////////////// initialize r^n table
	mov $h0,$r0 // r^1
	add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
	mov $h1,$r1
	mov $h2,xzr
	add $ctx,$ctx,#48+12
	bl poly1305_splat

	bl poly1305_mult // r^2
	sub $ctx,$ctx,#4
	bl poly1305_splat

	bl poly1305_mult // r^3
	sub $ctx,$ctx,#4
	bl poly1305_splat

	bl poly1305_mult // r^4
	sub $ctx,$ctx,#4
	bl poly1305_splat
	sub $ctx,$ctx,#48 // restore original $ctx
	b .Ldo_neon

	.align 4
	.Leven_neon:
	fmov ${H0},x10
	fmov ${H1},x11
	fmov ${H2},x12
	fmov ${H3},x13
	fmov ${H4},x14

	.Ldo_neon:
	ldp x8,x12,[$inp,#32] // inp[2:3]
	subs $len,$len,#64
	ldp x9,x13,[$inp,#48]
	add $in2,$inp,#96
	adrp $zeros,.Lzeros
	add $zeros,$zeros,#:lo12:.Lzeros

	lsl $padbit,$padbit,#24
	add x15,$ctx,#48

	#ifdef __AARCH64EB__
	rev x8,x8
	rev x12,x12
	rev x9,x9
	rev x13,x13
	#endif
	and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
	and x5,x9,#0x03ffffff
	ubfx x6,x8,#26,#26
	ubfx x7,x9,#26,#26
	add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
	extr x8,x12,x8,#52
	extr x9,x13,x9,#52
	add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
	fmov $IN23_0,x4
	and x8,x8,#0x03ffffff
	and x9,x9,#0x03ffffff
	ubfx x10,x12,#14,#26
	ubfx x11,x13,#14,#26
	add x12,$padbit,x12,lsr#40
	add x13,$padbit,x13,lsr#40
	add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
	fmov $IN23_1,x6
	add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
	add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
	fmov $IN23_2,x8
	fmov $IN23_3,x10
	fmov $IN23_4,x12

	ldp x8,x12,[$inp],#16 // inp[0:1]
	ldp x9,x13,[$inp],#48

	ld1 {$R0,$R1,$S1,$R2},[x15],#64
	ld1 {$S2,$R3,$S3,$R4},[x15],#64
	ld1 {$S4},[x15]

	#ifdef __AARCH64EB__
	rev x8,x8
	rev x12,x12
	rev x9,x9
	rev x13,x13
	#endif
	and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
	and x5,x9,#0x03ffffff
	ubfx x6,x8,#26,#26
	ubfx x7,x9,#26,#26
	add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
	extr x8,x12,x8,#52
	extr x9,x13,x9,#52
	add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
	fmov $IN01_0,x4
	and x8,x8,#0x03ffffff
	and x9,x9,#0x03ffffff
	ubfx x10,x12,#14,#26
	ubfx x11,x13,#14,#26
	add x12,$padbit,x12,lsr#40
	add x13,$padbit,x13,lsr#40
	add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
	fmov $IN01_1,x6
	add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
	add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
	movi $MASK.2d,#-1
	fmov $IN01_2,x8
	fmov $IN01_3,x10
	fmov $IN01_4,x12
	ushr $MASK.2d,$MASK.2d,#38

	b.ls .Lskip_loop

	.align 4
	.Loop_neon:
	////////////////////////////////////////////////////////////////
	// ((inp[0]r^4+inp[2]r^2+inp[4])r^4+inp[6]r^2
	// ((inp[1]r^4+inp[3]r^2+inp[5])r^3+inp[7]r
	// \___________________/
	// ((inp[0]r^4+inp[2]r^2+inp[4])r^4+inp[6]r^2+inp[8])*r^2
	// ((inp[1]r^4+inp[3]r^2+inp[5])r^4+inp[7]r^2+inp[9])*r
	// \___________________/ \____________________/
	//
	// Note that we start with inp[2:3]*r^2. This is because it
	// doesn't depend on reduction in previous iteration.
	////////////////////////////////////////////////////////////////
	// d4 = h0r4 + h1r3 + h2r2 + h3r1 + h4*r0
	// d3 = h0r3 + h1r2 + h2r1 + h3r0 + h45r4
	// d2 = h0r2 + h1r1 + h2r0 + h35r4 + h45*r3
	// d1 = h0r1 + h1r0 + h25r4 + h35r3 + h45r2
	// d0 = h0r0 + h15r4 + h25r3 + h35r2 + h45*r1

	subs $len,$len,#64
	umull $ACC4,$IN23_0,${R4}[2]
	csel $in2,$zeros,$in2,lo
	umull $ACC3,$IN23_0,${R3}[2]
	umull $ACC2,$IN23_0,${R2}[2]
	ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
	umull $ACC1,$IN23_0,${R1}[2]
	ldp x9,x13,[$in2],#48
	umull $ACC0,$IN23_0,${R0}[2]
	#ifdef __AARCH64EB__
	rev x8,x8
	rev x12,x12
	rev x9,x9
	rev x13,x13
	#endif

	umlal $ACC4,$IN23_1,${R3}[2]
	and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
	umlal $ACC3,$IN23_1,${R2}[2]
	and x5,x9,#0x03ffffff
	umlal $ACC2,$IN23_1,${R1}[2]
	ubfx x6,x8,#26,#26
	umlal $ACC1,$IN23_1,${R0}[2]
	ubfx x7,x9,#26,#26
	umlal $ACC0,$IN23_1,${S4}[2]
	add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32

	umlal $ACC4,$IN23_2,${R2}[2]
	extr x8,x12,x8,#52
	umlal $ACC3,$IN23_2,${R1}[2]
	extr x9,x13,x9,#52
	umlal $ACC2,$IN23_2,${R0}[2]
	add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
	umlal $ACC1,$IN23_2,${S4}[2]
	fmov $IN23_0,x4
	umlal $ACC0,$IN23_2,${S3}[2]
	and x8,x8,#0x03ffffff

	umlal $ACC4,$IN23_3,${R1}[2]
	and x9,x9,#0x03ffffff
	umlal $ACC3,$IN23_3,${R0}[2]
	ubfx x10,x12,#14,#26
	umlal $ACC2,$IN23_3,${S4}[2]
	ubfx x11,x13,#14,#26
	umlal $ACC1,$IN23_3,${S3}[2]
	add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
	umlal $ACC0,$IN23_3,${S2}[2]
	fmov $IN23_1,x6

	add $IN01_2,$IN01_2,$H2
	add x12,$padbit,x12,lsr#40
	umlal $ACC4,$IN23_4,${R0}[2]
	add x13,$padbit,x13,lsr#40
	umlal $ACC3,$IN23_4,${S4}[2]
	add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
	umlal $ACC2,$IN23_4,${S3}[2]
	add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
	umlal $ACC1,$IN23_4,${S2}[2]
	fmov $IN23_2,x8
	umlal $ACC0,$IN23_4,${S1}[2]
	fmov $IN23_3,x10

	////////////////////////////////////////////////////////////////
	// (hash+inp[0:1])*r^4 and accumulate

	add $IN01_0,$IN01_0,$H0
	fmov $IN23_4,x12
	umlal $ACC3,$IN01_2,${R1}[0]
	ldp x8,x12,[$inp],#16 // inp[0:1]
	umlal $ACC0,$IN01_2,${S3}[0]
	ldp x9,x13,[$inp],#48
	umlal $ACC4,$IN01_2,${R2}[0]
	umlal $ACC1,$IN01_2,${S4}[0]
	umlal $ACC2,$IN01_2,${R0}[0]
	#ifdef __AARCH64EB__
	rev x8,x8
	rev x12,x12
	rev x9,x9
	rev x13,x13
	#endif

	add $IN01_1,$IN01_1,$H1
	umlal $ACC3,$IN01_0,${R3}[0]
	umlal $ACC4,$IN01_0,${R4}[0]
	and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
	umlal $ACC2,$IN01_0,${R2}[0]
	and x5,x9,#0x03ffffff
	umlal $ACC0,$IN01_0,${R0}[0]
	ubfx x6,x8,#26,#26
	umlal $ACC1,$IN01_0,${R1}[0]
	ubfx x7,x9,#26,#26

	add $IN01_3,$IN01_3,$H3
	add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
	umlal $ACC3,$IN01_1,${R2}[0]
	extr x8,x12,x8,#52
	umlal $ACC4,$IN01_1,${R3}[0]
	extr x9,x13,x9,#52
	umlal $ACC0,$IN01_1,${S4}[0]
	add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
	umlal $ACC2,$IN01_1,${R1}[0]
	fmov $IN01_0,x4
	umlal $ACC1,$IN01_1,${R0}[0]
	and x8,x8,#0x03ffffff

	add $IN01_4,$IN01_4,$H4
	and x9,x9,#0x03ffffff
	umlal $ACC3,$IN01_3,${R0}[0]
	ubfx x10,x12,#14,#26
	umlal $ACC0,$IN01_3,${S2}[0]
	ubfx x11,x13,#14,#26
	umlal $ACC4,$IN01_3,${R1}[0]
	add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
	umlal $ACC1,$IN01_3,${S3}[0]
	fmov $IN01_1,x6
	umlal $ACC2,$IN01_3,${S4}[0]
	add x12,$padbit,x12,lsr#40

	umlal $ACC3,$IN01_4,${S4}[0]
	add x13,$padbit,x13,lsr#40
	umlal $ACC0,$IN01_4,${S1}[0]
	add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
	umlal $ACC4,$IN01_4,${R0}[0]
	add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
	umlal $ACC1,$IN01_4,${S2}[0]
	fmov $IN01_2,x8
	umlal $ACC2,$IN01_4,${S3}[0]
	fmov $IN01_3,x10
	fmov $IN01_4,x12

	/////////////////////////////////////////////////////////////////
	// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
	// and P. Schwabe
	//
	// [see discussion in poly1305-armv4 module]

	ushr $T0.2d,$ACC3,#26
	xtn $H3,$ACC3
	ushr $T1.2d,$ACC0,#26
	and $ACC0,$ACC0,$MASK.2d
	add $ACC4,$ACC4,$T0.2d // h3 -> h4
	bic $H3,#0xfc,lsl#24 // &=0x03ffffff
	add $ACC1,$ACC1,$T1.2d // h0 -> h1

	ushr $T0.2d,$ACC4,#26
	xtn $H4,$ACC4
	ushr $T1.2d,$ACC1,#26
	xtn $H1,$ACC1
	bic $H4,#0xfc,lsl#24
	add $ACC2,$ACC2,$T1.2d // h1 -> h2

	add $ACC0,$ACC0,$T0.2d
	shl $T0.2d,$T0.2d,#2
	shrn $T1.2s,$ACC2,#26
	xtn $H2,$ACC2
	add $ACC0,$ACC0,$T0.2d // h4 -> h0
	bic $H1,#0xfc,lsl#24
	add $H3,$H3,$T1.2s // h2 -> h3
	bic $H2,#0xfc,lsl#24

	shrn $T0.2s,$ACC0,#26
	xtn $H0,$ACC0
	ushr $T1.2s,$H3,#26
	bic $H3,#0xfc,lsl#24
	bic $H0,#0xfc,lsl#24
	add $H1,$H1,$T0.2s // h0 -> h1
	add $H4,$H4,$T1.2s // h3 -> h4

	b.hi .Loop_neon

	.Lskip_loop:
	dup $IN23_2,${IN23_2}[0]
	add $IN01_2,$IN01_2,$H2

	////////////////////////////////////////////////////////////////
	// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1

	adds $len,$len,#32
	b.ne .Long_tail

	dup $IN23_2,${IN01_2}[0]
	add $IN23_0,$IN01_0,$H0
	add $IN23_3,$IN01_3,$H3
	add $IN23_1,$IN01_1,$H1
	add $IN23_4,$IN01_4,$H4

	.Long_tail:
	dup $IN23_0,${IN23_0}[0]
	umull2 $ACC0,$IN23_2,${S3}
	umull2 $ACC3,$IN23_2,${R1}
	umull2 $ACC4,$IN23_2,${R2}
	umull2 $ACC2,$IN23_2,${R0}
	umull2 $ACC1,$IN23_2,${S4}

	dup $IN23_1,${IN23_1}[0]
	umlal2 $ACC0,$IN23_0,${R0}
	umlal2 $ACC2,$IN23_0,${R2}
	umlal2 $ACC3,$IN23_0,${R3}
	umlal2 $ACC4,$IN23_0,${R4}
	umlal2 $ACC1,$IN23_0,${R1}

	dup $IN23_3,${IN23_3}[0]
	umlal2 $ACC0,$IN23_1,${S4}
	umlal2 $ACC3,$IN23_1,${R2}
	umlal2 $ACC2,$IN23_1,${R1}
	umlal2 $ACC4,$IN23_1,${R3}
	umlal2 $ACC1,$IN23_1,${R0}

	dup $IN23_4,${IN23_4}[0]
	umlal2 $ACC3,$IN23_3,${R0}
	umlal2 $ACC4,$IN23_3,${R1}
	umlal2 $ACC0,$IN23_3,${S2}
	umlal2 $ACC1,$IN23_3,${S3}
	umlal2 $ACC2,$IN23_3,${S4}

	umlal2 $ACC3,$IN23_4,${S4}
	umlal2 $ACC0,$IN23_4,${S1}
	umlal2 $ACC4,$IN23_4,${R0}
	umlal2 $ACC1,$IN23_4,${S2}
	umlal2 $ACC2,$IN23_4,${S3}

	b.eq .Lshort_tail

	////////////////////////////////////////////////////////////////
	// (hash+inp[0:1])*r^4:r^3 and accumulate

	add $IN01_0,$IN01_0,$H0
	umlal $ACC3,$IN01_2,${R1}
	umlal $ACC0,$IN01_2,${S3}
	umlal $ACC4,$IN01_2,${R2}
	umlal $ACC1,$IN01_2,${S4}
	umlal $ACC2,$IN01_2,${R0}

	add $IN01_1,$IN01_1,$H1
	umlal $ACC3,$IN01_0,${R3}
	umlal $ACC0,$IN01_0,${R0}
	umlal $ACC4,$IN01_0,${R4}
	umlal $ACC1,$IN01_0,${R1}
	umlal $ACC2,$IN01_0,${R2}

	add $IN01_3,$IN01_3,$H3
	umlal $ACC3,$IN01_1,${R2}
	umlal $ACC0,$IN01_1,${S4}
	umlal $ACC4,$IN01_1,${R3}
	umlal $ACC1,$IN01_1,${R0}
	umlal $ACC2,$IN01_1,${R1}

	add $IN01_4,$IN01_4,$H4
	umlal $ACC3,$IN01_3,${R0}
	umlal $ACC0,$IN01_3,${S2}
	umlal $ACC4,$IN01_3,${R1}
	umlal $ACC1,$IN01_3,${S3}
	umlal $ACC2,$IN01_3,${S4}

	umlal $ACC3,$IN01_4,${S4}
	umlal $ACC0,$IN01_4,${S1}
	umlal $ACC4,$IN01_4,${R0}
	umlal $ACC1,$IN01_4,${S2}
	umlal $ACC2,$IN01_4,${S3}

	.Lshort_tail:
	////////////////////////////////////////////////////////////////
	// horizontal add

	addp $ACC3,$ACC3,$ACC3
	ldp d8,d9,[sp,#16] // meet ABI requirements
	addp $ACC0,$ACC0,$ACC0
	ldp d10,d11,[sp,#32]
	addp $ACC4,$ACC4,$ACC4
	ldp d12,d13,[sp,#48]
	addp $ACC1,$ACC1,$ACC1
	ldp d14,d15,[sp,#64]
	addp $ACC2,$ACC2,$ACC2
	ldr x30,[sp,#8]

	////////////////////////////////////////////////////////////////
	// lazy reduction, but without narrowing

	ushr $T0.2d,$ACC3,#26
	and $ACC3,$ACC3,$MASK.2d
	ushr $T1.2d,$ACC0,#26
	and $ACC0,$ACC0,$MASK.2d

	add $ACC4,$ACC4,$T0.2d // h3 -> h4
	add $ACC1,$ACC1,$T1.2d // h0 -> h1

	ushr $T0.2d,$ACC4,#26
	and $ACC4,$ACC4,$MASK.2d
	ushr $T1.2d,$ACC1,#26
	and $ACC1,$ACC1,$MASK.2d
	add $ACC2,$ACC2,$T1.2d // h1 -> h2

	add $ACC0,$ACC0,$T0.2d
	shl $T0.2d,$T0.2d,#2
	ushr $T1.2d,$ACC2,#26
	and $ACC2,$ACC2,$MASK.2d
	add $ACC0,$ACC0,$T0.2d // h4 -> h0
	add $ACC3,$ACC3,$T1.2d // h2 -> h3

	ushr $T0.2d,$ACC0,#26
	and $ACC0,$ACC0,$MASK.2d
	ushr $T1.2d,$ACC3,#26
	and $ACC3,$ACC3,$MASK.2d
	add $ACC1,$ACC1,$T0.2d // h0 -> h1
	add $ACC4,$ACC4,$T1.2d // h3 -> h4

	////////////////////////////////////////////////////////////////
	// write the result, can be partially reduced

	st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
	mov x4,#1
	st1 {$ACC4}[0],[$ctx]
	str x4,[$ctx,#8] // set is_base2_26

	ldr x29,[sp],#80
	.inst 0xd50323bf // autiasp
	ret
	.size poly1305_blocks_neon,.-poly1305_blocks_neon

	.pushsection .rodata
	.align 5
	.Lzeros:
	.long 0,0,0,0,0,0,0,0
	.asciz "Poly1305 for ARMv8, CRYPTOGAMS by \@dot-asm"
	.popsection

	.align 2
	#if !defined(__KERNEL__) && !defined(_WIN64)
	.comm OPENSSL_armcap_P,4,4
	.hidden OPENSSL_armcap_P
	#endif
	___

	foreach (split("\n",$code)) {
	s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or
	s/\b(fmov\s+)v([0-9]+)[^,],\sx([0-9]+)/$1d$2,x$3/ or
	(m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or
	(m/\b(eor\|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or
	(m/\bum(ul\|la)l\b/ and (s/\.4s/.2s/g or 1)) or
	(m/\bum(ul\|la)l2\b/ and (s/\.2s/.4s/g or 1)) or
	(m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));

	s/\.[124]([sd])\[/.$1\[/;
	s/w#x([0-9]+)/w$1/g;

	print $_,"\n";
	}
	close STDOUT;