src/kernel/linux/v4.19/arch/arm64/crypto/aes-modes.S

/*
 * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
 *
 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

/* included by aes-ce.S and aes-neon.S */

	.text
	.align		4

aes_encrypt_block4x:
	encrypt_block4x	v0, v1, v2, v3, w22, x21, x8, w7
	ret
ENDPROC(aes_encrypt_block4x)

aes_decrypt_block4x:
	decrypt_block4x	v0, v1, v2, v3, w22, x21, x8, w7
	ret
ENDPROC(aes_decrypt_block4x)

	/*
	 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		   int blocks)
	 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		   int blocks)
	 */

AES_ENTRY(aes_ecb_encrypt)
	frame_push	5

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4

.Lecbencrestart:
	enc_prepare	w22, x21, x5

.LecbencloopNx:
	subs		w23, w23, #4
	bmi		.Lecbenc1x
	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 pt blocks */
	bl		aes_encrypt_block4x
	st1		{v0.16b-v3.16b}, [x19], #64
	cond_yield_neon	.Lecbencrestart
	b		.LecbencloopNx
.Lecbenc1x:
	adds		w23, w23, #4
	beq		.Lecbencout
.Lecbencloop:
	ld1		{v0.16b}, [x20], #16		/* get next pt block */
	encrypt_block	v0, w22, x21, x5, w6
	st1		{v0.16b}, [x19], #16
	subs		w23, w23, #1
	bne		.Lecbencloop
.Lecbencout:
	frame_pop
	ret
AES_ENDPROC(aes_ecb_encrypt)


AES_ENTRY(aes_ecb_decrypt)
	frame_push	5

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4

.Lecbdecrestart:
	dec_prepare	w22, x21, x5

.LecbdecloopNx:
	subs		w23, w23, #4
	bmi		.Lecbdec1x
	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 ct blocks */
	bl		aes_decrypt_block4x
	st1		{v0.16b-v3.16b}, [x19], #64
	cond_yield_neon	.Lecbdecrestart
	b		.LecbdecloopNx
.Lecbdec1x:
	adds		w23, w23, #4
	beq		.Lecbdecout
.Lecbdecloop:
	ld1		{v0.16b}, [x20], #16		/* get next ct block */
	decrypt_block	v0, w22, x21, x5, w6
	st1		{v0.16b}, [x19], #16
	subs		w23, w23, #1
	bne		.Lecbdecloop
.Lecbdecout:
	frame_pop
	ret
AES_ENDPROC(aes_ecb_decrypt)


	/*
	 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		   int blocks, u8 iv[])
	 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		   int blocks, u8 iv[])
	 */

AES_ENTRY(aes_cbc_encrypt)
	frame_push	6

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4
	mov		x24, x5

.Lcbcencrestart:
	ld1		{v4.16b}, [x24]			/* get iv */
	enc_prepare	w22, x21, x6

.Lcbcencloop4x:
	subs		w23, w23, #4
	bmi		.Lcbcenc1x
	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 pt blocks */
	eor		v0.16b, v0.16b, v4.16b		/* ..and xor with iv */
	encrypt_block	v0, w22, x21, x6, w7
	eor		v1.16b, v1.16b, v0.16b
	encrypt_block	v1, w22, x21, x6, w7
	eor		v2.16b, v2.16b, v1.16b
	encrypt_block	v2, w22, x21, x6, w7
	eor		v3.16b, v3.16b, v2.16b
	encrypt_block	v3, w22, x21, x6, w7
	st1		{v0.16b-v3.16b}, [x19], #64
	mov		v4.16b, v3.16b
	st1		{v4.16b}, [x24]			/* return iv */
	cond_yield_neon	.Lcbcencrestart
	b		.Lcbcencloop4x
.Lcbcenc1x:
	adds		w23, w23, #4
	beq		.Lcbcencout
.Lcbcencloop:
	ld1		{v0.16b}, [x20], #16		/* get next pt block */
	eor		v4.16b, v4.16b, v0.16b		/* ..and xor with iv */
	encrypt_block	v4, w22, x21, x6, w7
	st1		{v4.16b}, [x19], #16
	subs		w23, w23, #1
	bne		.Lcbcencloop
.Lcbcencout:
	st1		{v4.16b}, [x24]			/* return iv */
	frame_pop
	ret
AES_ENDPROC(aes_cbc_encrypt)


AES_ENTRY(aes_cbc_decrypt)
	frame_push	6

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4
	mov		x24, x5

.Lcbcdecrestart:
	ld1		{v7.16b}, [x24]			/* get iv */
	dec_prepare	w22, x21, x6

.LcbcdecloopNx:
	subs		w23, w23, #4
	bmi		.Lcbcdec1x
	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 ct blocks */
	mov		v4.16b, v0.16b
	mov		v5.16b, v1.16b
	mov		v6.16b, v2.16b
	bl		aes_decrypt_block4x
	sub		x20, x20, #16
	eor		v0.16b, v0.16b, v7.16b
	eor		v1.16b, v1.16b, v4.16b
	ld1		{v7.16b}, [x20], #16		/* reload 1 ct block */
	eor		v2.16b, v2.16b, v5.16b
	eor		v3.16b, v3.16b, v6.16b
	st1		{v0.16b-v3.16b}, [x19], #64
	st1		{v7.16b}, [x24]			/* return iv */
	cond_yield_neon	.Lcbcdecrestart
	b		.LcbcdecloopNx
.Lcbcdec1x:
	adds		w23, w23, #4
	beq		.Lcbcdecout
.Lcbcdecloop:
	ld1		{v1.16b}, [x20], #16		/* get next ct block */
	mov		v0.16b, v1.16b			/* ...and copy to v0 */
	decrypt_block	v0, w22, x21, x6, w7
	eor		v0.16b, v0.16b, v7.16b		/* xor with iv => pt */
	mov		v7.16b, v1.16b			/* ct is next iv */
	st1		{v0.16b}, [x19], #16
	subs		w23, w23, #1
	bne		.Lcbcdecloop
.Lcbcdecout:
	st1		{v7.16b}, [x24]			/* return iv */
	frame_pop
	ret
AES_ENDPROC(aes_cbc_decrypt)


	/*
	 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
	 *		   int blocks, u8 ctr[])
	 */

AES_ENTRY(aes_ctr_encrypt)
	frame_push	6

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4
	mov		x24, x5

.Lctrrestart:
	enc_prepare	w22, x21, x6
	ld1		{v4.16b}, [x24]

	umov		x6, v4.d[1]		/* keep swabbed ctr in reg */
	rev		x6, x6
.LctrloopNx:
	subs		w23, w23, #4
	bmi		.Lctr1x
	cmn		w6, #4			/* 32 bit overflow? */
	bcs		.Lctr1x
	ldr		q8, =0x30000000200000001	/* addends 1,2,3[,0] */
	dup		v7.4s, w6
	mov		v0.16b, v4.16b
	add		v7.4s, v7.4s, v8.4s
	mov		v1.16b, v4.16b
	rev32		v8.16b, v7.16b
	mov		v2.16b, v4.16b
	mov		v3.16b, v4.16b
	mov		v1.s[3], v8.s[0]
	mov		v2.s[3], v8.s[1]
	mov		v3.s[3], v8.s[2]
	ld1		{v5.16b-v7.16b}, [x20], #48	/* get 3 input blocks */
	bl		aes_encrypt_block4x
	eor		v0.16b, v5.16b, v0.16b
	ld1		{v5.16b}, [x20], #16		/* get 1 input block  */
	eor		v1.16b, v6.16b, v1.16b
	eor		v2.16b, v7.16b, v2.16b
	eor		v3.16b, v5.16b, v3.16b
	st1		{v0.16b-v3.16b}, [x19], #64
	add		x6, x6, #4
	rev		x7, x6
	ins		v4.d[1], x7
	cbz		w23, .Lctrout
	st1		{v4.16b}, [x24]		/* return next CTR value */
	cond_yield_neon	.Lctrrestart
	b		.LctrloopNx
.Lctr1x:
	adds		w23, w23, #4
	beq		.Lctrout
.Lctrloop:
	mov		v0.16b, v4.16b
	encrypt_block	v0, w22, x21, x8, w7

	adds		x6, x6, #1		/* increment BE ctr */
	rev		x7, x6
	ins		v4.d[1], x7
	bcs		.Lctrcarry		/* overflow? */

.Lctrcarrydone:
	subs		w23, w23, #1
	bmi		.Lctrtailblock		/* blocks <0 means tail block */
	ld1		{v3.16b}, [x20], #16
	eor		v3.16b, v0.16b, v3.16b
	st1		{v3.16b}, [x19], #16
	bne		.Lctrloop

.Lctrout:
	st1		{v4.16b}, [x24]		/* return next CTR value */
.Lctrret:
	frame_pop
	ret

.Lctrtailblock:
	st1		{v0.16b}, [x19]
	b		.Lctrret

.Lctrcarry:
	umov		x7, v4.d[0]		/* load upper word of ctr  */
	rev		x7, x7			/* ... to handle the carry */
	add		x7, x7, #1
	rev		x7, x7
	ins		v4.d[0], x7
	b		.Lctrcarrydone
AES_ENDPROC(aes_ctr_encrypt)
	.ltorg


	/*
	 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
	 *		   int blocks, u8 const rk2[], u8 iv[], int first)
	 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
	 *		   int blocks, u8 const rk2[], u8 iv[], int first)
	 */

	.macro		next_tweak, out, in, const, tmp
	sshr		\tmp\().2d,  \in\().2d,   #63
	and		\tmp\().16b, \tmp\().16b, \const\().16b
	add		\out\().2d,  \in\().2d,   \in\().2d
	ext		\tmp\().16b, \tmp\().16b, \tmp\().16b, #8
	eor		\out\().16b, \out\().16b, \tmp\().16b
	.endm

.Lxts_mul_x:
CPU_LE(	.quad		1, 0x87		)
CPU_BE(	.quad		0x87, 1		)

AES_ENTRY(aes_xts_encrypt)
	frame_push	6

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4
	mov		x24, x6

	ld1		{v4.16b}, [x24]
	cbz		w7, .Lxtsencnotfirst

	enc_prepare	w3, x5, x8
	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
	enc_switch_key	w3, x2, x8
	ldr		q7, .Lxts_mul_x
	b		.LxtsencNx

.Lxtsencrestart:
	ld1		{v4.16b}, [x24]
.Lxtsencnotfirst:
	enc_prepare	w22, x21, x8
.LxtsencloopNx:
	ldr		q7, .Lxts_mul_x
	next_tweak	v4, v4, v7, v8
.LxtsencNx:
	subs		w23, w23, #4
	bmi		.Lxtsenc1x
	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 pt blocks */
	next_tweak	v5, v4, v7, v8
	eor		v0.16b, v0.16b, v4.16b
	next_tweak	v6, v5, v7, v8
	eor		v1.16b, v1.16b, v5.16b
	eor		v2.16b, v2.16b, v6.16b
	next_tweak	v7, v6, v7, v8
	eor		v3.16b, v3.16b, v7.16b
	bl		aes_encrypt_block4x
	eor		v3.16b, v3.16b, v7.16b
	eor		v0.16b, v0.16b, v4.16b
	eor		v1.16b, v1.16b, v5.16b
	eor		v2.16b, v2.16b, v6.16b
	st1		{v0.16b-v3.16b}, [x19], #64
	mov		v4.16b, v7.16b
	cbz		w23, .Lxtsencout
	st1		{v4.16b}, [x24]
	cond_yield_neon	.Lxtsencrestart
	b		.LxtsencloopNx
.Lxtsenc1x:
	adds		w23, w23, #4
	beq		.Lxtsencout
.Lxtsencloop:
	ld1		{v1.16b}, [x20], #16
	eor		v0.16b, v1.16b, v4.16b
	encrypt_block	v0, w22, x21, x8, w7
	eor		v0.16b, v0.16b, v4.16b
	st1		{v0.16b}, [x19], #16
	subs		w23, w23, #1
	beq		.Lxtsencout
	next_tweak	v4, v4, v7, v8
	b		.Lxtsencloop
.Lxtsencout:
	st1		{v4.16b}, [x24]
	frame_pop
	ret
AES_ENDPROC(aes_xts_encrypt)


AES_ENTRY(aes_xts_decrypt)
	frame_push	6

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4
	mov		x24, x6

	ld1		{v4.16b}, [x24]
	cbz		w7, .Lxtsdecnotfirst

	enc_prepare	w3, x5, x8
	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
	dec_prepare	w3, x2, x8
	ldr		q7, .Lxts_mul_x
	b		.LxtsdecNx

.Lxtsdecrestart:
	ld1		{v4.16b}, [x24]
.Lxtsdecnotfirst:
	dec_prepare	w22, x21, x8
.LxtsdecloopNx:
	ldr		q7, .Lxts_mul_x
	next_tweak	v4, v4, v7, v8
.LxtsdecNx:
	subs		w23, w23, #4
	bmi		.Lxtsdec1x
	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 ct blocks */
	next_tweak	v5, v4, v7, v8
	eor		v0.16b, v0.16b, v4.16b
	next_tweak	v6, v5, v7, v8
	eor		v1.16b, v1.16b, v5.16b
	eor		v2.16b, v2.16b, v6.16b
	next_tweak	v7, v6, v7, v8
	eor		v3.16b, v3.16b, v7.16b
	bl		aes_decrypt_block4x
	eor		v3.16b, v3.16b, v7.16b
	eor		v0.16b, v0.16b, v4.16b
	eor		v1.16b, v1.16b, v5.16b
	eor		v2.16b, v2.16b, v6.16b
	st1		{v0.16b-v3.16b}, [x19], #64
	mov		v4.16b, v7.16b
	cbz		w23, .Lxtsdecout
	st1		{v4.16b}, [x24]
	cond_yield_neon	.Lxtsdecrestart
	b		.LxtsdecloopNx
.Lxtsdec1x:
	adds		w23, w23, #4
	beq		.Lxtsdecout
.Lxtsdecloop:
	ld1		{v1.16b}, [x20], #16
	eor		v0.16b, v1.16b, v4.16b
	decrypt_block	v0, w22, x21, x8, w7
	eor		v0.16b, v0.16b, v4.16b
	st1		{v0.16b}, [x19], #16
	subs		w23, w23, #1
	beq		.Lxtsdecout
	next_tweak	v4, v4, v7, v8
	b		.Lxtsdecloop
.Lxtsdecout:
	st1		{v4.16b}, [x24]
	frame_pop
	ret
AES_ENDPROC(aes_xts_decrypt)

	/*
	 * aes_mac_update(u8 const in[], u32 const rk[], int rounds,
	 *		  int blocks, u8 dg[], int enc_before, int enc_after)
	 */
AES_ENTRY(aes_mac_update)
	frame_push	6

	mov		x19, x0
	mov		x20, x1
	mov		x21, x2
	mov		x22, x3
	mov		x23, x4
	mov		x24, x6

	ld1		{v0.16b}, [x23]			/* get dg */
	enc_prepare	w2, x1, x7
	cbz		w5, .Lmacloop4x

	encrypt_block	v0, w2, x1, x7, w8

.Lmacloop4x:
	subs		w22, w22, #4
	bmi		.Lmac1x
	ld1		{v1.16b-v4.16b}, [x19], #64	/* get next pt block */
	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
	encrypt_block	v0, w21, x20, x7, w8
	eor		v0.16b, v0.16b, v2.16b
	encrypt_block	v0, w21, x20, x7, w8
	eor		v0.16b, v0.16b, v3.16b
	encrypt_block	v0, w21, x20, x7, w8
	eor		v0.16b, v0.16b, v4.16b
	cmp		w22, wzr
	csinv		x5, x24, xzr, eq
	cbz		w5, .Lmacout
	encrypt_block	v0, w21, x20, x7, w8
	st1		{v0.16b}, [x23]			/* return dg */
	cond_yield_neon	.Lmacrestart
	b		.Lmacloop4x
.Lmac1x:
	add		w22, w22, #4
.Lmacloop:
	cbz		w22, .Lmacout
	ld1		{v1.16b}, [x19], #16		/* get next pt block */
	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */

	subs		w22, w22, #1
	csinv		x5, x24, xzr, eq
	cbz		w5, .Lmacout

.Lmacenc:
	encrypt_block	v0, w21, x20, x7, w8
	b		.Lmacloop

.Lmacout:
	st1		{v0.16b}, [x23]			/* return dg */
	frame_pop
	ret

.Lmacrestart:
	ld1		{v0.16b}, [x23]			/* get dg */
	enc_prepare	w21, x20, x0
	b		.Lmacloop4x
AES_ENDPROC(aes_mac_update)