Gitiles
Code Review Sign In
192.168.1.100 / R306 / e08f1058ff217e55feb109a64cd5cac3a2da9a6d / . / ap / lib / libssl / openssl-1.1.1o / crypto / poly1305 / asm / poly1305-x86.pl
blob: 2ae16a230b661be6ac05ac1a961b4b00c2f23c4a [file] [log] [blame]
yuezonghe824eb0c2024-06-27 02:32:26 -0700[diff] [blame]1#! /usr/bin/env perl
2# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the OpenSSL license (the "License"). You may not use
5# this file except in compliance with the License. You can obtain a copy
6# in the file LICENSE in the source distribution or at
7# https://www.openssl.org/source/license.html
8
9#
10# ====================================================================
11# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12# project. The module is, however, dual licensed under OpenSSL and
13# CRYPTOGAMS licenses depending on where you obtain it. For further
14# details see http://www.openssl.org/~appro/cryptogams/.
15# ====================================================================
16#
17# This module implements Poly1305 hash for x86.
18#
19# April 2015
20#
21# Numbers are cycles per processed byte with poly1305_blocks alone,
22# measured with rdtsc at fixed clock frequency.
23#
24# IALU/gcc-3.4(*) SSE2(**) AVX2
25# Pentium 15.7/+80% -
26# PIII 6.21/+90% -
27# P4 19.8/+40% 3.24
28# Core 2 4.85/+90% 1.80
29# Westmere 4.58/+100% 1.43
30# Sandy Bridge 3.90/+100% 1.36
31# Haswell 3.88/+70% 1.18 0.72
32# Skylake 3.10/+60% 1.14 0.62
33# Silvermont 11.0/+40% 4.80
34# Goldmont 4.10/+200% 2.10
35# VIA Nano 6.71/+90% 2.47
36# Sledgehammer 3.51/+180% 4.27
37# Bulldozer 4.53/+140% 1.31
38#
39# (*) gcc 4.8 for some reason generated worse code;
40# (**) besides SSE2 there are floating-point and AVX options; FP
41# is deemed unnecessary, because pre-SSE2 processor are too
42# old to care about, while it's not the fastest option on
43# SSE2-capable ones; AVX is omitted, because it doesn't give
44# a lot of improvement, 5-10% depending on processor;
45
46$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
47push(@INC,"${dir}","${dir}../../perlasm");
48require "x86asm.pl";
49
50$output=pop;
51open STDOUT,">$output";
52
53&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
54
55$sse2=$avx=0;
56for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
57
58if ($sse2) {
59 &static_label("const_sse2");
60 &static_label("enter_blocks");
61 &static_label("enter_emit");
62 &external_label("OPENSSL_ia32cap_P");
63
64 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
65 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
66 $avx = ($1>=2.19) + ($1>=2.22);
67 }
68
69 if (!$avx && $ARGV[0] eq "win32n" &&
70 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
71 $avx = ($1>=2.09) + ($1>=2.10);
72 }
73
74 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|based on LLVM) ([0-9]+\.[0-9]+)/) {
75 $avx = ($2>=3.0) + ($2>3.0);
76 }
77}
78
79########################################################################
80# Layout of opaque area is following.
81#
82# unsigned __int32 h[5]; # current hash value base 2^32
83# unsigned __int32 pad; # is_base2_26 in vector context
84# unsigned __int32 r[4]; # key value base 2^32
85
86&align(64);
87&function_begin("poly1305_init");
88 &mov ("edi",&wparam(0)); # context
89 &mov ("esi",&wparam(1)); # key
90 &mov ("ebp",&wparam(2)); # function table
91
92 &xor ("eax","eax");
93 &mov (&DWP(4*0,"edi"),"eax"); # zero hash value
94 &mov (&DWP(4*1,"edi"),"eax");
95 &mov (&DWP(4*2,"edi"),"eax");
96 &mov (&DWP(4*3,"edi"),"eax");
97 &mov (&DWP(4*4,"edi"),"eax");
98 &mov (&DWP(4*5,"edi"),"eax"); # is_base2_26
99
100 &cmp ("esi",0);
101 &je (&label("nokey"));
102
103 if ($sse2) {
104 &call (&label("pic_point"));
105 &set_label("pic_point");
106 &blindpop("ebx");
107
108 &lea ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
109 &lea ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
110
111 &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
112 &mov ("ecx",&DWP(0,"edi"));
113 &and ("ecx",1<<26|1<<24);
114 &cmp ("ecx",1<<26|1<<24); # SSE2 and XMM?
115 &jne (&label("no_sse2"));
116
117 &lea ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
118 &lea ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
119
120 if ($avx>1) {
121 &mov ("ecx",&DWP(8,"edi"));
122 &test ("ecx",1<<5); # AVX2?
123 &jz (&label("no_sse2"));
124
125 &lea ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
126 }
127 &set_label("no_sse2");
128 &mov ("edi",&wparam(0)); # reload context
129 &mov (&DWP(0,"ebp"),"eax"); # fill function table
130 &mov (&DWP(4,"ebp"),"edx");
131 }
132
133 &mov ("eax",&DWP(4*0,"esi")); # load input key
134 &mov ("ebx",&DWP(4*1,"esi"));
135 &mov ("ecx",&DWP(4*2,"esi"));
136 &mov ("edx",&DWP(4*3,"esi"));
137 &and ("eax",0x0fffffff);
138 &and ("ebx",0x0ffffffc);
139 &and ("ecx",0x0ffffffc);
140 &and ("edx",0x0ffffffc);
141 &mov (&DWP(4*6,"edi"),"eax");
142 &mov (&DWP(4*7,"edi"),"ebx");
143 &mov (&DWP(4*8,"edi"),"ecx");
144 &mov (&DWP(4*9,"edi"),"edx");
145
146 &mov ("eax",$sse2);
147&set_label("nokey");
148&function_end("poly1305_init");
149
150($h0,$h1,$h2,$h3,$h4,
151 $d0,$d1,$d2,$d3,
152 $r0,$r1,$r2,$r3,
153 $s1,$s2,$s3)=map(4*$_,(0..15));
154
155&function_begin("poly1305_blocks");
156 &mov ("edi",&wparam(0)); # ctx
157 &mov ("esi",&wparam(1)); # inp
158 &mov ("ecx",&wparam(2)); # len
159&set_label("enter_blocks");
160 &and ("ecx",-15);
161 &jz (&label("nodata"));
162
163 &stack_push(16);
164 &mov ("eax",&DWP(4*6,"edi")); # r0
165 &mov ("ebx",&DWP(4*7,"edi")); # r1
166 &lea ("ebp",&DWP(0,"esi","ecx")); # end of input
167 &mov ("ecx",&DWP(4*8,"edi")); # r2
168 &mov ("edx",&DWP(4*9,"edi")); # r3
169
170 &mov (&wparam(2),"ebp");
171 &mov ("ebp","esi");
172
173 &mov (&DWP($r0,"esp"),"eax"); # r0
174 &mov ("eax","ebx");
175 &shr ("eax",2);
176 &mov (&DWP($r1,"esp"),"ebx"); # r1
177 &add ("eax","ebx"); # s1
178 &mov ("ebx","ecx");
179 &shr ("ebx",2);
180 &mov (&DWP($r2,"esp"),"ecx"); # r2
181 &add ("ebx","ecx"); # s2
182 &mov ("ecx","edx");
183 &shr ("ecx",2);
184 &mov (&DWP($r3,"esp"),"edx"); # r3
185 &add ("ecx","edx"); # s3
186 &mov (&DWP($s1,"esp"),"eax"); # s1
187 &mov (&DWP($s2,"esp"),"ebx"); # s2
188 &mov (&DWP($s3,"esp"),"ecx"); # s3
189
190 &mov ("eax",&DWP(4*0,"edi")); # load hash value
191 &mov ("ebx",&DWP(4*1,"edi"));
192 &mov ("ecx",&DWP(4*2,"edi"));
193 &mov ("esi",&DWP(4*3,"edi"));
194 &mov ("edi",&DWP(4*4,"edi"));
195 &jmp (&label("loop"));
196
197&set_label("loop",32);
198 &add ("eax",&DWP(4*0,"ebp")); # accumulate input
199 &adc ("ebx",&DWP(4*1,"ebp"));
200 &adc ("ecx",&DWP(4*2,"ebp"));
201 &adc ("esi",&DWP(4*3,"ebp"));
202 &lea ("ebp",&DWP(4*4,"ebp"));
203 &adc ("edi",&wparam(3)); # padbit
204
205 &mov (&DWP($h0,"esp"),"eax"); # put aside hash[+inp]
206 &mov (&DWP($h3,"esp"),"esi");
207
208 &mul (&DWP($r0,"esp")); # h0*r0
209 &mov (&DWP($h4,"esp"),"edi");
210 &mov ("edi","eax");
211 &mov ("eax","ebx"); # h1
212 &mov ("esi","edx");
213 &mul (&DWP($s3,"esp")); # h1*s3
214 &add ("edi","eax");
215 &mov ("eax","ecx"); # h2
216 &adc ("esi","edx");
217 &mul (&DWP($s2,"esp")); # h2*s2
218 &add ("edi","eax");
219 &mov ("eax",&DWP($h3,"esp"));
220 &adc ("esi","edx");
221 &mul (&DWP($s1,"esp")); # h3*s1
222 &add ("edi","eax");
223 &mov ("eax",&DWP($h0,"esp"));
224 &adc ("esi","edx");
225
226 &mul (&DWP($r1,"esp")); # h0*r1
227 &mov (&DWP($d0,"esp"),"edi");
228 &xor ("edi","edi");
229 &add ("esi","eax");
230 &mov ("eax","ebx"); # h1
231 &adc ("edi","edx");
232 &mul (&DWP($r0,"esp")); # h1*r0
233 &add ("esi","eax");
234 &mov ("eax","ecx"); # h2
235 &adc ("edi","edx");
236 &mul (&DWP($s3,"esp")); # h2*s3
237 &add ("esi","eax");
238 &mov ("eax",&DWP($h3,"esp"));
239 &adc ("edi","edx");
240 &mul (&DWP($s2,"esp")); # h3*s2
241 &add ("esi","eax");
242 &mov ("eax",&DWP($h4,"esp"));
243 &adc ("edi","edx");
244 &imul ("eax",&DWP($s1,"esp")); # h4*s1
245 &add ("esi","eax");
246 &mov ("eax",&DWP($h0,"esp"));
247 &adc ("edi",0);
248
249 &mul (&DWP($r2,"esp")); # h0*r2
250 &mov (&DWP($d1,"esp"),"esi");
251 &xor ("esi","esi");
252 &add ("edi","eax");
253 &mov ("eax","ebx"); # h1
254 &adc ("esi","edx");
255 &mul (&DWP($r1,"esp")); # h1*r1
256 &add ("edi","eax");
257 &mov ("eax","ecx"); # h2
258 &adc ("esi","edx");
259 &mul (&DWP($r0,"esp")); # h2*r0
260 &add ("edi","eax");
261 &mov ("eax",&DWP($h3,"esp"));
262 &adc ("esi","edx");
263 &mul (&DWP($s3,"esp")); # h3*s3
264 &add ("edi","eax");
265 &mov ("eax",&DWP($h4,"esp"));
266 &adc ("esi","edx");
267 &imul ("eax",&DWP($s2,"esp")); # h4*s2
268 &add ("edi","eax");
269 &mov ("eax",&DWP($h0,"esp"));
270 &adc ("esi",0);
271
272 &mul (&DWP($r3,"esp")); # h0*r3
273 &mov (&DWP($d2,"esp"),"edi");
274 &xor ("edi","edi");
275 &add ("esi","eax");
276 &mov ("eax","ebx"); # h1
277 &adc ("edi","edx");
278 &mul (&DWP($r2,"esp")); # h1*r2
279 &add ("esi","eax");
280 &mov ("eax","ecx"); # h2
281 &adc ("edi","edx");
282 &mul (&DWP($r1,"esp")); # h2*r1
283 &add ("esi","eax");
284 &mov ("eax",&DWP($h3,"esp"));
285 &adc ("edi","edx");
286 &mul (&DWP($r0,"esp")); # h3*r0
287 &add ("esi","eax");
288 &mov ("ecx",&DWP($h4,"esp"));
289 &adc ("edi","edx");
290
291 &mov ("edx","ecx");
292 &imul ("ecx",&DWP($s3,"esp")); # h4*s3
293 &add ("esi","ecx");
294 &mov ("eax",&DWP($d0,"esp"));
295 &adc ("edi",0);
296
297 &imul ("edx",&DWP($r0,"esp")); # h4*r0
298 &add ("edx","edi");
299
300 &mov ("ebx",&DWP($d1,"esp"));
301 &mov ("ecx",&DWP($d2,"esp"));
302
303 &mov ("edi","edx"); # last reduction step
304 &shr ("edx",2);
305 &and ("edi",3);
306 &lea ("edx",&DWP(0,"edx","edx",4)); # *5
307 &add ("eax","edx");
308 &adc ("ebx",0);
309 &adc ("ecx",0);
310 &adc ("esi",0);
311 &adc ("edi",0);
312
313 &cmp ("ebp",&wparam(2)); # done yet?
314 &jne (&label("loop"));
315
316 &mov ("edx",&wparam(0)); # ctx
317 &stack_pop(16);
318 &mov (&DWP(4*0,"edx"),"eax"); # store hash value
319 &mov (&DWP(4*1,"edx"),"ebx");
320 &mov (&DWP(4*2,"edx"),"ecx");
321 &mov (&DWP(4*3,"edx"),"esi");
322 &mov (&DWP(4*4,"edx"),"edi");
323&set_label("nodata");
324&function_end("poly1305_blocks");
325
326&function_begin("poly1305_emit");
327 &mov ("ebp",&wparam(0)); # context
328&set_label("enter_emit");
329 &mov ("edi",&wparam(1)); # output
330 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
331 &mov ("ebx",&DWP(4*1,"ebp"));
332 &mov ("ecx",&DWP(4*2,"ebp"));
333 &mov ("edx",&DWP(4*3,"ebp"));
334 &mov ("esi",&DWP(4*4,"ebp"));
335
336 &add ("eax",5); # compare to modulus
337 &adc ("ebx",0);
338 &adc ("ecx",0);
339 &adc ("edx",0);
340 &adc ("esi",0);
341 &shr ("esi",2); # did it carry/borrow?
342 &neg ("esi"); # do we choose hash-modulus?
343
344 &and ("eax","esi");
345 &and ("ebx","esi");
346 &and ("ecx","esi");
347 &and ("edx","esi");
348 &mov (&DWP(4*0,"edi"),"eax");
349 &mov (&DWP(4*1,"edi"),"ebx");
350 &mov (&DWP(4*2,"edi"),"ecx");
351 &mov (&DWP(4*3,"edi"),"edx");
352
353 &not ("esi"); # or original hash value?
354 &mov ("eax",&DWP(4*0,"ebp"));
355 &mov ("ebx",&DWP(4*1,"ebp"));
356 &mov ("ecx",&DWP(4*2,"ebp"));
357 &mov ("edx",&DWP(4*3,"ebp"));
358 &mov ("ebp",&wparam(2));
359 &and ("eax","esi");
360 &and ("ebx","esi");
361 &and ("ecx","esi");
362 &and ("edx","esi");
363 &or ("eax",&DWP(4*0,"edi"));
364 &or ("ebx",&DWP(4*1,"edi"));
365 &or ("ecx",&DWP(4*2,"edi"));
366 &or ("edx",&DWP(4*3,"edi"));
367
368 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
369 &adc ("ebx",&DWP(4*1,"ebp"));
370 &adc ("ecx",&DWP(4*2,"ebp"));
371 &adc ("edx",&DWP(4*3,"ebp"));
372
373 &mov (&DWP(4*0,"edi"),"eax");
374 &mov (&DWP(4*1,"edi"),"ebx");
375 &mov (&DWP(4*2,"edi"),"ecx");
376 &mov (&DWP(4*3,"edi"),"edx");
377&function_end("poly1305_emit");
378
379if ($sse2) {
380########################################################################
381# Layout of opaque area is following.
382#
383# unsigned __int32 h[5]; # current hash value base 2^26
384# unsigned __int32 is_base2_26;
385# unsigned __int32 r[4]; # key value base 2^32
386# unsigned __int32 pad[2];
387# struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
388#
389# where r^n are base 2^26 digits of degrees of multiplier key. There are
390# 5 digits, but last four are interleaved with multiples of 5, totalling
391# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
392
393my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
394my $MASK=$T2; # borrow and keep in mind
395
396&align (32);
397&function_begin_B("_poly1305_init_sse2");
398 &movdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
399 &lea ("edi",&DWP(16*3,"edi")); # size optimization
400 &mov ("ebp","esp");
401 &sub ("esp",16*(9+5));
402 &and ("esp",-16);
403
404 #&pand ($D4,&QWP(96,"ebx")); # magic mask
405 &movq ($MASK,&QWP(64,"ebx"));
406
407 &movdqa ($D0,$D4);
408 &movdqa ($D1,$D4);
409 &movdqa ($D2,$D4);
410
411 &pand ($D0,$MASK); # -> base 2^26
412 &psrlq ($D1,26);
413 &psrldq ($D2,6);
414 &pand ($D1,$MASK);
415 &movdqa ($D3,$D2);
416 &psrlq ($D2,4)
417 &psrlq ($D3,30);
418 &pand ($D2,$MASK);
419 &pand ($D3,$MASK);
420 &psrldq ($D4,13);
421
422 &lea ("edx",&DWP(16*9,"esp")); # size optimization
423 &mov ("ecx",2);
424&set_label("square");
425 &movdqa (&QWP(16*0,"esp"),$D0);
426 &movdqa (&QWP(16*1,"esp"),$D1);
427 &movdqa (&QWP(16*2,"esp"),$D2);
428 &movdqa (&QWP(16*3,"esp"),$D3);
429 &movdqa (&QWP(16*4,"esp"),$D4);
430
431 &movdqa ($T1,$D1);
432 &movdqa ($T0,$D2);
433 &pslld ($T1,2);
434 &pslld ($T0,2);
435 &paddd ($T1,$D1); # *5
436 &paddd ($T0,$D2); # *5
437 &movdqa (&QWP(16*5,"esp"),$T1);
438 &movdqa (&QWP(16*6,"esp"),$T0);
439 &movdqa ($T1,$D3);
440 &movdqa ($T0,$D4);
441 &pslld ($T1,2);
442 &pslld ($T0,2);
443 &paddd ($T1,$D3); # *5
444 &paddd ($T0,$D4); # *5
445 &movdqa (&QWP(16*7,"esp"),$T1);
446 &movdqa (&QWP(16*8,"esp"),$T0);
447
448 &pshufd ($T1,$D0,0b01000100);
449 &movdqa ($T0,$D1);
450 &pshufd ($D1,$D1,0b01000100);
451 &pshufd ($D2,$D2,0b01000100);
452 &pshufd ($D3,$D3,0b01000100);
453 &pshufd ($D4,$D4,0b01000100);
454 &movdqa (&QWP(16*0,"edx"),$T1);
455 &movdqa (&QWP(16*1,"edx"),$D1);
456 &movdqa (&QWP(16*2,"edx"),$D2);
457 &movdqa (&QWP(16*3,"edx"),$D3);
458 &movdqa (&QWP(16*4,"edx"),$D4);
459
460 ################################################################
461 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
462 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
463 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
464 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
465 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
466
467 &pmuludq ($D4,$D0); # h4*r0
468 &pmuludq ($D3,$D0); # h3*r0
469 &pmuludq ($D2,$D0); # h2*r0
470 &pmuludq ($D1,$D0); # h1*r0
471 &pmuludq ($D0,$T1); # h0*r0
472
473sub pmuladd {
474my $load = shift;
475my $base = shift; $base = "esp" if (!defined($base));
476
477 ################################################################
478 # As for choice to "rotate" $T0-$T2 in order to move paddq
479 # past next multiplication. While it makes code harder to read
480 # and doesn't have significant effect on most processors, it
481 # makes a lot of difference on Atom, up to 30% improvement.
482
483 &movdqa ($T1,$T0);
484 &pmuludq ($T0,&QWP(16*3,$base)); # r1*h3
485 &movdqa ($T2,$T1);
486 &pmuludq ($T1,&QWP(16*2,$base)); # r1*h2
487 &paddq ($D4,$T0);
488 &movdqa ($T0,$T2);
489 &pmuludq ($T2,&QWP(16*1,$base)); # r1*h1
490 &paddq ($D3,$T1);
491 &$load ($T1,5); # s1
492 &pmuludq ($T0,&QWP(16*0,$base)); # r1*h0
493 &paddq ($D2,$T2);
494 &pmuludq ($T1,&QWP(16*4,$base)); # s1*h4
495 &$load ($T2,2); # r2^n
496 &paddq ($D1,$T0);
497
498 &movdqa ($T0,$T2);
499 &pmuludq ($T2,&QWP(16*2,$base)); # r2*h2
500 &paddq ($D0,$T1);
501 &movdqa ($T1,$T0);
502 &pmuludq ($T0,&QWP(16*1,$base)); # r2*h1
503 &paddq ($D4,$T2);
504 &$load ($T2,6); # s2^n
505 &pmuludq ($T1,&QWP(16*0,$base)); # r2*h0
506 &paddq ($D3,$T0);
507 &movdqa ($T0,$T2);
508 &pmuludq ($T2,&QWP(16*4,$base)); # s2*h4
509 &paddq ($D2,$T1);
510 &pmuludq ($T0,&QWP(16*3,$base)); # s2*h3
511 &$load ($T1,3); # r3^n
512 &paddq ($D1,$T2);
513
514 &movdqa ($T2,$T1);
515 &pmuludq ($T1,&QWP(16*1,$base)); # r3*h1
516 &paddq ($D0,$T0);
517 &$load ($T0,7); # s3^n
518 &pmuludq ($T2,&QWP(16*0,$base)); # r3*h0
519 &paddq ($D4,$T1);
520 &movdqa ($T1,$T0);
521 &pmuludq ($T0,&QWP(16*4,$base)); # s3*h4
522 &paddq ($D3,$T2);
523 &movdqa ($T2,$T1);
524 &pmuludq ($T1,&QWP(16*3,$base)); # s3*h3
525 &paddq ($D2,$T0);
526 &pmuludq ($T2,&QWP(16*2,$base)); # s3*h2
527 &$load ($T0,4); # r4^n
528 &paddq ($D1,$T1);
529
530 &$load ($T1,8); # s4^n
531 &pmuludq ($T0,&QWP(16*0,$base)); # r4*h0
532 &paddq ($D0,$T2);
533 &movdqa ($T2,$T1);
534 &pmuludq ($T1,&QWP(16*4,$base)); # s4*h4
535 &paddq ($D4,$T0);
536 &movdqa ($T0,$T2);
537 &pmuludq ($T2,&QWP(16*1,$base)); # s4*h1
538 &paddq ($D3,$T1);
539 &movdqa ($T1,$T0);
540 &pmuludq ($T0,&QWP(16*2,$base)); # s4*h2
541 &paddq ($D0,$T2);
542 &pmuludq ($T1,&QWP(16*3,$base)); # s4*h3
543 &movdqa ($MASK,&QWP(64,"ebx"));
544 &paddq ($D1,$T0);
545 &paddq ($D2,$T1);
546}
547 &pmuladd (sub { my ($reg,$i)=@_;
548 &movdqa ($reg,&QWP(16*$i,"esp"));
549 },"edx");
550
551sub lazy_reduction {
552my $extra = shift;
553
554 ################################################################
555 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
556 # and P. Schwabe
557 #
558 # [(*) see discussion in poly1305-armv4 module]
559
560 &movdqa ($T0,$D3);
561 &pand ($D3,$MASK);
562 &psrlq ($T0,26);
563 &$extra () if (defined($extra));
564 &paddq ($T0,$D4); # h3 -> h4
565 &movdqa ($T1,$D0);
566 &pand ($D0,$MASK);
567 &psrlq ($T1,26);
568 &movdqa ($D4,$T0);
569 &paddq ($T1,$D1); # h0 -> h1
570 &psrlq ($T0,26);
571 &pand ($D4,$MASK);
572 &movdqa ($D1,$T1);
573 &psrlq ($T1,26);
574 &paddd ($D0,$T0); # favour paddd when
575 # possible, because
576 # paddq is "broken"
577 # on Atom
578 &psllq ($T0,2);
579 &paddq ($T1,$D2); # h1 -> h2
580 &paddq ($T0,$D0); # h4 -> h0 (*)
581 &pand ($D1,$MASK);
582 &movdqa ($D2,$T1);
583 &psrlq ($T1,26);
584 &pand ($D2,$MASK);
585 &paddd ($T1,$D3); # h2 -> h3
586 &movdqa ($D0,$T0);
587 &psrlq ($T0,26);
588 &movdqa ($D3,$T1);
589 &psrlq ($T1,26);
590 &pand ($D0,$MASK);
591 &paddd ($D1,$T0); # h0 -> h1
592 &pand ($D3,$MASK);
593 &paddd ($D4,$T1); # h3 -> h4
594}
595 &lazy_reduction ();
596
597 &dec ("ecx");
598 &jz (&label("square_break"));
599
600 &punpcklqdq ($D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
601 &punpcklqdq ($D1,&QWP(16*1,"esp"));
602 &punpcklqdq ($D2,&QWP(16*2,"esp"));
603 &punpcklqdq ($D3,&QWP(16*3,"esp"));
604 &punpcklqdq ($D4,&QWP(16*4,"esp"));
605 &jmp (&label("square"));
606
607&set_label("square_break");
608 &psllq ($D0,32); # -> r^3:0:r^4:0
609 &psllq ($D1,32);
610 &psllq ($D2,32);
611 &psllq ($D3,32);
612 &psllq ($D4,32);
613 &por ($D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
614 &por ($D1,&QWP(16*1,"esp"));
615 &por ($D2,&QWP(16*2,"esp"));
616 &por ($D3,&QWP(16*3,"esp"));
617 &por ($D4,&QWP(16*4,"esp"));
618
619 &pshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
620 &pshufd ($D1,$D1,0b10001101);
621 &pshufd ($D2,$D2,0b10001101);
622 &pshufd ($D3,$D3,0b10001101);
623 &pshufd ($D4,$D4,0b10001101);
624
625 &movdqu (&QWP(16*0,"edi"),$D0); # save the table
626 &movdqu (&QWP(16*1,"edi"),$D1);
627 &movdqu (&QWP(16*2,"edi"),$D2);
628 &movdqu (&QWP(16*3,"edi"),$D3);
629 &movdqu (&QWP(16*4,"edi"),$D4);
630
631 &movdqa ($T1,$D1);
632 &movdqa ($T0,$D2);
633 &pslld ($T1,2);
634 &pslld ($T0,2);
635 &paddd ($T1,$D1); # *5
636 &paddd ($T0,$D2); # *5
637 &movdqu (&QWP(16*5,"edi"),$T1);
638 &movdqu (&QWP(16*6,"edi"),$T0);
639 &movdqa ($T1,$D3);
640 &movdqa ($T0,$D4);
641 &pslld ($T1,2);
642 &pslld ($T0,2);
643 &paddd ($T1,$D3); # *5
644 &paddd ($T0,$D4); # *5
645 &movdqu (&QWP(16*7,"edi"),$T1);
646 &movdqu (&QWP(16*8,"edi"),$T0);
647
648 &mov ("esp","ebp");
649 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
650 &ret ();
651&function_end_B("_poly1305_init_sse2");
652
653&align (32);
654&function_begin("_poly1305_blocks_sse2");
655 &mov ("edi",&wparam(0)); # ctx
656 &mov ("esi",&wparam(1)); # inp
657 &mov ("ecx",&wparam(2)); # len
658
659 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
660 &and ("ecx",-16);
661 &jz (&label("nodata"));
662 &cmp ("ecx",64);
663 &jae (&label("enter_sse2"));
664 &test ("eax","eax"); # is_base2_26?
665 &jz (&label("enter_blocks"));
666
667&set_label("enter_sse2",16);
668 &call (&label("pic_point"));
669&set_label("pic_point");
670 &blindpop("ebx");
671 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
672
673 &test ("eax","eax"); # is_base2_26?
674 &jnz (&label("base2_26"));
675
676 &call ("_poly1305_init_sse2");
677
678 ################################################# base 2^32 -> base 2^26
679 &mov ("eax",&DWP(0,"edi"));
680 &mov ("ecx",&DWP(3,"edi"));
681 &mov ("edx",&DWP(6,"edi"));
682 &mov ("esi",&DWP(9,"edi"));
683 &mov ("ebp",&DWP(13,"edi"));
684 &mov (&DWP(4*5,"edi"),1); # is_base2_26
685
686 &shr ("ecx",2);
687 &and ("eax",0x3ffffff);
688 &shr ("edx",4);
689 &and ("ecx",0x3ffffff);
690 &shr ("esi",6);
691 &and ("edx",0x3ffffff);
692
693 &movd ($D0,"eax");
694 &movd ($D1,"ecx");
695 &movd ($D2,"edx");
696 &movd ($D3,"esi");
697 &movd ($D4,"ebp");
698
699 &mov ("esi",&wparam(1)); # [reload] inp
700 &mov ("ecx",&wparam(2)); # [reload] len
701 &jmp (&label("base2_32"));
702
703&set_label("base2_26",16);
704 &movd ($D0,&DWP(4*0,"edi")); # load hash value
705 &movd ($D1,&DWP(4*1,"edi"));
706 &movd ($D2,&DWP(4*2,"edi"));
707 &movd ($D3,&DWP(4*3,"edi"));
708 &movd ($D4,&DWP(4*4,"edi"));
709 &movdqa ($MASK,&QWP(64,"ebx"));
710
711&set_label("base2_32");
712 &mov ("eax",&wparam(3)); # padbit
713 &mov ("ebp","esp");
714
715 &sub ("esp",16*(5+5+5+9+9));
716 &and ("esp",-16);
717
718 &lea ("edi",&DWP(16*3,"edi")); # size optimization
719 &shl ("eax",24); # padbit
720
721 &test ("ecx",31);
722 &jz (&label("even"));
723
724 ################################################################
725 # process single block, with SSE2, because it's still faster
726 # even though half of result is discarded
727
728 &movdqu ($T1,&QWP(0,"esi")); # input
729 &lea ("esi",&DWP(16,"esi"));
730
731 &movdqa ($T0,$T1); # -> base 2^26 ...
732 &pand ($T1,$MASK);
733 &paddd ($D0,$T1); # ... and accumulate
734
735 &movdqa ($T1,$T0);
736 &psrlq ($T0,26);
737 &psrldq ($T1,6);
738 &pand ($T0,$MASK);
739 &paddd ($D1,$T0);
740
741 &movdqa ($T0,$T1);
742 &psrlq ($T1,4);
743 &pand ($T1,$MASK);
744 &paddd ($D2,$T1);
745
746 &movdqa ($T1,$T0);
747 &psrlq ($T0,30);
748 &pand ($T0,$MASK);
749 &psrldq ($T1,7);
750 &paddd ($D3,$T0);
751
752 &movd ($T0,"eax"); # padbit
753 &paddd ($D4,$T1);
754 &movd ($T1,&DWP(16*0+12,"edi")); # r0
755 &paddd ($D4,$T0);
756
757 &movdqa (&QWP(16*0,"esp"),$D0);
758 &movdqa (&QWP(16*1,"esp"),$D1);
759 &movdqa (&QWP(16*2,"esp"),$D2);
760 &movdqa (&QWP(16*3,"esp"),$D3);
761 &movdqa (&QWP(16*4,"esp"),$D4);
762
763 ################################################################
764 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
765 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
766 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
767 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
768 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
769
770 &pmuludq ($D0,$T1); # h4*r0
771 &pmuludq ($D1,$T1); # h3*r0
772 &pmuludq ($D2,$T1); # h2*r0
773 &movd ($T0,&DWP(16*1+12,"edi")); # r1
774 &pmuludq ($D3,$T1); # h1*r0
775 &pmuludq ($D4,$T1); # h0*r0
776
777 &pmuladd (sub { my ($reg,$i)=@_;
778 &movd ($reg,&DWP(16*$i+12,"edi"));
779 });
780
781 &lazy_reduction ();
782
783 &sub ("ecx",16);
784 &jz (&label("done"));
785
786&set_label("even");
787 &lea ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
788 &lea ("eax",&DWP(-16*2,"esi"));
789 &sub ("ecx",64);
790
791 ################################################################
792 # expand and copy pre-calculated table to stack
793
794 &movdqu ($T0,&QWP(16*0,"edi")); # r^1:r^2:r^3:r^4
795 &pshufd ($T1,$T0,0b01000100); # duplicate r^3:r^4
796 &cmovb ("esi","eax");
797 &pshufd ($T0,$T0,0b11101110); # duplicate r^1:r^2
798 &movdqa (&QWP(16*0,"edx"),$T1);
799 &lea ("eax",&DWP(16*10,"esp"));
800 &movdqu ($T1,&QWP(16*1,"edi"));
801 &movdqa (&QWP(16*(0-9),"edx"),$T0);
802 &pshufd ($T0,$T1,0b01000100);
803 &pshufd ($T1,$T1,0b11101110);
804 &movdqa (&QWP(16*1,"edx"),$T0);
805 &movdqu ($T0,&QWP(16*2,"edi"));
806 &movdqa (&QWP(16*(1-9),"edx"),$T1);
807 &pshufd ($T1,$T0,0b01000100);
808 &pshufd ($T0,$T0,0b11101110);
809 &movdqa (&QWP(16*2,"edx"),$T1);
810 &movdqu ($T1,&QWP(16*3,"edi"));
811 &movdqa (&QWP(16*(2-9),"edx"),$T0);
812 &pshufd ($T0,$T1,0b01000100);
813 &pshufd ($T1,$T1,0b11101110);
814 &movdqa (&QWP(16*3,"edx"),$T0);
815 &movdqu ($T0,&QWP(16*4,"edi"));
816 &movdqa (&QWP(16*(3-9),"edx"),$T1);
817 &pshufd ($T1,$T0,0b01000100);
818 &pshufd ($T0,$T0,0b11101110);
819 &movdqa (&QWP(16*4,"edx"),$T1);
820 &movdqu ($T1,&QWP(16*5,"edi"));
821 &movdqa (&QWP(16*(4-9),"edx"),$T0);
822 &pshufd ($T0,$T1,0b01000100);
823 &pshufd ($T1,$T1,0b11101110);
824 &movdqa (&QWP(16*5,"edx"),$T0);
825 &movdqu ($T0,&QWP(16*6,"edi"));
826 &movdqa (&QWP(16*(5-9),"edx"),$T1);
827 &pshufd ($T1,$T0,0b01000100);
828 &pshufd ($T0,$T0,0b11101110);
829 &movdqa (&QWP(16*6,"edx"),$T1);
830 &movdqu ($T1,&QWP(16*7,"edi"));
831 &movdqa (&QWP(16*(6-9),"edx"),$T0);
832 &pshufd ($T0,$T1,0b01000100);
833 &pshufd ($T1,$T1,0b11101110);
834 &movdqa (&QWP(16*7,"edx"),$T0);
835 &movdqu ($T0,&QWP(16*8,"edi"));
836 &movdqa (&QWP(16*(7-9),"edx"),$T1);
837 &pshufd ($T1,$T0,0b01000100);
838 &pshufd ($T0,$T0,0b11101110);
839 &movdqa (&QWP(16*8,"edx"),$T1);
840 &movdqa (&QWP(16*(8-9),"edx"),$T0);
841
842sub load_input {
843my ($inpbase,$offbase)=@_;
844
845 &movdqu ($T0,&QWP($inpbase+0,"esi")); # load input
846 &movdqu ($T1,&QWP($inpbase+16,"esi"));
847 &lea ("esi",&DWP(16*2,"esi"));
848
849 &movdqa (&QWP($offbase+16*2,"esp"),$D2);
850 &movdqa (&QWP($offbase+16*3,"esp"),$D3);
851 &movdqa (&QWP($offbase+16*4,"esp"),$D4);
852
853 &movdqa ($D2,$T0); # splat input
854 &movdqa ($D3,$T1);
855 &psrldq ($D2,6);
856 &psrldq ($D3,6);
857 &movdqa ($D4,$T0);
858 &punpcklqdq ($D2,$D3); # 2:3
859 &punpckhqdq ($D4,$T1); # 4
860 &punpcklqdq ($T0,$T1); # 0:1
861
862 &movdqa ($D3,$D2);
863 &psrlq ($D2,4);
864 &psrlq ($D3,30);
865 &movdqa ($T1,$T0);
866 &psrlq ($D4,40); # 4
867 &psrlq ($T1,26);
868 &pand ($T0,$MASK); # 0
869 &pand ($T1,$MASK); # 1
870 &pand ($D2,$MASK); # 2
871 &pand ($D3,$MASK); # 3
872 &por ($D4,&QWP(0,"ebx")); # padbit, yes, always
873
874 &movdqa (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
875 &movdqa (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
876}
877 &load_input (16*2,16*5);
878
879 &jbe (&label("skip_loop"));
880 &jmp (&label("loop"));
881
882&set_label("loop",32);
883 ################################################################
884 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
885 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
886 # \___________________/
887 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
888 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
889 # \___________________/ \____________________/
890 ################################################################
891
892 &movdqa ($T2,&QWP(16*(0-9),"edx")); # r0^2
893 &movdqa (&QWP(16*1,"eax"),$T1);
894 &movdqa (&QWP(16*2,"eax"),$D2);
895 &movdqa (&QWP(16*3,"eax"),$D3);
896 &movdqa (&QWP(16*4,"eax"),$D4);
897
898 ################################################################
899 # d4 = h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
900 # d3 = h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
901 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
902 # d1 = h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
903 # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
904
905 &movdqa ($D1,$T0);
906 &pmuludq ($T0,$T2); # h0*r0
907 &movdqa ($D0,$T1);
908 &pmuludq ($T1,$T2); # h1*r0
909 &pmuludq ($D2,$T2); # h2*r0
910 &pmuludq ($D3,$T2); # h3*r0
911 &pmuludq ($D4,$T2); # h4*r0
912
913sub pmuladd_alt {
914my $addr = shift;
915
916 &pmuludq ($D0,&$addr(8)); # h1*s4
917 &movdqa ($T2,$D1);
918 &pmuludq ($D1,&$addr(1)); # h0*r1
919 &paddq ($D0,$T0);
920 &movdqa ($T0,$T2);
921 &pmuludq ($T2,&$addr(2)); # h0*r2
922 &paddq ($D1,$T1);
923 &movdqa ($T1,$T0);
924 &pmuludq ($T0,&$addr(3)); # h0*r3
925 &paddq ($D2,$T2);
926 &movdqa ($T2,&QWP(16*1,"eax")); # pull h1
927 &pmuludq ($T1,&$addr(4)); # h0*r4
928 &paddq ($D3,$T0);
929
930 &movdqa ($T0,$T2);
931 &pmuludq ($T2,&$addr(1)); # h1*r1
932 &paddq ($D4,$T1);
933 &movdqa ($T1,$T0);
934 &pmuludq ($T0,&$addr(2)); # h1*r2
935 &paddq ($D2,$T2);
936 &movdqa ($T2,&QWP(16*2,"eax")); # pull h2
937 &pmuludq ($T1,&$addr(3)); # h1*r3
938 &paddq ($D3,$T0);
939 &movdqa ($T0,$T2);
940 &pmuludq ($T2,&$addr(7)); # h2*s3
941 &paddq ($D4,$T1);
942 &movdqa ($T1,$T0);
943 &pmuludq ($T0,&$addr(8)); # h2*s4
944 &paddq ($D0,$T2);
945
946 &movdqa ($T2,$T1);
947 &pmuludq ($T1,&$addr(1)); # h2*r1
948 &paddq ($D1,$T0);
949 &movdqa ($T0,&QWP(16*3,"eax")); # pull h3
950 &pmuludq ($T2,&$addr(2)); # h2*r2
951 &paddq ($D3,$T1);
952 &movdqa ($T1,$T0);
953 &pmuludq ($T0,&$addr(6)); # h3*s2
954 &paddq ($D4,$T2);
955 &movdqa ($T2,$T1);
956 &pmuludq ($T1,&$addr(7)); # h3*s3
957 &paddq ($D0,$T0);
958 &movdqa ($T0,$T2);
959 &pmuludq ($T2,&$addr(8)); # h3*s4
960 &paddq ($D1,$T1);
961
962 &movdqa ($T1,&QWP(16*4,"eax")); # pull h4
963 &pmuludq ($T0,&$addr(1)); # h3*r1
964 &paddq ($D2,$T2);
965 &movdqa ($T2,$T1);
966 &pmuludq ($T1,&$addr(8)); # h4*s4
967 &paddq ($D4,$T0);
968 &movdqa ($T0,$T2);
969 &pmuludq ($T2,&$addr(5)); # h4*s1
970 &paddq ($D3,$T1);
971 &movdqa ($T1,$T0);
972 &pmuludq ($T0,&$addr(6)); # h4*s2
973 &paddq ($D0,$T2);
974 &movdqa ($MASK,&QWP(64,"ebx"));
975 &pmuludq ($T1,&$addr(7)); # h4*s3
976 &paddq ($D1,$T0);
977 &paddq ($D2,$T1);
978}
979 &pmuladd_alt (sub { my $i=shift; &QWP(16*($i-9),"edx"); });
980
981 &load_input (-16*2,0);
982 &lea ("eax",&DWP(-16*2,"esi"));
983 &sub ("ecx",64);
984
985 &paddd ($T0,&QWP(16*(5+0),"esp")); # add hash value
986 &paddd ($T1,&QWP(16*(5+1),"esp"));
987 &paddd ($D2,&QWP(16*(5+2),"esp"));
988 &paddd ($D3,&QWP(16*(5+3),"esp"));
989 &paddd ($D4,&QWP(16*(5+4),"esp"));
990
991 &cmovb ("esi","eax");
992 &lea ("eax",&DWP(16*10,"esp"));
993
994 &movdqa ($T2,&QWP(16*0,"edx")); # r0^4
995 &movdqa (&QWP(16*1,"esp"),$D1);
996 &movdqa (&QWP(16*1,"eax"),$T1);
997 &movdqa (&QWP(16*2,"eax"),$D2);
998 &movdqa (&QWP(16*3,"eax"),$D3);
999 &movdqa (&QWP(16*4,"eax"),$D4);
1000
1001 ################################################################
1002 # d4 += h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
1003 # d3 += h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
1004 # d2 += h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
1005 # d1 += h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
1006 # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
1007
1008 &movdqa ($D1,$T0);
1009 &pmuludq ($T0,$T2); # h0*r0
1010 &paddq ($T0,$D0);
1011 &movdqa ($D0,$T1);
1012 &pmuludq ($T1,$T2); # h1*r0
1013 &pmuludq ($D2,$T2); # h2*r0
1014 &pmuludq ($D3,$T2); # h3*r0
1015 &pmuludq ($D4,$T2); # h4*r0
1016
1017 &paddq ($T1,&QWP(16*1,"esp"));
1018 &paddq ($D2,&QWP(16*2,"esp"));
1019 &paddq ($D3,&QWP(16*3,"esp"));
1020 &paddq ($D4,&QWP(16*4,"esp"));
1021
1022 &pmuladd_alt (sub { my $i=shift; &QWP(16*$i,"edx"); });
1023
1024 &lazy_reduction ();
1025
1026 &load_input (16*2,16*5);
1027
1028 &ja (&label("loop"));
1029
1030&set_label("skip_loop");
1031 ################################################################
1032 # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1033
1034 &pshufd ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1035 &add ("ecx",32);
1036 &jnz (&label("long_tail"));
1037
1038 &paddd ($T0,$D0); # add hash value
1039 &paddd ($T1,$D1);
1040 &paddd ($D2,&QWP(16*7,"esp"));
1041 &paddd ($D3,&QWP(16*8,"esp"));
1042 &paddd ($D4,&QWP(16*9,"esp"));
1043
1044&set_label("long_tail");
1045
1046 &movdqa (&QWP(16*0,"eax"),$T0);
1047 &movdqa (&QWP(16*1,"eax"),$T1);
1048 &movdqa (&QWP(16*2,"eax"),$D2);
1049 &movdqa (&QWP(16*3,"eax"),$D3);
1050 &movdqa (&QWP(16*4,"eax"),$D4);
1051
1052 ################################################################
1053 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1054 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1055 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1056 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1057 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1058
1059 &pmuludq ($T0,$T2); # h0*r0
1060 &pmuludq ($T1,$T2); # h1*r0
1061 &pmuludq ($D2,$T2); # h2*r0
1062 &movdqa ($D0,$T0);
1063 &pshufd ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1064 &pmuludq ($D3,$T2); # h3*r0
1065 &movdqa ($D1,$T1);
1066 &pmuludq ($D4,$T2); # h4*r0
1067
1068 &pmuladd (sub { my ($reg,$i)=@_;
1069 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1070 },"eax");
1071
1072 &jz (&label("short_tail"));
1073
1074 &load_input (-16*2,0);
1075
1076 &pshufd ($T2,&QWP(16*0,"edx"),0x10); # r0^n
1077 &paddd ($T0,&QWP(16*5,"esp")); # add hash value
1078 &paddd ($T1,&QWP(16*6,"esp"));
1079 &paddd ($D2,&QWP(16*7,"esp"));
1080 &paddd ($D3,&QWP(16*8,"esp"));
1081 &paddd ($D4,&QWP(16*9,"esp"));
1082
1083 ################################################################
1084 # multiply inp[0:1] by r^4:r^3 and accumulate
1085
1086 &movdqa (&QWP(16*0,"esp"),$T0);
1087 &pmuludq ($T0,$T2); # h0*r0
1088 &movdqa (&QWP(16*1,"esp"),$T1);
1089 &pmuludq ($T1,$T2); # h1*r0
1090 &paddq ($D0,$T0);
1091 &movdqa ($T0,$D2);
1092 &pmuludq ($D2,$T2); # h2*r0
1093 &paddq ($D1,$T1);
1094 &movdqa ($T1,$D3);
1095 &pmuludq ($D3,$T2); # h3*r0
1096 &paddq ($D2,&QWP(16*2,"esp"));
1097 &movdqa (&QWP(16*2,"esp"),$T0);
1098 &pshufd ($T0,&QWP(16*1,"edx"),0x10); # r1^n
1099 &paddq ($D3,&QWP(16*3,"esp"));
1100 &movdqa (&QWP(16*3,"esp"),$T1);
1101 &movdqa ($T1,$D4);
1102 &pmuludq ($D4,$T2); # h4*r0
1103 &paddq ($D4,&QWP(16*4,"esp"));
1104 &movdqa (&QWP(16*4,"esp"),$T1);
1105
1106 &pmuladd (sub { my ($reg,$i)=@_;
1107 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1108 });
1109
1110&set_label("short_tail");
1111
1112 ################################################################
1113 # horizontal addition
1114
1115 &pshufd ($T1,$D4,0b01001110);
1116 &pshufd ($T0,$D3,0b01001110);
1117 &paddq ($D4,$T1);
1118 &paddq ($D3,$T0);
1119 &pshufd ($T1,$D0,0b01001110);
1120 &pshufd ($T0,$D1,0b01001110);
1121 &paddq ($D0,$T1);
1122 &paddq ($D1,$T0);
1123 &pshufd ($T1,$D2,0b01001110);
1124 #&paddq ($D2,$T1);
1125
1126 &lazy_reduction (sub { &paddq ($D2,$T1) });
1127
1128&set_label("done");
1129 &movd (&DWP(-16*3+4*0,"edi"),$D0); # store hash value
1130 &movd (&DWP(-16*3+4*1,"edi"),$D1);
1131 &movd (&DWP(-16*3+4*2,"edi"),$D2);
1132 &movd (&DWP(-16*3+4*3,"edi"),$D3);
1133 &movd (&DWP(-16*3+4*4,"edi"),$D4);
1134 &mov ("esp","ebp");
1135&set_label("nodata");
1136&function_end("_poly1305_blocks_sse2");
1137
1138&align (32);
1139&function_begin("_poly1305_emit_sse2");
1140 &mov ("ebp",&wparam(0)); # context
1141
1142 &cmp (&DWP(4*5,"ebp"),0); # is_base2_26?
1143 &je (&label("enter_emit"));
1144
1145 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
1146 &mov ("edi",&DWP(4*1,"ebp"));
1147 &mov ("ecx",&DWP(4*2,"ebp"));
1148 &mov ("edx",&DWP(4*3,"ebp"));
1149 &mov ("esi",&DWP(4*4,"ebp"));
1150
1151 &mov ("ebx","edi"); # base 2^26 -> base 2^32
1152 &shl ("edi",26);
1153 &shr ("ebx",6);
1154 &add ("eax","edi");
1155 &mov ("edi","ecx");
1156 &adc ("ebx",0);
1157
1158 &shl ("edi",20);
1159 &shr ("ecx",12);
1160 &add ("ebx","edi");
1161 &mov ("edi","edx");
1162 &adc ("ecx",0);
1163
1164 &shl ("edi",14);
1165 &shr ("edx",18);
1166 &add ("ecx","edi");
1167 &mov ("edi","esi");
1168 &adc ("edx",0);
1169
1170 &shl ("edi",8);
1171 &shr ("esi",24);
1172 &add ("edx","edi");
1173 &adc ("esi",0); # can be partially reduced
1174
1175 &mov ("edi","esi"); # final reduction
1176 &and ("esi",3);
1177 &shr ("edi",2);
1178 &lea ("ebp",&DWP(0,"edi","edi",4)); # *5
1179 &mov ("edi",&wparam(1)); # output
1180 &add ("eax","ebp");
1181 &mov ("ebp",&wparam(2)); # key
1182 &adc ("ebx",0);
1183 &adc ("ecx",0);
1184 &adc ("edx",0);
1185 &adc ("esi",0);
1186
1187 &movd ($D0,"eax"); # offload original hash value
1188 &add ("eax",5); # compare to modulus
1189 &movd ($D1,"ebx");
1190 &adc ("ebx",0);
1191 &movd ($D2,"ecx");
1192 &adc ("ecx",0);
1193 &movd ($D3,"edx");
1194 &adc ("edx",0);
1195 &adc ("esi",0);
1196 &shr ("esi",2); # did it carry/borrow?
1197
1198 &neg ("esi"); # do we choose (hash-modulus) ...
1199 &and ("eax","esi");
1200 &and ("ebx","esi");
1201 &and ("ecx","esi");
1202 &and ("edx","esi");
1203 &mov (&DWP(4*0,"edi"),"eax");
1204 &movd ("eax",$D0);
1205 &mov (&DWP(4*1,"edi"),"ebx");
1206 &movd ("ebx",$D1);
1207 &mov (&DWP(4*2,"edi"),"ecx");
1208 &movd ("ecx",$D2);
1209 &mov (&DWP(4*3,"edi"),"edx");
1210 &movd ("edx",$D3);
1211
1212 &not ("esi"); # ... or original hash value?
1213 &and ("eax","esi");
1214 &and ("ebx","esi");
1215 &or ("eax",&DWP(4*0,"edi"));
1216 &and ("ecx","esi");
1217 &or ("ebx",&DWP(4*1,"edi"));
1218 &and ("edx","esi");
1219 &or ("ecx",&DWP(4*2,"edi"));
1220 &or ("edx",&DWP(4*3,"edi"));
1221
1222 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
1223 &adc ("ebx",&DWP(4*1,"ebp"));
1224 &mov (&DWP(4*0,"edi"),"eax");
1225 &adc ("ecx",&DWP(4*2,"ebp"));
1226 &mov (&DWP(4*1,"edi"),"ebx");
1227 &adc ("edx",&DWP(4*3,"ebp"));
1228 &mov (&DWP(4*2,"edi"),"ecx");
1229 &mov (&DWP(4*3,"edi"),"edx");
1230&function_end("_poly1305_emit_sse2");
1231
1232if ($avx>1) {
1233########################################################################
1234# Note that poly1305_init_avx2 operates on %xmm, I could have used
1235# poly1305_init_sse2...
1236
1237&align (32);
1238&function_begin_B("_poly1305_init_avx2");
1239 &vmovdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
1240 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1241 &mov ("ebp","esp");
1242 &sub ("esp",16*(9+5));
1243 &and ("esp",-16);
1244
1245 #&vpand ($D4,$D4,&QWP(96,"ebx")); # magic mask
1246 &vmovdqa ($MASK,&QWP(64,"ebx"));
1247
1248 &vpand ($D0,$D4,$MASK); # -> base 2^26
1249 &vpsrlq ($D1,$D4,26);
1250 &vpsrldq ($D3,$D4,6);
1251 &vpand ($D1,$D1,$MASK);
1252 &vpsrlq ($D2,$D3,4)
1253 &vpsrlq ($D3,$D3,30);
1254 &vpand ($D2,$D2,$MASK);
1255 &vpand ($D3,$D3,$MASK);
1256 &vpsrldq ($D4,$D4,13);
1257
1258 &lea ("edx",&DWP(16*9,"esp")); # size optimization
1259 &mov ("ecx",2);
1260&set_label("square");
1261 &vmovdqa (&QWP(16*0,"esp"),$D0);
1262 &vmovdqa (&QWP(16*1,"esp"),$D1);
1263 &vmovdqa (&QWP(16*2,"esp"),$D2);
1264 &vmovdqa (&QWP(16*3,"esp"),$D3);
1265 &vmovdqa (&QWP(16*4,"esp"),$D4);
1266
1267 &vpslld ($T1,$D1,2);
1268 &vpslld ($T0,$D2,2);
1269 &vpaddd ($T1,$T1,$D1); # *5
1270 &vpaddd ($T0,$T0,$D2); # *5
1271 &vmovdqa (&QWP(16*5,"esp"),$T1);
1272 &vmovdqa (&QWP(16*6,"esp"),$T0);
1273 &vpslld ($T1,$D3,2);
1274 &vpslld ($T0,$D4,2);
1275 &vpaddd ($T1,$T1,$D3); # *5
1276 &vpaddd ($T0,$T0,$D4); # *5
1277 &vmovdqa (&QWP(16*7,"esp"),$T1);
1278 &vmovdqa (&QWP(16*8,"esp"),$T0);
1279
1280 &vpshufd ($T0,$D0,0b01000100);
1281 &vmovdqa ($T1,$D1);
1282 &vpshufd ($D1,$D1,0b01000100);
1283 &vpshufd ($D2,$D2,0b01000100);
1284 &vpshufd ($D3,$D3,0b01000100);
1285 &vpshufd ($D4,$D4,0b01000100);
1286 &vmovdqa (&QWP(16*0,"edx"),$T0);
1287 &vmovdqa (&QWP(16*1,"edx"),$D1);
1288 &vmovdqa (&QWP(16*2,"edx"),$D2);
1289 &vmovdqa (&QWP(16*3,"edx"),$D3);
1290 &vmovdqa (&QWP(16*4,"edx"),$D4);
1291
1292 ################################################################
1293 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1294 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1295 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1296 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1297 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1298
1299 &vpmuludq ($D4,$D4,$D0); # h4*r0
1300 &vpmuludq ($D3,$D3,$D0); # h3*r0
1301 &vpmuludq ($D2,$D2,$D0); # h2*r0
1302 &vpmuludq ($D1,$D1,$D0); # h1*r0
1303 &vpmuludq ($D0,$T0,$D0); # h0*r0
1304
1305 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # r1*h3
1306 &vpaddq ($D4,$D4,$T0);
1307 &vpmuludq ($T2,$T1,&QWP(16*2,"edx")); # r1*h2
1308 &vpaddq ($D3,$D3,$T2);
1309 &vpmuludq ($T0,$T1,&QWP(16*1,"edx")); # r1*h1
1310 &vpaddq ($D2,$D2,$T0);
1311 &vmovdqa ($T2,&QWP(16*5,"esp")); # s1
1312 &vpmuludq ($T1,$T1,&QWP(16*0,"edx")); # r1*h0
1313 &vpaddq ($D1,$D1,$T1);
1314 &vmovdqa ($T0,&QWP(16*2,"esp")); # r2
1315 &vpmuludq ($T2,$T2,&QWP(16*4,"edx")); # s1*h4
1316 &vpaddq ($D0,$D0,$T2);
1317
1318 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # r2*h2
1319 &vpaddq ($D4,$D4,$T1);
1320 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r2*h1
1321 &vpaddq ($D3,$D3,$T2);
1322 &vmovdqa ($T1,&QWP(16*6,"esp")); # s2
1323 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r2*h0
1324 &vpaddq ($D2,$D2,$T0);
1325 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s2*h4
1326 &vpaddq ($D1,$D1,$T2);
1327 &vmovdqa ($T0,&QWP(16*3,"esp")); # r3
1328 &vpmuludq ($T1,$T1,&QWP(16*3,"edx")); # s2*h3
1329 &vpaddq ($D0,$D0,$T1);
1330
1331 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r3*h1
1332 &vpaddq ($D4,$D4,$T2);
1333 &vmovdqa ($T1,&QWP(16*7,"esp")); # s3
1334 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r3*h0
1335 &vpaddq ($D3,$D3,$T0);
1336 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s3*h4
1337 &vpaddq ($D2,$D2,$T2);
1338 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # s3*h3
1339 &vpaddq ($D1,$D1,$T0);
1340 &vmovdqa ($T2,&QWP(16*4,"esp")); # r4
1341 &vpmuludq ($T1,$T1,&QWP(16*2,"edx")); # s3*h2
1342 &vpaddq ($D0,$D0,$T1);
1343
1344 &vmovdqa ($T0,&QWP(16*8,"esp")); # s4
1345 &vpmuludq ($T2,$T2,&QWP(16*0,"edx")); # r4*h0
1346 &vpaddq ($D4,$D4,$T2);
1347 &vpmuludq ($T1,$T0,&QWP(16*4,"edx")); # s4*h4
1348 &vpaddq ($D3,$D3,$T1);
1349 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # s4*h1
1350 &vpaddq ($D0,$D0,$T2);
1351 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # s4*h2
1352 &vpaddq ($D1,$D1,$T1);
1353 &vmovdqa ($MASK,&QWP(64,"ebx"));
1354 &vpmuludq ($T0,$T0,&QWP(16*3,"edx")); # s4*h3
1355 &vpaddq ($D2,$D2,$T0);
1356
1357 ################################################################
1358 # lazy reduction
1359 &vpsrlq ($T0,$D3,26);
1360 &vpand ($D3,$D3,$MASK);
1361 &vpsrlq ($T1,$D0,26);
1362 &vpand ($D0,$D0,$MASK);
1363 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1364 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1365 &vpsrlq ($T0,$D4,26);
1366 &vpand ($D4,$D4,$MASK);
1367 &vpsrlq ($T1,$D1,26);
1368 &vpand ($D1,$D1,$MASK);
1369 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1370 &vpaddd ($D0,$D0,$T0);
1371 &vpsllq ($T0,$T0,2);
1372 &vpsrlq ($T1,$D2,26);
1373 &vpand ($D2,$D2,$MASK);
1374 &vpaddd ($D0,$D0,$T0); # h4 -> h0
1375 &vpaddd ($D3,$D3,$T1); # h2 -> h3
1376 &vpsrlq ($T1,$D3,26);
1377 &vpsrlq ($T0,$D0,26);
1378 &vpand ($D0,$D0,$MASK);
1379 &vpand ($D3,$D3,$MASK);
1380 &vpaddd ($D1,$D1,$T0); # h0 -> h1
1381 &vpaddd ($D4,$D4,$T1); # h3 -> h4
1382
1383 &dec ("ecx");
1384 &jz (&label("square_break"));
1385
1386 &vpunpcklqdq ($D0,$D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
1387 &vpunpcklqdq ($D1,$D1,&QWP(16*1,"esp"));
1388 &vpunpcklqdq ($D2,$D2,&QWP(16*2,"esp"));
1389 &vpunpcklqdq ($D3,$D3,&QWP(16*3,"esp"));
1390 &vpunpcklqdq ($D4,$D4,&QWP(16*4,"esp"));
1391 &jmp (&label("square"));
1392
1393&set_label("square_break");
1394 &vpsllq ($D0,$D0,32); # -> r^3:0:r^4:0
1395 &vpsllq ($D1,$D1,32);
1396 &vpsllq ($D2,$D2,32);
1397 &vpsllq ($D3,$D3,32);
1398 &vpsllq ($D4,$D4,32);
1399 &vpor ($D0,$D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
1400 &vpor ($D1,$D1,&QWP(16*1,"esp"));
1401 &vpor ($D2,$D2,&QWP(16*2,"esp"));
1402 &vpor ($D3,$D3,&QWP(16*3,"esp"));
1403 &vpor ($D4,$D4,&QWP(16*4,"esp"));
1404
1405 &vpshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
1406 &vpshufd ($D1,$D1,0b10001101);
1407 &vpshufd ($D2,$D2,0b10001101);
1408 &vpshufd ($D3,$D3,0b10001101);
1409 &vpshufd ($D4,$D4,0b10001101);
1410
1411 &vmovdqu (&QWP(16*0,"edi"),$D0); # save the table
1412 &vmovdqu (&QWP(16*1,"edi"),$D1);
1413 &vmovdqu (&QWP(16*2,"edi"),$D2);
1414 &vmovdqu (&QWP(16*3,"edi"),$D3);
1415 &vmovdqu (&QWP(16*4,"edi"),$D4);
1416
1417 &vpslld ($T1,$D1,2);
1418 &vpslld ($T0,$D2,2);
1419 &vpaddd ($T1,$T1,$D1); # *5
1420 &vpaddd ($T0,$T0,$D2); # *5
1421 &vmovdqu (&QWP(16*5,"edi"),$T1);
1422 &vmovdqu (&QWP(16*6,"edi"),$T0);
1423 &vpslld ($T1,$D3,2);
1424 &vpslld ($T0,$D4,2);
1425 &vpaddd ($T1,$T1,$D3); # *5
1426 &vpaddd ($T0,$T0,$D4); # *5
1427 &vmovdqu (&QWP(16*7,"edi"),$T1);
1428 &vmovdqu (&QWP(16*8,"edi"),$T0);
1429
1430 &mov ("esp","ebp");
1431 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
1432 &ret ();
1433&function_end_B("_poly1305_init_avx2");
1434
1435########################################################################
1436# now it's time to switch to %ymm
1437
1438my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1439my $MASK=$T2;
1440
1441sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1442
1443&align (32);
1444&function_begin("_poly1305_blocks_avx2");
1445 &mov ("edi",&wparam(0)); # ctx
1446 &mov ("esi",&wparam(1)); # inp
1447 &mov ("ecx",&wparam(2)); # len
1448
1449 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
1450 &and ("ecx",-16);
1451 &jz (&label("nodata"));
1452 &cmp ("ecx",64);
1453 &jae (&label("enter_avx2"));
1454 &test ("eax","eax"); # is_base2_26?
1455 &jz (&label("enter_blocks"));
1456
1457&set_label("enter_avx2");
1458 &vzeroupper ();
1459
1460 &call (&label("pic_point"));
1461&set_label("pic_point");
1462 &blindpop("ebx");
1463 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1464
1465 &test ("eax","eax"); # is_base2_26?
1466 &jnz (&label("base2_26"));
1467
1468 &call ("_poly1305_init_avx2");
1469
1470 ################################################# base 2^32 -> base 2^26
1471 &mov ("eax",&DWP(0,"edi"));
1472 &mov ("ecx",&DWP(3,"edi"));
1473 &mov ("edx",&DWP(6,"edi"));
1474 &mov ("esi",&DWP(9,"edi"));
1475 &mov ("ebp",&DWP(13,"edi"));
1476
1477 &shr ("ecx",2);
1478 &and ("eax",0x3ffffff);
1479 &shr ("edx",4);
1480 &and ("ecx",0x3ffffff);
1481 &shr ("esi",6);
1482 &and ("edx",0x3ffffff);
1483
1484 &mov (&DWP(4*0,"edi"),"eax");
1485 &mov (&DWP(4*1,"edi"),"ecx");
1486 &mov (&DWP(4*2,"edi"),"edx");
1487 &mov (&DWP(4*3,"edi"),"esi");
1488 &mov (&DWP(4*4,"edi"),"ebp");
1489 &mov (&DWP(4*5,"edi"),1); # is_base2_26
1490
1491 &mov ("esi",&wparam(1)); # [reload] inp
1492 &mov ("ecx",&wparam(2)); # [reload] len
1493
1494&set_label("base2_26");
1495 &mov ("eax",&wparam(3)); # padbit
1496 &mov ("ebp","esp");
1497
1498 &sub ("esp",32*(5+9));
1499 &and ("esp",-512); # ensure that frame
1500 # doesn't cross page
1501 # boundary, which is
1502 # essential for
1503 # misaligned 32-byte
1504 # loads
1505
1506 ################################################################
1507 # expand and copy pre-calculated table to stack
1508
1509 &vmovdqu (&X($D0),&QWP(16*(3+0),"edi"));
1510 &lea ("edx",&DWP(32*5+128,"esp")); # +128 size optimization
1511 &vmovdqu (&X($D1),&QWP(16*(3+1),"edi"));
1512 &vmovdqu (&X($D2),&QWP(16*(3+2),"edi"));
1513 &vmovdqu (&X($D3),&QWP(16*(3+3),"edi"));
1514 &vmovdqu (&X($D4),&QWP(16*(3+4),"edi"));
1515 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1516 &vpermq ($D0,$D0,0b01000000); # 00001234 -> 12343434
1517 &vpermq ($D1,$D1,0b01000000);
1518 &vpermq ($D2,$D2,0b01000000);
1519 &vpermq ($D3,$D3,0b01000000);
1520 &vpermq ($D4,$D4,0b01000000);
1521 &vpshufd ($D0,$D0,0b11001000); # 12343434 -> 14243444
1522 &vpshufd ($D1,$D1,0b11001000);
1523 &vpshufd ($D2,$D2,0b11001000);
1524 &vpshufd ($D3,$D3,0b11001000);
1525 &vpshufd ($D4,$D4,0b11001000);
1526 &vmovdqa (&QWP(32*0-128,"edx"),$D0);
1527 &vmovdqu (&X($D0),&QWP(16*5,"edi"));
1528 &vmovdqa (&QWP(32*1-128,"edx"),$D1);
1529 &vmovdqu (&X($D1),&QWP(16*6,"edi"));
1530 &vmovdqa (&QWP(32*2-128,"edx"),$D2);
1531 &vmovdqu (&X($D2),&QWP(16*7,"edi"));
1532 &vmovdqa (&QWP(32*3-128,"edx"),$D3);
1533 &vmovdqu (&X($D3),&QWP(16*8,"edi"));
1534 &vmovdqa (&QWP(32*4-128,"edx"),$D4);
1535 &vpermq ($D0,$D0,0b01000000);
1536 &vpermq ($D1,$D1,0b01000000);
1537 &vpermq ($D2,$D2,0b01000000);
1538 &vpermq ($D3,$D3,0b01000000);
1539 &vpshufd ($D0,$D0,0b11001000);
1540 &vpshufd ($D1,$D1,0b11001000);
1541 &vpshufd ($D2,$D2,0b11001000);
1542 &vpshufd ($D3,$D3,0b11001000);
1543 &vmovdqa (&QWP(32*5-128,"edx"),$D0);
1544 &vmovd (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1545 &vmovdqa (&QWP(32*6-128,"edx"),$D1);
1546 &vmovd (&X($D1),&DWP(-16*3+4*1,"edi"));
1547 &vmovdqa (&QWP(32*7-128,"edx"),$D2);
1548 &vmovd (&X($D2),&DWP(-16*3+4*2,"edi"));
1549 &vmovdqa (&QWP(32*8-128,"edx"),$D3);
1550 &vmovd (&X($D3),&DWP(-16*3+4*3,"edi"));
1551 &vmovd (&X($D4),&DWP(-16*3+4*4,"edi"));
1552 &vmovdqa ($MASK,&QWP(64,"ebx"));
1553 &neg ("eax"); # padbit
1554
1555 &test ("ecx",63);
1556 &jz (&label("even"));
1557
1558 &mov ("edx","ecx");
1559 &and ("ecx",-64);
1560 &and ("edx",63);
1561
1562 &vmovdqu (&X($T0),&QWP(16*0,"esi"));
1563 &cmp ("edx",32);
1564 &jb (&label("one"));
1565
1566 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1567 &je (&label("two"));
1568
1569 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1570 &lea ("esi",&DWP(16*3,"esi"));
1571 &lea ("ebx",&DWP(8,"ebx")); # three padbits
1572 &lea ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1573 &jmp (&label("tail"));
1574
1575&set_label("two");
1576 &lea ("esi",&DWP(16*2,"esi"));
1577 &lea ("ebx",&DWP(16,"ebx")); # two padbits
1578 &lea ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1579 &jmp (&label("tail"));
1580
1581&set_label("one");
1582 &lea ("esi",&DWP(16*1,"esi"));
1583 &vpxor ($T1,$T1,$T1);
1584 &lea ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1585 &lea ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1586 &jmp (&label("tail"));
1587
1588# (*) spots marked with '--' are data from next table entry, but they
1589# are multiplied by 0 and therefore rendered insignificant
1590
1591&set_label("even",32);
1592 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1593 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1594 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1595 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1596 &lea ("esi",&DWP(16*4,"esi"));
1597 &sub ("ecx",64);
1598 &jz (&label("tail"));
1599
1600&set_label("loop");
1601 ################################################################
1602 # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1603 # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1604 # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1605 # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1606 # \________/ \_______/
1607 ################################################################
1608
1609sub vsplat_input {
1610 &vmovdqa (&QWP(32*2,"esp"),$D2);
1611 &vpsrldq ($D2,$T0,6); # splat input
1612 &vmovdqa (&QWP(32*0,"esp"),$D0);
1613 &vpsrldq ($D0,$T1,6);
1614 &vmovdqa (&QWP(32*1,"esp"),$D1);
1615 &vpunpckhqdq ($D1,$T0,$T1); # 4
1616 &vpunpcklqdq ($T0,$T0,$T1); # 0:1
1617 &vpunpcklqdq ($D2,$D2,$D0); # 2:3
1618
1619 &vpsrlq ($D0,$D2,30);
1620 &vpsrlq ($D2,$D2,4);
1621 &vpsrlq ($T1,$T0,26);
1622 &vpsrlq ($D1,$D1,40); # 4
1623 &vpand ($D2,$D2,$MASK); # 2
1624 &vpand ($T0,$T0,$MASK); # 0
1625 &vpand ($T1,$T1,$MASK); # 1
1626 &vpand ($D0,$D0,$MASK); # 3 (*)
1627 &vpor ($D1,$D1,&QWP(0,"ebx")); # padbit, yes, always
1628
1629 # (*) note that output is counterintuitive, inp[3:4] is
1630 # returned in $D1-2, while $D3-4 are preserved;
1631}
1632 &vsplat_input ();
1633
1634sub vpmuladd {
1635my $addr = shift;
1636
1637 &vpaddq ($D2,$D2,&QWP(32*2,"esp")); # add hash value
1638 &vpaddq ($T0,$T0,&QWP(32*0,"esp"));
1639 &vpaddq ($T1,$T1,&QWP(32*1,"esp"));
1640 &vpaddq ($D0,$D0,$D3);
1641 &vpaddq ($D1,$D1,$D4);
1642
1643 ################################################################
1644 # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4
1645 # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0
1646 # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1647 # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2
1648 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
1649
1650 &vpmuludq ($D3,$D2,&$addr(1)); # d3 = h2*r1
1651 &vmovdqa (QWP(32*1,"esp"),$T1);
1652 &vpmuludq ($D4,$D2,&$addr(2)); # d4 = h2*r2
1653 &vmovdqa (QWP(32*3,"esp"),$D0);
1654 &vpmuludq ($D0,$D2,&$addr(7)); # d0 = h2*s3
1655 &vmovdqa (QWP(32*4,"esp"),$D1);
1656 &vpmuludq ($D1,$D2,&$addr(8)); # d1 = h2*s4
1657 &vpmuludq ($D2,$D2,&$addr(0)); # d2 = h2*r0
1658
1659 &vpmuludq ($T2,$T0,&$addr(3)); # h0*r3
1660 &vpaddq ($D3,$D3,$T2); # d3 += h0*r3
1661 &vpmuludq ($T1,$T0,&$addr(4)); # h0*r4
1662 &vpaddq ($D4,$D4,$T1); # d4 + h0*r4
1663 &vpmuludq ($T2,$T0,&$addr(0)); # h0*r0
1664 &vpaddq ($D0,$D0,$T2); # d0 + h0*r0
1665 &vmovdqa ($T2,&QWP(32*1,"esp")); # h1
1666 &vpmuludq ($T1,$T0,&$addr(1)); # h0*r1
1667 &vpaddq ($D1,$D1,$T1); # d1 += h0*r1
1668 &vpmuludq ($T0,$T0,&$addr(2)); # h0*r2
1669 &vpaddq ($D2,$D2,$T0); # d2 += h0*r2
1670
1671 &vpmuludq ($T1,$T2,&$addr(2)); # h1*r2
1672 &vpaddq ($D3,$D3,$T1); # d3 += h1*r2
1673 &vpmuludq ($T0,$T2,&$addr(3)); # h1*r3
1674 &vpaddq ($D4,$D4,$T0); # d4 += h1*r3
1675 &vpmuludq ($T1,$T2,&$addr(8)); # h1*s4
1676 &vpaddq ($D0,$D0,$T1); # d0 += h1*s4
1677 &vmovdqa ($T1,&QWP(32*3,"esp")); # h3
1678 &vpmuludq ($T0,$T2,&$addr(0)); # h1*r0
1679 &vpaddq ($D1,$D1,$T0); # d1 += h1*r0
1680 &vpmuludq ($T2,$T2,&$addr(1)); # h1*r1
1681 &vpaddq ($D2,$D2,$T2); # d2 += h1*r1
1682
1683 &vpmuludq ($T0,$T1,&$addr(0)); # h3*r0
1684 &vpaddq ($D3,$D3,$T0); # d3 += h3*r0
1685 &vpmuludq ($T2,$T1,&$addr(1)); # h3*r1
1686 &vpaddq ($D4,$D4,$T2); # d4 += h3*r1
1687 &vpmuludq ($T0,$T1,&$addr(6)); # h3*s2
1688 &vpaddq ($D0,$D0,$T0); # d0 += h3*s2
1689 &vmovdqa ($T0,&QWP(32*4,"esp")); # h4
1690 &vpmuludq ($T2,$T1,&$addr(7)); # h3*s3
1691 &vpaddq ($D1,$D1,$T2); # d1+= h3*s3
1692 &vpmuludq ($T1,$T1,&$addr(8)); # h3*s4
1693 &vpaddq ($D2,$D2,$T1); # d2 += h3*s4
1694
1695 &vpmuludq ($T2,$T0,&$addr(8)); # h4*s4
1696 &vpaddq ($D3,$D3,$T2); # d3 += h4*s4
1697 &vpmuludq ($T1,$T0,&$addr(5)); # h4*s1
1698 &vpaddq ($D0,$D0,$T1); # d0 += h4*s1
1699 &vpmuludq ($T2,$T0,&$addr(0)); # h4*r0
1700 &vpaddq ($D4,$D4,$T2); # d4 += h4*r0
1701 &vmovdqa ($MASK,&QWP(64,"ebx"));
1702 &vpmuludq ($T1,$T0,&$addr(6)); # h4*s2
1703 &vpaddq ($D1,$D1,$T1); # d1 += h4*s2
1704 &vpmuludq ($T0,$T0,&$addr(7)); # h4*s3
1705 &vpaddq ($D2,$D2,$T0); # d2 += h4*s3
1706}
1707 &vpmuladd (sub { my $i=shift; &QWP(32*$i-128,"edx"); });
1708
1709sub vlazy_reduction {
1710 ################################################################
1711 # lazy reduction
1712
1713 &vpsrlq ($T0,$D3,26);
1714 &vpand ($D3,$D3,$MASK);
1715 &vpsrlq ($T1,$D0,26);
1716 &vpand ($D0,$D0,$MASK);
1717 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1718 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1719 &vpsrlq ($T0,$D4,26);
1720 &vpand ($D4,$D4,$MASK);
1721 &vpsrlq ($T1,$D1,26);
1722 &vpand ($D1,$D1,$MASK);
1723 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1724 &vpaddq ($D0,$D0,$T0);
1725 &vpsllq ($T0,$T0,2);
1726 &vpsrlq ($T1,$D2,26);
1727 &vpand ($D2,$D2,$MASK);
1728 &vpaddq ($D0,$D0,$T0); # h4 -> h0
1729 &vpaddq ($D3,$D3,$T1); # h2 -> h3
1730 &vpsrlq ($T1,$D3,26);
1731 &vpsrlq ($T0,$D0,26);
1732 &vpand ($D0,$D0,$MASK);
1733 &vpand ($D3,$D3,$MASK);
1734 &vpaddq ($D1,$D1,$T0); # h0 -> h1
1735 &vpaddq ($D4,$D4,$T1); # h3 -> h4
1736}
1737 &vlazy_reduction();
1738
1739 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1740 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1741 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1742 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1743 &lea ("esi",&DWP(16*4,"esi"));
1744 &sub ("ecx",64);
1745 &jnz (&label("loop"));
1746
1747&set_label("tail");
1748 &vsplat_input ();
1749 &and ("ebx",-64); # restore pointer
1750
1751 &vpmuladd (sub { my $i=shift; &QWP(4+32*$i-128,"edx"); });
1752
1753 ################################################################
1754 # horizontal addition
1755
1756 &vpsrldq ($T0,$D4,8);
1757 &vpsrldq ($T1,$D3,8);
1758 &vpaddq ($D4,$D4,$T0);
1759 &vpsrldq ($T0,$D0,8);
1760 &vpaddq ($D3,$D3,$T1);
1761 &vpsrldq ($T1,$D1,8);
1762 &vpaddq ($D0,$D0,$T0);
1763 &vpsrldq ($T0,$D2,8);
1764 &vpaddq ($D1,$D1,$T1);
1765 &vpermq ($T1,$D4,2); # keep folding
1766 &vpaddq ($D2,$D2,$T0);
1767 &vpermq ($T0,$D3,2);
1768 &vpaddq ($D4,$D4,$T1);
1769 &vpermq ($T1,$D0,2);
1770 &vpaddq ($D3,$D3,$T0);
1771 &vpermq ($T0,$D1,2);
1772 &vpaddq ($D0,$D0,$T1);
1773 &vpermq ($T1,$D2,2);
1774 &vpaddq ($D1,$D1,$T0);
1775 &vpaddq ($D2,$D2,$T1);
1776
1777 &vlazy_reduction();
1778
1779 &cmp ("ecx",0);
1780 &je (&label("done"));
1781
1782 ################################################################
1783 # clear all but single word
1784
1785 &vpshufd (&X($D0),&X($D0),0b11111100);
1786 &lea ("edx",&DWP(32*5+128,"esp")); # restore pointer
1787 &vpshufd (&X($D1),&X($D1),0b11111100);
1788 &vpshufd (&X($D2),&X($D2),0b11111100);
1789 &vpshufd (&X($D3),&X($D3),0b11111100);
1790 &vpshufd (&X($D4),&X($D4),0b11111100);
1791 &jmp (&label("even"));
1792
1793&set_label("done",16);
1794 &vmovd (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1795 &vmovd (&DWP(-16*3+4*1,"edi"),&X($D1));
1796 &vmovd (&DWP(-16*3+4*2,"edi"),&X($D2));
1797 &vmovd (&DWP(-16*3+4*3,"edi"),&X($D3));
1798 &vmovd (&DWP(-16*3+4*4,"edi"),&X($D4));
1799 &vzeroupper ();
1800 &mov ("esp","ebp");
1801&set_label("nodata");
1802&function_end("_poly1305_blocks_avx2");
1803}
1804&set_label("const_sse2",64);
1805 &data_word(1<<24,0, 1<<24,0, 1<<24,0, 1<<24,0);
1806 &data_word(0,0, 0,0, 0,0, 0,0);
1807 &data_word(0x03ffffff,0,0x03ffffff,0, 0x03ffffff,0, 0x03ffffff,0);
1808 &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1809}
1810&asciz ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1811&align (4);
1812
1813&asm_finish();
1814
1815close STDOUT or die "error closing STDOUT: $!";