Gitiles
Code Review Sign In
realtimeroboticsgroup.org / RealtimeRoboticsGroup / test / 9a24b370cbc07492fa6c205f1027367fa55b6742 / . / kernel / avx / kernel_sgesc_lib8.S
blob: 43ff7086c5568104b68b60f5da165505e7a5a23f [file] [log] [blame]
Austin Schuh9a24b372018-01-28 16:12:29 -0800[diff] [blame^]1/**************************************************************************************************
2* *
3* This file is part of BLASFEO. *
4* *
5* BLASFEO -- BLAS For Embedded Optimization. *
6* Copyright (C) 2016-2017 by Gianluca Frison. *
7* Developed at IMTEK (University of Freiburg) under the supervision of Moritz Diehl. *
8* All rights reserved. *
9* *
10* HPMPC is free software; you can redistribute it and/or *
11* modify it under the terms of the GNU Lesser General Public *
12* License as published by the Free Software Foundation; either *
13* version 2.1 of the License, or (at your option) any later version. *
14* *
15* HPMPC is distributed in the hope that it will be useful, *
16* but WITHOUT ANY WARRANTY; without even the implied warranty of *
17* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. *
18* See the GNU Lesser General Public License for more details. *
19* *
20* You should have received a copy of the GNU Lesser General Public *
21* License along with HPMPC; if not, write to the Free Software *
22* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA *
23* *
24* Author: Gianluca Frison, giaf (at) dtu.dk *
25* gianluca.frison (at) imtek.uni-freiburg.de *
26* *
27**************************************************************************************************/
28
29
30
31#if defined(OS_LINUX) | defined(OS_MAC)
32
33//#define STACKSIZE 96
34#define STACKSIZE 64
35#define ARG1 %rdi
36#define ARG2 %rsi
37#define ARG3 %rdx
38#define ARG4 %rcx
39#define ARG5 %r8
40#define ARG6 %r9
41#define ARG7 STACKSIZE + 8(%rsp)
42#define ARG8 STACKSIZE + 16(%rsp)
43#define ARG9 STACKSIZE + 24(%rsp)
44#define ARG10 STACKSIZE + 32(%rsp)
45#define ARG11 STACKSIZE + 40(%rsp)
46#define ARG12 STACKSIZE + 48(%rsp)
47#define ARG13 STACKSIZE + 56(%rsp)
48#define ARG14 STACKSIZE + 64(%rsp)
49#define ARG15 STACKSIZE + 72(%rsp)
50#define ARG16 STACKSIZE + 80(%rsp)
51#define ARG17 STACKSIZE + 88(%rsp)
52#define ARG18 STACKSIZE + 96(%rsp)
53#define PROLOGUE \
54 subq $STACKSIZE, %rsp; \
55 movq %rbx, (%rsp); \
56 movq %rbp, 8(%rsp); \
57 movq %r12, 16(%rsp); \
58 movq %r13, 24(%rsp); \
59 movq %r14, 32(%rsp); \
60 movq %r15, 40(%rsp); \
61 vzeroupper;
62#define EPILOGUE \
63 vzeroupper; \
64 movq (%rsp), %rbx; \
65 movq 8(%rsp), %rbp; \
66 movq 16(%rsp), %r12; \
67 movq 24(%rsp), %r13; \
68 movq 32(%rsp), %r14; \
69 movq 40(%rsp), %r15; \
70 addq $STACKSIZE, %rsp;
71
72#elif defined(OS_WINDOWS)
73
74#define STACKSIZE 256
75#define ARG1 %rcx
76#define ARG2 %rdx
77#define ARG3 %r8
78#define ARG4 %r9
79#define ARG5 STACKSIZE + 40(%rsp)
80#define ARG6 STACKSIZE + 48(%rsp)
81#define ARG7 STACKSIZE + 56(%rsp)
82#define ARG8 STACKSIZE + 64(%rsp)
83#define ARG9 STACKSIZE + 72(%rsp)
84#define ARG10 STACKSIZE + 80(%rsp)
85#define ARG11 STACKSIZE + 88(%rsp)
86#define ARG12 STACKSIZE + 96(%rsp)
87#define ARG13 STACKSIZE + 104(%rsp)
88#define ARG14 STACKSIZE + 112(%rsp)
89#define ARG15 STACKSIZE + 120(%rsp)
90#define ARG16 STACKSIZE + 128(%rsp)
91#define ARG17 STACKSIZE + 136(%rsp)
92#define ARG18 STACKSIZE + 144(%rsp)
93#define PROLOGUE \
94 subq $STACKSIZE, %rsp; \
95 movq %rbx, (%rsp); \
96 movq %rbp, 8(%rsp); \
97 movq %r12, 16(%rsp); \
98 movq %r13, 24(%rsp); \
99 movq %r14, 32(%rsp); \
100 movq %r15, 40(%rsp); \
101 movq %rdi, 48(%rsp); \
102 movq %rsi, 56(%rsp); \
103 vmovups %xmm6, 64(%rsp); \
104 vmovups %xmm7, 80(%rsp); \
105 vmovups %xmm8, 96(%rsp); \
106 vmovups %xmm9, 112(%rsp); \
107 vmovups %xmm10, 128(%rsp); \
108 vmovups %xmm11, 144(%rsp); \
109 vmovups %xmm12, 160(%rsp); \
110 vmovups %xmm13, 176(%rsp); \
111 vmovups %xmm14, 192(%rsp); \
112 vmovups %xmm15, 208(%rsp); \
113 vzeroupper;
114#define EPILOGUE \
115 vzeroupper; \
116 movq (%rsp), %rbx; \
117 movq 8(%rsp), %rbp; \
118 movq 16(%rsp), %r12; \
119 movq 24(%rsp), %r13; \
120 movq 32(%rsp), %r14; \
121 movq 40(%rsp), %r15; \
122 movq 48(%rsp), %rdi; \
123 movq 56(%rsp), %rsi; \
124 vmovups 64(%rsp), %xmm6; \
125 vmovups 80(%rsp), %xmm7; \
126 vmovups 96(%rsp), %xmm8; \
127 vmovups 112(%rsp), %xmm9; \
128 vmovups 128(%rsp), %xmm10; \
129 vmovups 144(%rsp), %xmm11; \
130 vmovups 160(%rsp), %xmm12; \
131 vmovups 176(%rsp), %xmm13; \
132 vmovups 192(%rsp), %xmm14; \
133 vmovups 208(%rsp), %xmm15; \
134 addq $STACKSIZE, %rsp;
135
136#else
137
138#error wrong OS
139
140#endif
141
142
143
144#if defined(OS_LINUX) | defined(OS_WINDOWS)
145 .text
146#elif defined(OS_MAC)
147 .section __TEXT,__text,regular,pure_instructions
148#endif
149
150
151
152// subroutine
153//
154// input arguments:
155// r10d <- k
156// r11 <- alpha
157// r12 <- A
158
159#if MACRO_LEVEL>=1
160 .macro INNER_KERNEL_SGESC_8_LIB8
161#else
162 .p2align 4,,15
163#if defined(OS_LINUX)
164 .type inner_kernel_sgesc_8_lib8, @function
165inner_kernel_sgesc_8_lib8:
166#elif defined(OS_MAC)
167_inner_kernel_sgesc_8_lib8:
168#elif defined(OS_WINDOWS)
169 .def inner_kernel_sgesc_8_lib8; .scl 2; .type 32; .endef
170inner_kernel_sgesc_8_lib8:
171#endif
172#endif
173
174 vbroadcastss 0(%r11), %ymm15
175
176 cmpl $3, %r10d
177 jle 0f // consider clean-up
178
179 // main loop
180 .p2align 3
1811: // main loop
182
183 vmovaps 0(%r12), %ymm0
184 vmulps %ymm15, %ymm0, %ymm0
185 vmovaps %ymm0, 0(%r12)
186 subl $4, %r10d
187
188 vmovaps 32(%r12), %ymm0
189 vmulps %ymm15, %ymm0, %ymm0
190 vmovaps %ymm0, 32(%r12)
191
192 vmovaps 64(%r12), %ymm0
193 vmulps %ymm15, %ymm0, %ymm0
194 vmovaps %ymm0, 64(%r12)
195 addq $128, %r12
196
197 vmovaps -32(%r12), %ymm0
198 vmulps %ymm15, %ymm0, %ymm0
199 vmovaps %ymm0, -32(%r12)
200
201 cmpl $4, %r10d
202 jg 1b // main loop
203
2040: // consider clean-up
205 cmpl $0, %r10d
206 jle 2f // return
207
2083: // clean-up loop
209
210 vmovaps 0(%r12), %ymm0
211 vmulps %ymm15, %ymm0, %ymm0
212 vmovaps %ymm0, 0(%r12)
213 subl $1, %r10d
214 addq $32, %r12
215
216 cmpl $0, %r10d
217 jg 3b // clean-up loop
218
2192: // return
220
221#if MACRO_LEVEL>=1
222 .endm
223#else
224 ret
225
226#if defined(OS_LINUX)
227 .size inner_kernel_sgesc_8_lib8, .-inner_kernel_sgesc_8_lib8
228#endif
229#endif
230
231
232
233
234
235// subroutine
236//
237// input arguments:
238// r10d <- k
239// r11 <- alpha
240// r12 <- A
241// r13d <- m1
242
243#if MACRO_LEVEL>=1
244 .macro INNER_KERNEL_SGESC_8_GEN_LIB8
245#else
246 .p2align 4,,15
247#if defined(OS_LINUX)
248 .type inner_kernel_sgesc_8_gen_lib8, @function
249inner_kernel_sgesc_8_gen_lib8:
250#elif defined(OS_MAC)
251_inner_kernel_sgesc_8_gen_lib8:
252#elif defined(OS_WINDOWS)
253 .def inner_kernel_sgesc_8_gen_lib8; .scl 2; .type 32; .endef
254inner_kernel_sgesc_8_gen_lib8:
255#endif
256#endif
257
258 // compute mask for rows
259 vcvtsi2ss %r13d, %xmm15, %xmm15
260#if defined(OS_LINUX) | defined(OS_WINDOWS)
261 vmovups .LC00(%rip), %ymm12
262#elif defined(OS_MAC)
263 vmovups LC00(%rip), %ymm12
264#endif
265 vshufps $0x00, %xmm15, %xmm15, %xmm15
266 vinsertf128 $0x1, %xmm15, %ymm15, %ymm15
267 vsubps %ymm15, %ymm12, %ymm15
268
269 vbroadcastss 0(%r11), %ymm14
270
271 cmpl $3, %r10d
272 jle 0f // consider clean-up
273
274 // main loop
275 .p2align 3
2761: // main loop
277
278 vmovaps 0(%r12), %ymm0
279 vmulps %ymm14, %ymm0, %ymm0
280 vmaskmovps %ymm0, %ymm15, 0(%r12)
281 subl $4, %r10d
282
283 vmovaps 32(%r12), %ymm0
284 vmulps %ymm14, %ymm0, %ymm0
285 vmaskmovps %ymm0, %ymm15, 32(%r12)
286
287 vmovaps 64(%r12), %ymm0
288 vmulps %ymm14, %ymm0, %ymm0
289 vmaskmovps %ymm0, %ymm15, 64(%r12)
290 addq $128, %r12
291
292 vmovaps -32(%r12), %ymm0
293 vmulps %ymm14, %ymm0, %ymm0
294 vmaskmovps %ymm0, %ymm15, -32(%r12)
295
296 cmpl $4, %r10d
297 jg 1b // main loop
298
2990: // consider clean-up
300 cmpl $0, %r10d
301 jle 2f // return
302
3033: // clean-up loop
304
305 vmovaps 0(%r12), %ymm0
306 vmulps %ymm14, %ymm0, %ymm0
307 vmaskmovps %ymm0, %ymm15, 0(%r12)
308 subl $1, %r10d
309 addq $32, %r12
310
311 cmpl $0, %r10d
312 jg 3b // clean-up loop
313
3142: // return
315
316#if MACRO_LEVEL>=1
317 .endm
318#else
319 ret
320
321#if defined(OS_LINUX)
322 .size inner_kernel_sgesc_8_lib8, .-inner_kernel_sgesc_8_lib8
323#endif
324#endif
325
326
327
328
329
330// rdi rsi rdx
331// void kernel_sgesc_8_lib8(int k, float *alpha, float *A);
332
333 .p2align 4,,15
334#if defined(OS_LINUX)
335 .globl kernel_sgesc_8_lib8
336 .type kernel_sgesc_8_lib8, @function
337kernel_sgesc_8_lib8:
338#elif defined(OS_MAC)
339 .globl _kernel_sgesc_8_lib8
340_kernel_sgesc_8_lib8:
341#elif defined(OS_WINDOWS)
342 .globl kernel_sgesc_8_lib8
343 .def kernel_sgesc_8_lib8; .scl 2; .type 32; .endef
344kernel_sgesc_8_lib8:
345#endif
346
347 PROLOGUE
348
349 // call inner dgemm kernel nt
350
351 movq ARG1, %r10 // k
352 movq ARG2, %r11 // alpha
353 movq ARG3, %r12 // A
354
355#if MACRO_LEVEL>=1
356 INNER_KERNEL_SGESC_8_LIB8
357#else
358#if defined(OS_LINUX) | defined(OS_WINDOWS)
359 call inner_kernel_sgesc_8_lib8
360#elif defined(OS_MAC)
361 callq _inner_kernel_sgesc_8_lib8
362#endif
363#endif
364
365 EPILOGUE
366
367 ret
368
369#if defined(OS_LINUX)
370 .size kernel_sgesc_8_lib8, .-kernel_sgesc_8_lib8
371#endif
372
373
374
375
376
377// rdi rsi rdx rcx
378// void kernel_sgecp_8_gen_lib8(int k, float *alpha, float *A, int m1);
379
380 .p2align 4,,15
381#if defined(OS_LINUX)
382 .globl kernel_sgesc_8_gen_lib8
383 .type kernel_sgesc_8_gen_lib8, @function
384kernel_sgesc_8_gen_lib8:
385#elif defined(OS_MAC)
386 .globl _kernel_sgesc_8_gen_lib8
387_kernel_sgesc_8_gen_lib8:
388#elif defined(OS_WINDOWS)
389 .globl kernel_sgesc_8_gen_lib8
390 .def kernel_sgesc_8_gen_lib8; .scl 2; .type 32; .endef
391kernel_sgesc_8_gen_lib8:
392#endif
393
394 PROLOGUE
395
396 // call inner dgemm kernel nt
397
398 movq ARG1, %r10 // k
399 movq ARG2, %r11 // alpha
400 movq ARG3, %r12 // A
401 movq ARG4, %r14 // m1
402
403#if MACRO_LEVEL>=1
404 INNER_KERNEL_SGESC_8_GEN_LIB8
405#else
406#if defined(OS_LINUX) | defined(OS_WINDOWS)
407 call inner_kernel_sgesc_8_gen_lib8
408#elif defined(OS_MAC)
409 callq _inner_kernel_sgesc_8_gen_lib8
410#endif
411#endif
412
413 EPILOGUE
414
415 ret
416
417#if defined(OS_LINUX)
418 .size kernel_sgesc_8_gen_lib8, .-kernel_sgesc_8_gen_lib8
419#endif
420
421
422
423
424
425
426 // read-only data
427#if defined(OS_LINUX)
428 .section .rodata.cst32,"aM",@progbits,32
429#elif defined(OS_MAC)
430 .section __TEXT,__const
431#elif defined(OS_WINDOWS)
432 .section .rdata,"dr"
433#endif
434
435#if defined(OS_LINUX) | defined(OS_WINDOWS)
436 .align 32
437.LC00: // { 7.5 6.5 5.5 4.5 3.5 2.5 1.5 0.5 }
438#elif defined(OS_MAC)
439 .align 5
440LC00: // { 7.5 6.5 5.5 4.5 3.5 2.5 1.5 0.5 }
441#endif
442 .long 1056964608
443 .long 1069547520
444 .long 1075838976
445 .long 1080033280
446 .long 1083179008
447 .long 1085276160
448 .long 1087373312
449 .long 1089470464
450
451#if defined(OS_LINUX) | defined(OS_WINDOWS)
452 .align 32
453.LC01: // { 15.5 14.5 13.5 12.5 11.5 10.5 9.5 8.5 }
454#elif defined(OS_MAC)
455 .align 5
456LC01: // { 15.5 14.5 13.5 12.5 11.5 10.5 9.5 8.5 }
457#endif
458 .long 1091043328
459 .long 1092091904
460 .long 1093140480
461 .long 1094189056
462 .long 1095237632
463 .long 1096286208
464 .long 1097334784
465 .long 1098383360
466
467#if defined(OS_LINUX) | defined(OS_WINDOWS)
468 .align 32
469.LC02: // { 23.5 22.5 21.5 20.5 19.5 18.5 17.5 16.5 }
470#elif defined(OS_MAC)
471 .align 5
472LC02: // { 23.5 22.5 21.5 20.5 19.5 18.5 17.5 16.5 }
473#endif
474 .long 1099169792
475 .long 1099694080
476 .long 1100218368
477 .long 1100742656
478 .long 1101266944
479 .long 1101791232
480 .long 1102315520
481 .long 1102839808
482
483#if defined(OS_LINUX) | defined(OS_WINDOWS)
484 .align 32
485.LC03: // { -1.0 -1.0 1.0 1.0 1.0 1.0 1.0 1.0 }
486#elif defined(OS_MAC)
487 .align 5
488LC03: // { -1.0 -1.0 1.0 1.0 1.0 1.0 1.0 1.0 }
489#endif
490 .long 1065353216
491 .long 1065353216
492 .long 1065353216
493 .long 1065353216
494 .long 1065353216
495 .long 1065353216
496 .long 3212836864
497 .long 3212836864
498
499
500
501#if defined(OS_LINUX)
502 .section .note.GNU-stack,"",@progbits
503#elif defined(OS_MAC)
504 .subsections_via_symbols
505#endif
506