sum_arm.s 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. // Copyright 2015 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build arm,!gccgo,!appengine,!nacl
  5. #include "textflag.h"
  6. // This code was translated into a form compatible with 5a from the public
  7. // domain source by Andrew Moon: github.com/floodyberry/poly1305-opt/blob/master/app/extensions/poly1305.
  8. DATA ·poly1305_init_constants_armv6<>+0x00(SB)/4, $0x3ffffff
  9. DATA ·poly1305_init_constants_armv6<>+0x04(SB)/4, $0x3ffff03
  10. DATA ·poly1305_init_constants_armv6<>+0x08(SB)/4, $0x3ffc0ff
  11. DATA ·poly1305_init_constants_armv6<>+0x0c(SB)/4, $0x3f03fff
  12. DATA ·poly1305_init_constants_armv6<>+0x10(SB)/4, $0x00fffff
  13. GLOBL ·poly1305_init_constants_armv6<>(SB), 8, $20
  14. // Warning: the linker may use R11 to synthesize certain instructions. Please
  15. // take care and verify that no synthetic instructions use it.
  16. TEXT poly1305_init_ext_armv6<>(SB), NOSPLIT, $0
  17. // Needs 16 bytes of stack and 64 bytes of space pointed to by R0. (It
  18. // might look like it's only 60 bytes of space but the final four bytes
  19. // will be written by another function.) We need to skip over four
  20. // bytes of stack because that's saving the value of 'g'.
  21. ADD $4, R13, R8
  22. MOVM.IB [R4-R7], (R8)
  23. MOVM.IA.W (R1), [R2-R5]
  24. MOVW $·poly1305_init_constants_armv6<>(SB), R7
  25. MOVW R2, R8
  26. MOVW R2>>26, R9
  27. MOVW R3>>20, g
  28. MOVW R4>>14, R11
  29. MOVW R5>>8, R12
  30. ORR R3<<6, R9, R9
  31. ORR R4<<12, g, g
  32. ORR R5<<18, R11, R11
  33. MOVM.IA (R7), [R2-R6]
  34. AND R8, R2, R2
  35. AND R9, R3, R3
  36. AND g, R4, R4
  37. AND R11, R5, R5
  38. AND R12, R6, R6
  39. MOVM.IA.W [R2-R6], (R0)
  40. EOR R2, R2, R2
  41. EOR R3, R3, R3
  42. EOR R4, R4, R4
  43. EOR R5, R5, R5
  44. EOR R6, R6, R6
  45. MOVM.IA.W [R2-R6], (R0)
  46. MOVM.IA.W (R1), [R2-R5]
  47. MOVM.IA [R2-R6], (R0)
  48. ADD $20, R13, R0
  49. MOVM.DA (R0), [R4-R7]
  50. RET
  51. #define MOVW_UNALIGNED(Rsrc, Rdst, Rtmp, offset) \
  52. MOVBU (offset+0)(Rsrc), Rtmp; \
  53. MOVBU Rtmp, (offset+0)(Rdst); \
  54. MOVBU (offset+1)(Rsrc), Rtmp; \
  55. MOVBU Rtmp, (offset+1)(Rdst); \
  56. MOVBU (offset+2)(Rsrc), Rtmp; \
  57. MOVBU Rtmp, (offset+2)(Rdst); \
  58. MOVBU (offset+3)(Rsrc), Rtmp; \
  59. MOVBU Rtmp, (offset+3)(Rdst)
  60. TEXT poly1305_blocks_armv6<>(SB), NOSPLIT, $0
  61. // Needs 24 bytes of stack for saved registers and then 88 bytes of
  62. // scratch space after that. We assume that 24 bytes at (R13) have
  63. // already been used: four bytes for the link register saved in the
  64. // prelude of poly1305_auth_armv6, four bytes for saving the value of g
  65. // in that function and 16 bytes of scratch space used around
  66. // poly1305_finish_ext_armv6_skip1.
  67. ADD $24, R13, R12
  68. MOVM.IB [R4-R8, R14], (R12)
  69. MOVW R0, 88(R13)
  70. MOVW R1, 92(R13)
  71. MOVW R2, 96(R13)
  72. MOVW R1, R14
  73. MOVW R2, R12
  74. MOVW 56(R0), R8
  75. WORD $0xe1180008 // TST R8, R8 not working see issue 5921
  76. EOR R6, R6, R6
  77. MOVW.EQ $(1<<24), R6
  78. MOVW R6, 84(R13)
  79. ADD $116, R13, g
  80. MOVM.IA (R0), [R0-R9]
  81. MOVM.IA [R0-R4], (g)
  82. CMP $16, R12
  83. BLO poly1305_blocks_armv6_done
  84. poly1305_blocks_armv6_mainloop:
  85. WORD $0xe31e0003 // TST R14, #3 not working see issue 5921
  86. BEQ poly1305_blocks_armv6_mainloop_aligned
  87. ADD $100, R13, g
  88. MOVW_UNALIGNED(R14, g, R0, 0)
  89. MOVW_UNALIGNED(R14, g, R0, 4)
  90. MOVW_UNALIGNED(R14, g, R0, 8)
  91. MOVW_UNALIGNED(R14, g, R0, 12)
  92. MOVM.IA (g), [R0-R3]
  93. ADD $16, R14
  94. B poly1305_blocks_armv6_mainloop_loaded
  95. poly1305_blocks_armv6_mainloop_aligned:
  96. MOVM.IA.W (R14), [R0-R3]
  97. poly1305_blocks_armv6_mainloop_loaded:
  98. MOVW R0>>26, g
  99. MOVW R1>>20, R11
  100. MOVW R2>>14, R12
  101. MOVW R14, 92(R13)
  102. MOVW R3>>8, R4
  103. ORR R1<<6, g, g
  104. ORR R2<<12, R11, R11
  105. ORR R3<<18, R12, R12
  106. BIC $0xfc000000, R0, R0
  107. BIC $0xfc000000, g, g
  108. MOVW 84(R13), R3
  109. BIC $0xfc000000, R11, R11
  110. BIC $0xfc000000, R12, R12
  111. ADD R0, R5, R5
  112. ADD g, R6, R6
  113. ORR R3, R4, R4
  114. ADD R11, R7, R7
  115. ADD $116, R13, R14
  116. ADD R12, R8, R8
  117. ADD R4, R9, R9
  118. MOVM.IA (R14), [R0-R4]
  119. MULLU R4, R5, (R11, g)
  120. MULLU R3, R5, (R14, R12)
  121. MULALU R3, R6, (R11, g)
  122. MULALU R2, R6, (R14, R12)
  123. MULALU R2, R7, (R11, g)
  124. MULALU R1, R7, (R14, R12)
  125. ADD R4<<2, R4, R4
  126. ADD R3<<2, R3, R3
  127. MULALU R1, R8, (R11, g)
  128. MULALU R0, R8, (R14, R12)
  129. MULALU R0, R9, (R11, g)
  130. MULALU R4, R9, (R14, R12)
  131. MOVW g, 76(R13)
  132. MOVW R11, 80(R13)
  133. MOVW R12, 68(R13)
  134. MOVW R14, 72(R13)
  135. MULLU R2, R5, (R11, g)
  136. MULLU R1, R5, (R14, R12)
  137. MULALU R1, R6, (R11, g)
  138. MULALU R0, R6, (R14, R12)
  139. MULALU R0, R7, (R11, g)
  140. MULALU R4, R7, (R14, R12)
  141. ADD R2<<2, R2, R2
  142. ADD R1<<2, R1, R1
  143. MULALU R4, R8, (R11, g)
  144. MULALU R3, R8, (R14, R12)
  145. MULALU R3, R9, (R11, g)
  146. MULALU R2, R9, (R14, R12)
  147. MOVW g, 60(R13)
  148. MOVW R11, 64(R13)
  149. MOVW R12, 52(R13)
  150. MOVW R14, 56(R13)
  151. MULLU R0, R5, (R11, g)
  152. MULALU R4, R6, (R11, g)
  153. MULALU R3, R7, (R11, g)
  154. MULALU R2, R8, (R11, g)
  155. MULALU R1, R9, (R11, g)
  156. ADD $52, R13, R0
  157. MOVM.IA (R0), [R0-R7]
  158. MOVW g>>26, R12
  159. MOVW R4>>26, R14
  160. ORR R11<<6, R12, R12
  161. ORR R5<<6, R14, R14
  162. BIC $0xfc000000, g, g
  163. BIC $0xfc000000, R4, R4
  164. ADD.S R12, R0, R0
  165. ADC $0, R1, R1
  166. ADD.S R14, R6, R6
  167. ADC $0, R7, R7
  168. MOVW R0>>26, R12
  169. MOVW R6>>26, R14
  170. ORR R1<<6, R12, R12
  171. ORR R7<<6, R14, R14
  172. BIC $0xfc000000, R0, R0
  173. BIC $0xfc000000, R6, R6
  174. ADD R14<<2, R14, R14
  175. ADD.S R12, R2, R2
  176. ADC $0, R3, R3
  177. ADD R14, g, g
  178. MOVW R2>>26, R12
  179. MOVW g>>26, R14
  180. ORR R3<<6, R12, R12
  181. BIC $0xfc000000, g, R5
  182. BIC $0xfc000000, R2, R7
  183. ADD R12, R4, R4
  184. ADD R14, R0, R0
  185. MOVW R4>>26, R12
  186. BIC $0xfc000000, R4, R8
  187. ADD R12, R6, R9
  188. MOVW 96(R13), R12
  189. MOVW 92(R13), R14
  190. MOVW R0, R6
  191. CMP $32, R12
  192. SUB $16, R12, R12
  193. MOVW R12, 96(R13)
  194. BHS poly1305_blocks_armv6_mainloop
  195. poly1305_blocks_armv6_done:
  196. MOVW 88(R13), R12
  197. MOVW R5, 20(R12)
  198. MOVW R6, 24(R12)
  199. MOVW R7, 28(R12)
  200. MOVW R8, 32(R12)
  201. MOVW R9, 36(R12)
  202. ADD $48, R13, R0
  203. MOVM.DA (R0), [R4-R8, R14]
  204. RET
  205. #define MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp) \
  206. MOVBU.P 1(Rsrc), Rtmp; \
  207. MOVBU.P Rtmp, 1(Rdst); \
  208. MOVBU.P 1(Rsrc), Rtmp; \
  209. MOVBU.P Rtmp, 1(Rdst)
  210. #define MOVWP_UNALIGNED(Rsrc, Rdst, Rtmp) \
  211. MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp); \
  212. MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp)
  213. // func poly1305_auth_armv6(out *[16]byte, m *byte, mlen uint32, key *[32]key)
  214. TEXT ·poly1305_auth_armv6(SB), $196-16
  215. // The value 196, just above, is the sum of 64 (the size of the context
  216. // structure) and 132 (the amount of stack needed).
  217. //
  218. // At this point, the stack pointer (R13) has been moved down. It
  219. // points to the saved link register and there's 196 bytes of free
  220. // space above it.
  221. //
  222. // The stack for this function looks like:
  223. //
  224. // +---------------------
  225. // |
  226. // | 64 bytes of context structure
  227. // |
  228. // +---------------------
  229. // |
  230. // | 112 bytes for poly1305_blocks_armv6
  231. // |
  232. // +---------------------
  233. // | 16 bytes of final block, constructed at
  234. // | poly1305_finish_ext_armv6_skip8
  235. // +---------------------
  236. // | four bytes of saved 'g'
  237. // +---------------------
  238. // | lr, saved by prelude <- R13 points here
  239. // +---------------------
  240. MOVW g, 4(R13)
  241. MOVW out+0(FP), R4
  242. MOVW m+4(FP), R5
  243. MOVW mlen+8(FP), R6
  244. MOVW key+12(FP), R7
  245. ADD $136, R13, R0 // 136 = 4 + 4 + 16 + 112
  246. MOVW R7, R1
  247. // poly1305_init_ext_armv6 will write to the stack from R13+4, but
  248. // that's ok because none of the other values have been written yet.
  249. BL poly1305_init_ext_armv6<>(SB)
  250. BIC.S $15, R6, R2
  251. BEQ poly1305_auth_armv6_noblocks
  252. ADD $136, R13, R0
  253. MOVW R5, R1
  254. ADD R2, R5, R5
  255. SUB R2, R6, R6
  256. BL poly1305_blocks_armv6<>(SB)
  257. poly1305_auth_armv6_noblocks:
  258. ADD $136, R13, R0
  259. MOVW R5, R1
  260. MOVW R6, R2
  261. MOVW R4, R3
  262. MOVW R0, R5
  263. MOVW R1, R6
  264. MOVW R2, R7
  265. MOVW R3, R8
  266. AND.S R2, R2, R2
  267. BEQ poly1305_finish_ext_armv6_noremaining
  268. EOR R0, R0
  269. ADD $8, R13, R9 // 8 = offset to 16 byte scratch space
  270. MOVW R0, (R9)
  271. MOVW R0, 4(R9)
  272. MOVW R0, 8(R9)
  273. MOVW R0, 12(R9)
  274. WORD $0xe3110003 // TST R1, #3 not working see issue 5921
  275. BEQ poly1305_finish_ext_armv6_aligned
  276. WORD $0xe3120008 // TST R2, #8 not working see issue 5921
  277. BEQ poly1305_finish_ext_armv6_skip8
  278. MOVWP_UNALIGNED(R1, R9, g)
  279. MOVWP_UNALIGNED(R1, R9, g)
  280. poly1305_finish_ext_armv6_skip8:
  281. WORD $0xe3120004 // TST $4, R2 not working see issue 5921
  282. BEQ poly1305_finish_ext_armv6_skip4
  283. MOVWP_UNALIGNED(R1, R9, g)
  284. poly1305_finish_ext_armv6_skip4:
  285. WORD $0xe3120002 // TST $2, R2 not working see issue 5921
  286. BEQ poly1305_finish_ext_armv6_skip2
  287. MOVHUP_UNALIGNED(R1, R9, g)
  288. B poly1305_finish_ext_armv6_skip2
  289. poly1305_finish_ext_armv6_aligned:
  290. WORD $0xe3120008 // TST R2, #8 not working see issue 5921
  291. BEQ poly1305_finish_ext_armv6_skip8_aligned
  292. MOVM.IA.W (R1), [g-R11]
  293. MOVM.IA.W [g-R11], (R9)
  294. poly1305_finish_ext_armv6_skip8_aligned:
  295. WORD $0xe3120004 // TST $4, R2 not working see issue 5921
  296. BEQ poly1305_finish_ext_armv6_skip4_aligned
  297. MOVW.P 4(R1), g
  298. MOVW.P g, 4(R9)
  299. poly1305_finish_ext_armv6_skip4_aligned:
  300. WORD $0xe3120002 // TST $2, R2 not working see issue 5921
  301. BEQ poly1305_finish_ext_armv6_skip2
  302. MOVHU.P 2(R1), g
  303. MOVH.P g, 2(R9)
  304. poly1305_finish_ext_armv6_skip2:
  305. WORD $0xe3120001 // TST $1, R2 not working see issue 5921
  306. BEQ poly1305_finish_ext_armv6_skip1
  307. MOVBU.P 1(R1), g
  308. MOVBU.P g, 1(R9)
  309. poly1305_finish_ext_armv6_skip1:
  310. MOVW $1, R11
  311. MOVBU R11, 0(R9)
  312. MOVW R11, 56(R5)
  313. MOVW R5, R0
  314. ADD $8, R13, R1
  315. MOVW $16, R2
  316. BL poly1305_blocks_armv6<>(SB)
  317. poly1305_finish_ext_armv6_noremaining:
  318. MOVW 20(R5), R0
  319. MOVW 24(R5), R1
  320. MOVW 28(R5), R2
  321. MOVW 32(R5), R3
  322. MOVW 36(R5), R4
  323. MOVW R4>>26, R12
  324. BIC $0xfc000000, R4, R4
  325. ADD R12<<2, R12, R12
  326. ADD R12, R0, R0
  327. MOVW R0>>26, R12
  328. BIC $0xfc000000, R0, R0
  329. ADD R12, R1, R1
  330. MOVW R1>>26, R12
  331. BIC $0xfc000000, R1, R1
  332. ADD R12, R2, R2
  333. MOVW R2>>26, R12
  334. BIC $0xfc000000, R2, R2
  335. ADD R12, R3, R3
  336. MOVW R3>>26, R12
  337. BIC $0xfc000000, R3, R3
  338. ADD R12, R4, R4
  339. ADD $5, R0, R6
  340. MOVW R6>>26, R12
  341. BIC $0xfc000000, R6, R6
  342. ADD R12, R1, R7
  343. MOVW R7>>26, R12
  344. BIC $0xfc000000, R7, R7
  345. ADD R12, R2, g
  346. MOVW g>>26, R12
  347. BIC $0xfc000000, g, g
  348. ADD R12, R3, R11
  349. MOVW $-(1<<26), R12
  350. ADD R11>>26, R12, R12
  351. BIC $0xfc000000, R11, R11
  352. ADD R12, R4, R9
  353. MOVW R9>>31, R12
  354. SUB $1, R12
  355. AND R12, R6, R6
  356. AND R12, R7, R7
  357. AND R12, g, g
  358. AND R12, R11, R11
  359. AND R12, R9, R9
  360. MVN R12, R12
  361. AND R12, R0, R0
  362. AND R12, R1, R1
  363. AND R12, R2, R2
  364. AND R12, R3, R3
  365. AND R12, R4, R4
  366. ORR R6, R0, R0
  367. ORR R7, R1, R1
  368. ORR g, R2, R2
  369. ORR R11, R3, R3
  370. ORR R9, R4, R4
  371. ORR R1<<26, R0, R0
  372. MOVW R1>>6, R1
  373. ORR R2<<20, R1, R1
  374. MOVW R2>>12, R2
  375. ORR R3<<14, R2, R2
  376. MOVW R3>>18, R3
  377. ORR R4<<8, R3, R3
  378. MOVW 40(R5), R6
  379. MOVW 44(R5), R7
  380. MOVW 48(R5), g
  381. MOVW 52(R5), R11
  382. ADD.S R6, R0, R0
  383. ADC.S R7, R1, R1
  384. ADC.S g, R2, R2
  385. ADC.S R11, R3, R3
  386. MOVM.IA [R0-R3], (R8)
  387. MOVW R5, R12
  388. EOR R0, R0, R0
  389. EOR R1, R1, R1
  390. EOR R2, R2, R2
  391. EOR R3, R3, R3
  392. EOR R4, R4, R4
  393. EOR R5, R5, R5
  394. EOR R6, R6, R6
  395. EOR R7, R7, R7
  396. MOVM.IA.W [R0-R7], (R12)
  397. MOVM.IA [R0-R7], (R12)
  398. MOVW 4(R13), g
  399. RET