arm.inc 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017
  1. {
  2. This file is part of the Free Pascal run time library.
  3. Copyright (c) 2003 by the Free Pascal development team.
  4. Processor dependent implementation for the system unit for
  5. ARM
  6. See the file COPYING.FPC, included in this distribution,
  7. for details about the copyright.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11. **********************************************************************}
  12. {$asmmode gas}
  13. {$ifndef FPC_SYSTEM_HAS_MOVE}
  14. {$define FPC_SYSTEM_FPC_MOVE}
  15. {$endif FPC_SYSTEM_HAS_MOVE}
  16. {$ifdef FPC_SYSTEM_FPC_MOVE}
  17. const
  18. cpu_has_edsp : boolean = false;
  19. in_edsp_test : boolean = false;
  20. {$endif FPC_SYSTEM_FPC_MOVE}
  21. {$if not(defined(wince)) and not(defined(gba)) and not(defined(nds)) and not(defined(FPUSOFT)) and not(defined(FPULIBGCC))}
  22. {$define FPC_SYSTEM_HAS_SYSINITFPU}
  23. {$if not defined(darwin) and not defined(FPUVFPV2) and not defined(FPUVFPV3) and not defined(FPUVFPV3_D16)}
  24. Procedure SysInitFPU;{$ifdef SYSTEMINLINE}inline;{$endif}
  25. begin
  26. { Enable FPU exceptions, but disable INEXACT, UNDERFLOW, DENORMAL }
  27. asm
  28. rfs r0
  29. and r0,r0,#0xffe0ffff
  30. orr r0,r0,#0x00070000
  31. wfs r0
  32. end;
  33. end;
  34. {$else}
  35. Procedure SysInitFPU;{$ifdef SYSTEMINLINE}inline;{$endif}
  36. begin
  37. { Enable FPU exceptions, but disable INEXACT, UNDERFLOW, DENORMAL }
  38. asm
  39. fmrx r0,fpscr
  40. // set "round to nearest" mode
  41. and r0,r0,#0xff3fffff
  42. // mask "exception happened" and overflow flags
  43. and r0,r0,#0xffffff20
  44. // mask exception flags
  45. and r0,r0,#0xffff40ff
  46. {$ifndef darwin}
  47. // Floating point exceptions cause kernel panics on iPhoneOS 2.2.1...
  48. // disable flush-to-zero mode (IEEE math compliant)
  49. and r0,r0,#0xfeffffff
  50. // enable invalid operation, div-by-zero and overflow exceptions
  51. orr r0,r0,#0x00000700
  52. {$endif}
  53. fmxr fpscr,r0
  54. end;
  55. end;
  56. {$endif}
  57. {$endif}
  58. procedure fpc_cpuinit;
  59. begin
  60. { don't let libraries influence the FPU cw set by the host program }
  61. if not IsLibrary then
  62. SysInitFPU;
  63. end;
  64. {$ifdef wince}
  65. function _controlfp(new: DWORD; mask: DWORD): DWORD; cdecl; external 'coredll';
  66. {$define FPC_SYSTEM_HAS_SYSRESETFPU}
  67. Procedure SysResetFPU;{$ifdef SYSTEMINLINE}inline;{$endif}
  68. begin
  69. softfloat_exception_flags:=0;
  70. end;
  71. {$define FPC_SYSTEM_HAS_SYSINITFPU}
  72. Procedure SysInitFPU;{$ifdef SYSTEMINLINE}inline;{$endif}
  73. begin
  74. softfloat_exception_mask:=float_flag_underflow or float_flag_inexact or float_flag_denormal;
  75. { Enable FPU exceptions, but disable INEXACT, UNDERFLOW, DENORMAL }
  76. { FPU precision 64 bit, rounding to nearest, affine infinity }
  77. _controlfp($000C0003, $030F031F);
  78. end;
  79. {$endif wince}
  80. {****************************************************************************
  81. stack frame related stuff
  82. ****************************************************************************}
  83. {$IFNDEF INTERNAL_BACKTRACE}
  84. {$define FPC_SYSTEM_HAS_GET_FRAME}
  85. function get_frame:pointer;assembler;nostackframe;
  86. asm
  87. {$ifndef darwin}
  88. mov r0,r11
  89. {$else}
  90. mov r0,r7
  91. {$endif}
  92. end;
  93. {$ENDIF not INTERNAL_BACKTRACE}
  94. {$define FPC_SYSTEM_HAS_GET_CALLER_ADDR}
  95. function get_caller_addr(framebp:pointer;addr:pointer=nil):pointer;assembler;nostackframe;
  96. asm
  97. cmp r0,#0
  98. {$ifndef darwin}
  99. ldrne r0,[r0,#-4]
  100. {$else}
  101. ldrne r0,[r0,#4]
  102. {$endif}
  103. end;
  104. {$define FPC_SYSTEM_HAS_GET_CALLER_FRAME}
  105. function get_caller_frame(framebp:pointer;addr:pointer=nil):pointer;assembler;nostackframe;
  106. asm
  107. cmp r0,#0
  108. {$ifndef darwin}
  109. ldrne r0,[r0,#-12]
  110. {$else}
  111. ldrne r0,[r0]
  112. {$endif}
  113. end;
  114. {$define FPC_SYSTEM_HAS_SPTR}
  115. Function Sptr : pointer;assembler;nostackframe;
  116. asm
  117. mov r0,sp
  118. end;
  119. {$ifndef FPC_SYSTEM_HAS_FILLCHAR}
  120. {$define FPC_SYSTEM_HAS_FILLCHAR}
  121. Procedure FillChar(var x;count:longint;value:byte);assembler;nostackframe;
  122. asm
  123. // less than 0?
  124. cmp r1,#0
  125. {$ifdef CPUARM_HAS_BX}
  126. bxle lr
  127. {$else}
  128. movle pc,lr
  129. {$endif}
  130. mov r3,r0
  131. orr r2,r2,r2,lsl #8
  132. orr r2,r2,r2,lsl #16
  133. tst r3, #3 // Aligned?
  134. bne .LFillchar_do_align
  135. .LFillchar_is_aligned:
  136. subs r1,r1,#8
  137. bmi .LFillchar_less_than_8bytes
  138. mov ip,r2
  139. .LFillchar_at_least_8bytes:
  140. // Do 16 bytes per loop
  141. // More unrolling is uncessary, as we'll just stall on the write buffers
  142. stmia r3!,{r2,ip}
  143. subs r1,r1,#8
  144. stmplia r3!,{r2,ip}
  145. subpls r1,r1,#8
  146. bpl .LFillchar_at_least_8bytes
  147. .LFillchar_less_than_8bytes:
  148. // Do the rest
  149. adds r1, r1, #8
  150. {$ifdef CPUARM_HAS_BX}
  151. bxeq lr
  152. {$else}
  153. moveq pc,lr
  154. {$endif}
  155. tst r1, #4
  156. strne r2,[r3],#4
  157. tst r1, #2
  158. strneh r2,[r3],#2
  159. tst r1, #1
  160. strneb r2,[r3],#1
  161. {$ifdef CPUARM_HAS_BX}
  162. bx lr
  163. {$else}
  164. mov pc,lr
  165. {$endif}
  166. // Special case for unaligned start
  167. // We make a maximum of 3 loops here
  168. .LFillchar_do_align:
  169. strb r2,[r3],#1
  170. subs r1, r1, #1
  171. {$ifdef CPUARM_HAS_BX}
  172. bxeq lr
  173. {$else}
  174. moveq pc,lr
  175. {$endif}
  176. tst r3,#3
  177. bne .LFillchar_do_align
  178. b .LFillchar_is_aligned
  179. end;
  180. {$endif FPC_SYSTEM_HAS_FILLCHAR}
  181. {$ifndef FPC_SYSTEM_HAS_MOVE}
  182. {$define FPC_SYSTEM_HAS_MOVE}
  183. {$ifdef CPUARM_HAS_EDSP}
  184. procedure Move(const source;var dest;count:longint);[public, alias: 'FPC_MOVE'];assembler;nostackframe;
  185. {$else CPUARM_HAS_EDSP}
  186. procedure Move_pld(const source;var dest;count:longint);assembler;nostackframe;
  187. {$endif CPUARM_HAS_EDSP}
  188. asm
  189. pld [r0]
  190. // count <=0 ?
  191. cmp r2,#0
  192. {$ifdef CPUARM_HAS_BX}
  193. bxle lr
  194. {$else}
  195. movle pc,lr
  196. {$endif}
  197. // overlap?
  198. subs r3, r1, r0 // if (dest > source) and
  199. cmphi r2, r3 // (count > dest - src) then
  200. bhi .Loverlapped // DoReverseByteCopy;
  201. cmp r2,#8 // if (count < 8) then
  202. blt .Lbyteloop // DoForwardByteCopy;
  203. // Any way to avoid the above jump and fuse the next two instructions?
  204. tst r0, #3 // if (source and 3) <> 0 or
  205. tsteq r1, #3 // (dest and 3) <> 0 then
  206. bne .Lbyteloop // DoForwardByteCopy;
  207. pld [r0,#32]
  208. .Ldwordloop:
  209. ldmia r0!, {r3, ip}
  210. // preload
  211. pld [r0,#64]
  212. sub r2,r2,#8
  213. cmp r2, #8
  214. stmia r1!, {r3, ip}
  215. bge .Ldwordloop
  216. cmp r2,#0
  217. {$ifdef CPUARM_HAS_BX}
  218. bxeq lr
  219. {$else}
  220. moveq pc,lr
  221. {$endif}
  222. .Lbyteloop:
  223. subs r2,r2,#1
  224. ldrb r3,[r0],#1
  225. strb r3,[r1],#1
  226. bne .Lbyteloop
  227. {$ifdef CPUARM_HAS_BX}
  228. bx lr
  229. {$else}
  230. mov pc,lr
  231. {$endif}
  232. .Loverlapped:
  233. subs r2,r2,#1
  234. ldrb r3,[r0,r2]
  235. strb r3,[r1,r2]
  236. bne .Loverlapped
  237. end;
  238. {$ifndef CPUARM_HAS_EDSP}
  239. procedure Move_blended(const source;var dest;count:longint);assembler;nostackframe;
  240. asm
  241. // count <=0 ?
  242. cmp r2,#0
  243. {$ifdef CPUARM_HAS_BX}
  244. bxle lr
  245. {$else}
  246. movle pc,lr
  247. {$endif}
  248. // overlap?
  249. subs r3, r1, r0 // if (dest > source) and
  250. cmphi r2, r3 // (count > dest - src) then
  251. bhi .Loverlapped // DoReverseByteCopy;
  252. cmp r2,#8 // if (count < 8) then
  253. blt .Lbyteloop // DoForwardByteCopy;
  254. // Any way to avoid the above jump and fuse the next two instructions?
  255. tst r0, #3 // if (source and 3) <> 0 or
  256. tsteq r1, #3 // (dest and 3) <> 0 then
  257. bne .Lbyteloop // DoForwardByteCopy;
  258. .Ldwordloop:
  259. ldmia r0!, {r3, ip}
  260. sub r2,r2,#8
  261. cmp r2, #8
  262. stmia r1!, {r3, ip}
  263. bge .Ldwordloop
  264. cmp r2,#0
  265. {$ifdef CPUARM_HAS_BX}
  266. bxeq lr
  267. {$else}
  268. moveq pc,lr
  269. {$endif}
  270. .Lbyteloop:
  271. subs r2,r2,#1
  272. ldrb r3,[r0],#1
  273. strb r3,[r1],#1
  274. bne .Lbyteloop
  275. {$ifdef CPUARM_HAS_BX}
  276. bx lr
  277. {$else}
  278. mov pc,lr
  279. {$endif}
  280. .Loverlapped:
  281. subs r2,r2,#1
  282. ldrb r3,[r0,r2]
  283. strb r3,[r1,r2]
  284. bne .Loverlapped
  285. end;
  286. const
  287. moveproc : pointer = @move_blended;
  288. procedure Move(const source;var dest;count:longint);[public, alias: 'FPC_MOVE'];assembler;nostackframe;
  289. asm
  290. ldr ip,.Lmoveproc
  291. ldr pc,[ip]
  292. .Lmoveproc:
  293. .long moveproc
  294. end;
  295. {$endif CPUARM_HAS_EDSP}
  296. {$endif FPC_SYSTEM_HAS_MOVE}
  297. {****************************************************************************
  298. String
  299. ****************************************************************************}
  300. {$ifndef FPC_SYSTEM_HAS_FPC_SHORTSTR_ASSIGN}
  301. {$define FPC_SYSTEM_HAS_FPC_SHORTSTR_ASSIGN}
  302. {$ifndef FPC_STRTOSHORTSTRINGPROC}
  303. function fpc_shortstr_to_shortstr(len:longint;const sstr:shortstring):shortstring;assembler;nostackframe;[public,alias: 'FPC_SHORTSTR_TO_SHORTSTR'];compilerproc;
  304. {$else}
  305. procedure fpc_shortstr_to_shortstr(out res:shortstring;const sstr:shortstring);assembler;nostackframe;[public,alias: 'FPC_SHORTSTR_TO_SHORTSTR'];compilerproc;
  306. {$endif}
  307. {r0: __RESULT
  308. r1: len
  309. r2: sstr}
  310. asm
  311. ldrb r12,[r2],#1
  312. cmp r12,r1
  313. movgt r12,r1
  314. strb r12,[r0],#1
  315. cmp r12,#6 (* 6 seems to be the break even point. *)
  316. blt .LStartTailCopy
  317. (* Align destination on 32bits. This is the only place where unrolling
  318. really seems to help, since in the common case, sstr is aligned on
  319. 32 bits, therefore in the common case we need to copy 3 bytes to
  320. align, i.e. in the case of a loop, you wouldn't branch out early.*)
  321. rsb r3,r0,#0
  322. ands r3,r3,#3
  323. sub r12,r12,r3
  324. ldrneb r1,[r2],#1
  325. strneb r1,[r0],#1
  326. subnes r3,r3,#1
  327. ldrneb r1,[r2],#1
  328. strneb r1,[r0],#1
  329. subnes r3,r3,#1
  330. ldrneb r1,[r2],#1
  331. strneb r1,[r0],#1
  332. subnes r3,r3,#1
  333. .LDoneAlign:
  334. (* Destination should be aligned now, but source might not be aligned,
  335. if this is the case, do a byte-per-byte copy. *)
  336. tst r2,#3
  337. bne .LStartTailCopy
  338. (* Start the main copy, 32 bit at a time. *)
  339. movs r3,r12,lsr #2
  340. and r12,r12,#3
  341. beq .LStartTailCopy
  342. .LNext4bytes:
  343. (* Unrolling this loop would save a little bit of time for long strings
  344. (>20 chars), but alas, it hurts for short strings and they are the
  345. common case.*)
  346. ldrne r1,[r2],#4
  347. strne r1,[r0],#4
  348. subnes r3,r3,#1
  349. bne .LNext4bytes
  350. .LStartTailCopy:
  351. (* Do remaining bytes. *)
  352. cmp r12,#0
  353. beq .LDoneTail
  354. .LNextChar3:
  355. ldrb r1,[r2],#1
  356. strb r1,[r0],#1
  357. subs r12,r12,#1
  358. bne .LNextChar3
  359. .LDoneTail:
  360. end;
  361. procedure fpc_shortstr_assign(len:longint;sstr,dstr:pointer);assembler;nostackframe;[public,alias:'FPC_SHORTSTR_ASSIGN'];compilerproc;
  362. {r0: len
  363. r1: sstr
  364. r2: dstr}
  365. asm
  366. ldrb r12,[r1],#1
  367. cmp r12,r0
  368. movgt r12,r0
  369. strb r12,[r2],#1
  370. cmp r12,#6 (* 6 seems to be the break even point. *)
  371. blt .LStartTailCopy
  372. (* Align destination on 32bits. This is the only place where unrolling
  373. really seems to help, since in the common case, sstr is aligned on
  374. 32 bits, therefore in the common case we need to copy 3 bytes to
  375. align, i.e. in the case of a loop, you wouldn't branch out early.*)
  376. rsb r3,r2,#0
  377. ands r3,r3,#3
  378. sub r12,r12,r3
  379. ldrneb r0,[r1],#1
  380. strneb r0,[r2],#1
  381. subnes r3,r3,#1
  382. ldrneb r0,[r1],#1
  383. strneb r0,[r2],#1
  384. subnes r3,r3,#1
  385. ldrneb r0,[r1],#1
  386. strneb r0,[r2],#1
  387. subnes r3,r3,#1
  388. .LDoneAlign:
  389. (* Destination should be aligned now, but source might not be aligned,
  390. if this is the case, do a byte-per-byte copy. *)
  391. tst r1,#3
  392. bne .LStartTailCopy
  393. (* Start the main copy, 32 bit at a time. *)
  394. movs r3,r12,lsr #2
  395. and r12,r12,#3
  396. beq .LStartTailCopy
  397. .LNext4bytes:
  398. (* Unrolling this loop would save a little bit of time for long strings
  399. (>20 chars), but alas, it hurts for short strings and they are the
  400. common case.*)
  401. ldrne r0,[r1],#4
  402. strne r0,[r2],#4
  403. subnes r3,r3,#1
  404. bne .LNext4bytes
  405. .LStartTailCopy:
  406. (* Do remaining bytes. *)
  407. cmp r12,#0
  408. beq .LDoneTail
  409. .LNextChar3:
  410. ldrb r0,[r1],#1
  411. strb r0,[r2],#1
  412. subs r12,r12,#1
  413. bne .LNextChar3
  414. .LDoneTail:
  415. end;
  416. {$endif FPC_SYSTEM_HAS_FPC_SHORTSTR_ASSIGN}
  417. {$ifndef FPC_SYSTEM_HAS_FPC_PCHAR_LENGTH}
  418. {$define FPC_SYSTEM_HAS_FPC_PCHAR_LENGTH}
  419. function fpc_Pchar_length(p:Pchar):sizeint;assembler;nostackframe;[public,alias:'FPC_PCHAR_LENGTH'];compilerproc;
  420. asm
  421. cmp r0,#0
  422. mov r1,r0
  423. beq .Ldone
  424. .Lnextchar:
  425. (*Are we aligned?*)
  426. tst r1,#3
  427. bne .Ltest_unaligned (*No, do byte per byte.*)
  428. ldr r3,.L01010101
  429. .Ltest_aligned:
  430. (*Aligned, load 4 bytes at a time.*)
  431. ldr r12,[r1],#4
  432. (*Check wether r12 contains a 0 byte.*)
  433. sub r2,r12,r3
  434. mvn r12,r12
  435. and r2,r2,r12
  436. ands r2,r2,r3,lsl #7 (*r3 lsl 7 = $80808080*)
  437. beq .Ltest_aligned (*No 0 byte, repeat.*)
  438. sub r1,r1,#4
  439. .Ltest_unaligned:
  440. ldrb r12,[r1],#1
  441. cmp r12,#1 (*r12<1 same as r12=0, but result in carry flag*)
  442. bcs .Lnextchar
  443. (*Dirty trick: we need to subtract 1 extra because we have counted the
  444. terminating 0, due to the known carry flag sbc can do this.*)
  445. sbc r0,r1,r0
  446. .Ldone:
  447. {$ifdef CPUARM_HAS_BX}
  448. bx lr
  449. {$else}
  450. mov pc,lr
  451. {$endif}
  452. .L01010101:
  453. .long 0x01010101
  454. end;
  455. {$endif}
  456. {$ifndef darwin}
  457. {$define FPC_SYSTEM_HAS_ANSISTR_DECR_REF}
  458. function fpc_freemem_x(p:pointer):ptrint; [external name 'FPC_FREEMEM_X'];
  459. Procedure fpc_ansistr_decr_ref (Var S : Pointer); [Public,Alias:'FPC_ANSISTR_DECR_REF'];assembler;nostackframe; compilerproc;
  460. asm
  461. ldr r1, [r0]
  462. // On return the pointer will always be set to zero, so utilize the delay slots
  463. mov r2, #0
  464. str r2, [r0]
  465. // Check for a zero string
  466. cmp r1, #0
  467. // Load reference counter
  468. ldrne r2, [r1, #-8]
  469. {$ifdef CPUARM_HAS_BX}
  470. bxeq lr
  471. {$else}
  472. moveq pc,lr
  473. {$endif}
  474. // Check for a constant string
  475. cmp r2, #0
  476. {$ifdef CPUARM_HAS_BX}
  477. bxlt lr
  478. {$else}
  479. movlt pc,lr
  480. {$endif}
  481. stmfd sp!, {r1, lr}
  482. sub r0, r1, #8
  483. {$ifdef CPUARM_HAS_BLX}
  484. blx InterLockedDecrement
  485. {$else}
  486. bl InterLockedDecrement
  487. {$endif}
  488. // InterLockedDecrement is a nice guy and sets the z flag for us
  489. // if the reference count dropped to 0
  490. ldmnefd sp!, {r1, pc}
  491. ldmfd sp!, {r0, lr}
  492. // We currently can not use constant symbols in ARM-Assembly
  493. // but we need to stay backward compatible with 2.6
  494. sub r0, r0, #12
  495. // Jump without a link, so freemem directly returns to our caller
  496. b FPC_FREEMEM_X
  497. end;
  498. {$endif not darwin}
  499. var
  500. fpc_system_lock: longint; export name 'fpc_system_lock';
  501. function InterLockedDecrement (var Target: longint) : longint; assembler; nostackframe;
  502. asm
  503. {$ifdef CPUARM_HAS_LDREX}
  504. .Lloop:
  505. ldrex r1, [r0]
  506. sub r1, r1, #1
  507. strex r2, r1, [r0]
  508. cmp r2, #0
  509. bne .Lloop
  510. movs r0, r1
  511. bx lr
  512. {$else}
  513. {$ifdef SYSTEM_HAS_KUSER_CMPXCHG}
  514. stmfd r13!, {lr}
  515. mov r2, r0 // kuser_cmpxchg does not clobber r2 by definition
  516. .Latomic_dec_loop:
  517. ldr r0, [r2] // Load the current value
  518. // We expect this to work without looping most of the time
  519. // R3 gets clobbered in kuser_cmpxchg so in the unlikely case that we have to
  520. // loop here again, we have to reload the value. Normaly this just fills the
  521. // load stall-cycles from the above ldr so in reality we'll not get any additional
  522. // delays because of this
  523. // Don't use ldr to load r3 to avoid cacheline trashing
  524. // Load 0xffff0fff into r3 and substract to 0xffff0fc0,
  525. // the kuser_cmpxchg entry point
  526. mvn r3, #0x0000f000
  527. sub r3, r3, #0x3F
  528. sub r1, r0, #1 // Decrement value
  529. blx r3 // Call kuser_cmpxchg, sets C-Flag on success
  530. // MOVS sets the Z flag when the result reaches zero, this can be used later on
  531. // The C-Flag will not be modified by this because we're not doing any shifting
  532. movcss r0, r1 // We expect that to work most of the time so keep it pipeline friendly
  533. ldmcsfd r13!, {pc}
  534. b .Latomic_dec_loop // kuser_cmpxchg sets C flag on error
  535. {$else}
  536. // lock
  537. ldr r3, .Lfpc_system_lock
  538. mov r1, #1
  539. .Lloop:
  540. swp r2, r1, [r3]
  541. cmp r2, #0
  542. bne .Lloop
  543. // do the job
  544. ldr r1, [r0]
  545. sub r1, r1, #1
  546. str r1, [r0]
  547. movs r0, r1
  548. // unlock and return
  549. str r2, [r3]
  550. bx lr
  551. .Lfpc_system_lock:
  552. .long fpc_system_lock
  553. {$endif}
  554. {$endif}
  555. end;
  556. {$ifndef darwin}
  557. {$define FPC_SYSTEM_HAS_ANSISTR_INCR_REF}
  558. Procedure fpc_ansistr_incr_ref (S : Pointer); [Public,Alias:'FPC_ANSISTR_INCR_REF'];assembler;nostackframe; compilerproc;
  559. asm
  560. // Null string?
  561. cmp r0, #0
  562. // Load reference counter
  563. ldrne r1, [r0, #-8]
  564. // pointer to counter, calculate here for delay slot utilization
  565. subne r0, r0, #8
  566. {$ifdef CPUARM_HAS_BX}
  567. bxeq lr
  568. {$else}
  569. moveq pc,lr
  570. {$endif}
  571. // Check for a constant string
  572. cmp r1, #0
  573. // Tailcall
  574. // Hopefully the linker will place InterLockedIncrement as layed out here
  575. bge InterLockedIncrement
  576. // Freepascal will generate a proper return here, save some cachespace
  577. end;
  578. {$endif not darwin}
  579. function InterLockedIncrement (var Target: longint) : longint; assembler; nostackframe;
  580. asm
  581. {$ifdef CPUARM_HAS_LDREX}
  582. .Lloop:
  583. ldrex r1, [r0]
  584. add r1, r1, #1
  585. strex r2, r1, [r0]
  586. cmp r2, #0
  587. bne .Lloop
  588. mov r0, r1
  589. bx lr
  590. {$else}
  591. {$ifdef SYSTEM_HAS_KUSER_CMPXCHG}
  592. stmfd r13!, {lr}
  593. mov r2, r0 // kuser_cmpxchg does not clobber r2 by definition
  594. .Latomic_inc_loop:
  595. ldr r0, [r2] // Load the current value
  596. // We expect this to work without looping most of the time
  597. // R3 gets clobbered in kuser_cmpxchg so in the unlikely case that we have to
  598. // loop here again, we have to reload the value. Normaly this just fills the
  599. // load stall-cycles from the above ldr so in reality we'll not get any additional
  600. // delays because of this
  601. // Don't use ldr to load r3 to avoid cacheline trashing
  602. // Load 0xffff0fff into r3 and substract to 0xffff0fc0,
  603. // the kuser_cmpxchg entry point
  604. mvn r3, #0x0000f000
  605. sub r3, r3, #0x3F
  606. add r1, r0, #1 // Increment value
  607. blx r3 // Call kuser_cmpxchg, sets C-Flag on success
  608. movcs r0, r1 // We expect that to work most of the time so keep it pipeline friendly
  609. ldmcsfd r13!, {pc}
  610. b .Latomic_inc_loop // kuser_cmpxchg sets C flag on error
  611. {$else}
  612. // lock
  613. ldr r3, .Lfpc_system_lock
  614. mov r1, #1
  615. .Lloop:
  616. swp r2, r1, [r3]
  617. cmp r2, #0
  618. bne .Lloop
  619. // do the job
  620. ldr r1, [r0]
  621. add r1, r1, #1
  622. str r1, [r0]
  623. mov r0, r1
  624. // unlock and return
  625. str r2, [r3]
  626. bx lr
  627. .Lfpc_system_lock:
  628. .long fpc_system_lock
  629. {$endif}
  630. {$endif}
  631. end;
  632. function InterLockedExchange (var Target: longint;Source : longint) : longint; assembler; nostackframe;
  633. asm
  634. {$ifdef CPUARM_HAS_LDREX}
  635. // swp is deprecated on ARMv6 and above
  636. .Lloop:
  637. ldrex r2, [r0]
  638. strex r3, r1, [r0]
  639. cmp r3, #0
  640. bne .Lloop
  641. mov r0, r2
  642. bx lr
  643. {$else}
  644. {$ifdef SYSTEM_HAS_KUSER_CMPXCHG}
  645. stmfd r13!, {r4, lr}
  646. mov r2, r0 // kuser_cmpxchg does not clobber r2 (and r1) by definition
  647. .Latomic_add_loop:
  648. ldr r0, [r2] // Load the current value
  649. // We expect this to work without looping most of the time
  650. // R3 gets clobbered in kuser_cmpxchg so in the unlikely case that we have to
  651. // loop here again, we have to reload the value. Normaly this just fills the
  652. // load stall-cycles from the above ldr so in reality we'll not get any additional
  653. // delays because of this
  654. // Don't use ldr to load r3 to avoid cacheline trashing
  655. // Load 0xffff0fff into r3 and substract to 0xffff0fc0,
  656. // the kuser_cmpxchg entry point
  657. mvn r3, #0x0000f000
  658. sub r3, r3, #0x3F
  659. mov r4, r0 // save the current value because kuser_cmpxchg clobbers r0
  660. blx r3 // Call kuser_cmpxchg, sets C-Flag on success
  661. // restore the original value if needed
  662. movcs r0, r4
  663. ldmcsfd r13!, {r4, pc}
  664. b .Latomic_add_loop // kuser_cmpxchg failed, loop back
  665. {$else}
  666. // lock
  667. ldr r3, .Lfpc_system_lock
  668. mov r2, #1
  669. .Lloop:
  670. swp r2, r2, [r3]
  671. cmp r2, #0
  672. bne .Lloop
  673. // do the job
  674. ldr r2, [r0]
  675. str r1, [r0]
  676. mov r0, r2
  677. // unlock and return
  678. mov r2, #0
  679. str r2, [r3]
  680. bx lr
  681. .Lfpc_system_lock:
  682. .long fpc_system_lock
  683. {$endif}
  684. {$endif}
  685. end;
  686. function InterLockedExchangeAdd (var Target: longint;Source : longint) : longint; assembler; nostackframe;
  687. asm
  688. {$ifdef CPUARM_HAS_LDREX}
  689. .Lloop:
  690. ldrex r2, [r0]
  691. add r12, r1, r2
  692. strex r3, r12, [r0]
  693. cmp r3, #0
  694. bne .Lloop
  695. mov r0, r2
  696. bx lr
  697. {$else}
  698. {$ifdef SYSTEM_HAS_KUSER_CMPXCHG}
  699. stmfd r13!, {r4, lr}
  700. mov r2, r0 // kuser_cmpxchg does not clobber r2 by definition
  701. mov r4, r1 // Save addend
  702. .Latomic_add_loop:
  703. ldr r0, [r2] // Load the current value
  704. // We expect this to work without looping most of the time
  705. // R3 gets clobbered in kuser_cmpxchg so in the unlikely case that we have to
  706. // loop here again, we have to reload the value. Normaly this just fills the
  707. // load stall-cycles from the above ldr so in reality we'll not get any additional
  708. // delays because of this
  709. // Don't use ldr to load r3 to avoid cacheline trashing
  710. // Load 0xffff0fff into r3 and substract to 0xffff0fc0,
  711. // the kuser_cmpxchg entry point
  712. mvn r3, #0x0000f000
  713. sub r3, r3, #0x3F
  714. add r1, r0, r4 // Add to value
  715. blx r3 // Call kuser_cmpxchg, sets C-Flag on success
  716. // r1 does not get clobbered, so just get back the original value
  717. // Otherwise we would have to allocate one more register and store the
  718. // temporary value
  719. subcs r0, r1, r4
  720. ldmcsfd r13!, {r4, pc}
  721. b .Latomic_add_loop // kuser_cmpxchg failed, loop back
  722. {$else}
  723. // lock
  724. ldr r3, .Lfpc_system_lock
  725. mov r2, #1
  726. .Lloop:
  727. swp r2, r2, [r3]
  728. cmp r2, #0
  729. bne .Lloop
  730. // do the job
  731. ldr r2, [r0]
  732. add r1, r1, r2
  733. str r1, [r0]
  734. mov r0, r2
  735. // unlock and return
  736. mov r2, #0
  737. str r2, [r3]
  738. bx lr
  739. .Lfpc_system_lock:
  740. .long fpc_system_lock
  741. {$endif}
  742. {$endif}
  743. end;
  744. function InterlockedCompareExchange(var Target: longint; NewValue: longint; Comperand: longint): longint; assembler; nostackframe;
  745. asm
  746. {$ifdef CPUARM_HAS_LDREX}
  747. .Lloop:
  748. ldrex r3, [r0]
  749. mov r12, #0
  750. cmp r3, r2
  751. strexeq r12, r1, [r0]
  752. cmp r12, #0
  753. bne .Lloop
  754. mov r0, r3
  755. bx lr
  756. {$else}
  757. {$ifdef SYSTEM_HAS_KUSER_CMPXCHG}
  758. stmfd r13!, {r4, lr}
  759. mvn r3, #0x0000f000
  760. sub r3, r3, #0x3F
  761. mov r4, r2 // Swap parameters around
  762. mov r2, r0
  763. mov r0, r4 // Use r4 because we'll need the new value for later
  764. // r1 and r2 will not be clobbered by kuser_cmpxchg
  765. // If we have to loop, r0 will be set to the original Comperand
  766. .Linterlocked_compare_exchange_loop:
  767. blx r3 // Call kuser_cmpxchg sets C-Flag on success
  768. movcs r0, r4 // Return the previous value on success
  769. ldmcsfd r13!, {r4, pc}
  770. // The error case is a bit tricky, kuser_cmpxchg does not return the current value
  771. // So we may need to loop to avoid race conditions
  772. // The loop case is HIGHLY unlikely, it would require that we got rescheduled between
  773. // calling kuser_cmpxchg and the ldr. While beeing rescheduled another process/thread
  774. // would have the set the value to our comperand
  775. ldr r0, [r2] // Load the currently set value
  776. cmp r0, r4 // Return if Comperand != current value, otherwise loop again
  777. ldmnefd r13!, {r4, pc}
  778. // If we need to loop here, we have to
  779. b .Linterlocked_compare_exchange_loop
  780. {$else}
  781. // lock
  782. ldr r12, .Lfpc_system_lock
  783. mov r3, #1
  784. .Lloop:
  785. swp r3, r3, [r12]
  786. cmp r3, #0
  787. bne .Lloop
  788. // do the job
  789. ldr r3, [r0]
  790. cmp r3, r2
  791. streq r1, [r0]
  792. mov r0, r3
  793. // unlock and return
  794. mov r3, #0
  795. str r3, [r12]
  796. bx lr
  797. .Lfpc_system_lock:
  798. .long fpc_system_lock
  799. {$endif}
  800. {$endif}
  801. end;
  802. {$define FPC_SYSTEM_HAS_DECLOCKED_LONGINT}
  803. function declocked(var l: longint) : boolean; inline;
  804. begin
  805. Result:=InterLockedDecrement(l) = 0;
  806. end;
  807. {$define FPC_SYSTEM_HAS_INCLOCKED_LONGINT}
  808. procedure inclocked(var l: longint); inline;
  809. begin
  810. InterLockedIncrement(l);
  811. end;
  812. procedure fpc_cpucodeinit;
  813. begin
  814. {$ifdef FPC_SYSTEM_FPC_MOVE}
  815. {$ifndef CPUARM_HAS_EDSP}
  816. cpu_has_edsp:=true;
  817. in_edsp_test:=true;
  818. asm
  819. bic r0,sp,#7
  820. ldrd r0,[r0]
  821. end;
  822. in_edsp_test:=false;
  823. if cpu_has_edsp then
  824. moveproc:=@move_pld
  825. else
  826. moveproc:=@move_blended;
  827. {$else CPUARM_HAS_EDSP}
  828. cpu_has_edsp:=true;
  829. {$endif CPUARM_HAS_EDSP}
  830. {$endif FPC_SYSTEM_FPC_MOVE}
  831. end;
  832. {$define FPC_SYSTEM_HAS_SWAPENDIAN}
  833. { SwapEndian(<16 Bit>) being inlined is faster than using assembler }
  834. function SwapEndian(const AValue: SmallInt): SmallInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  835. begin
  836. { the extra Word type cast is necessary because the "AValue shr 8" }
  837. { is turned into "longint(AValue) shr 8", so if AValue < 0 then }
  838. { the sign bits from the upper 16 bits are shifted in rather than }
  839. { zeroes. }
  840. Result := SmallInt((Word(AValue) shr 8) or (Word(AValue) shl 8));
  841. end;
  842. function SwapEndian(const AValue: Word): Word;{$ifdef SYSTEMINLINE}inline;{$endif}
  843. begin
  844. Result := Word((AValue shr 8) or (AValue shl 8));
  845. end;
  846. (*
  847. This is kept for reference. Thats what the compiler COULD generate in these cases.
  848. But FPC currently does not support inlining of asm-functions, so the whole call-overhead
  849. is bigger than the gain of the optimized function.
  850. function AsmSwapEndian(const AValue: SmallInt): SmallInt;{$ifdef SYSTEMINLINE}inline;{$endif};assembler;nostackframe;
  851. asm
  852. // We're starting with 4321
  853. {$if defined(CPUARM_HAS_REV)}
  854. rev r0, r0 // Reverse byteorder r0 = 1234
  855. mov r0, r0, shr #16 // Shift down to 16bits r0 = 0012
  856. {$else}
  857. mov r0, r0, shl #16 // Shift to make that 2100
  858. mov r0, r0, ror #24 // Rotate to 1002
  859. orr r0, r0, r0 shr #16 // Shift and combine into 0012
  860. {$endif}
  861. end;
  862. *)
  863. {
  864. These used to be an assembler-function, but with newer improvements to the compiler this
  865. generates a perfect 4 cycle code sequence and can be inlined.
  866. }
  867. function SwapEndian(const AValue: LongWord): LongWord;{$ifdef SYSTEMINLINE}inline;{$endif}
  868. begin
  869. Result:= AValue xor rordword(AValue,16);
  870. Result:= Result and $FF00FFFF;
  871. Result:= (Result shr 8) xor rordword(AValue,8);
  872. end;
  873. function SwapEndian(const AValue: LongInt): LongInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  874. begin
  875. Result:=LongInt(SwapEndian(DWord(AValue)));
  876. end;
  877. {
  878. Currently freepascal will not generate a good assembler sequence for
  879. Result:=(SwapEndian(longword(lo(AValue))) shl 32) or
  880. (SwapEndian(longword(hi(AValue))));
  881. So we keep an assembly version for now
  882. }
  883. function SwapEndian(const AValue: Int64): Int64; assembler; nostackframe;
  884. asm
  885. // fpc >2.6.0 adds the "rev" instruction in the internal assembler
  886. {$if defined(CPUARM_HAS_REV)}
  887. rev r2, r0
  888. rev r0, r1
  889. mov r1, r2
  890. {$else}
  891. mov ip, r1
  892. // We're starting with r0 = $87654321
  893. eor r1, r0, r0, ror #16 // r1 = $C444C444
  894. bic r1, r1, #16711680 // r1 = r1 and $ff00ffff = $C400C444
  895. mov r0, r0, ror #8 // r0 = $21876543
  896. eor r1, r0, r1, lsr #8 // r1 = $21436587
  897. eor r0, ip, ip, ror #16
  898. bic r0, r0, #16711680
  899. mov ip, ip, ror #8
  900. eor r0, ip, r0, lsr #8
  901. {$endif}
  902. end;
  903. function SwapEndian(const AValue: QWord): QWord; {$ifdef SYSTEMINLINE}inline;{$endif}
  904. begin
  905. Result:=QWord(SwapEndian(Int64(AValue)));
  906. end;
  907. {include hand-optimized assembler division code}
  908. { $i divide.inc}