cgcpu.pas 108 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693
  1. {
  2. Copyright (c) 2014 by Jonas Maebe
  3. This unit implements the code generator for AArch64
  4. This program is free software; you can redistribute it and/or modify
  5. it under the terms of the GNU General Public License as published by
  6. the Free Software Foundation; either version 2 of the License, or
  7. (at your option) any later version.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. GNU General Public License for more details.
  12. You should have received a copy of the GNU General Public License
  13. along with this program; if not, write to the Free Software
  14. Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  15. ****************************************************************************
  16. }
  17. unit cgcpu;
  18. {$i fpcdefs.inc}
  19. interface
  20. uses
  21. globtype,parabase,
  22. cgbase,cgutils,cgobj,
  23. aasmbase,aasmtai,aasmdata,aasmcpu,
  24. cpubase,cpuinfo,
  25. node,symconst,SymType,symdef,
  26. rgcpu;
  27. type
  28. { tcgaarch64 }
  29. tcgaarch64=class(tcg)
  30. protected
  31. { changes register size without adding register allocation info }
  32. function makeregsize(reg: tregister; size: tcgsize): tregister; overload;
  33. public
  34. { simplifies "ref" so it can be used with "op". If "ref" can be used
  35. with a different load/Store operation that has the same meaning as the
  36. original one, "op" will be replaced with the alternative }
  37. procedure make_simple_ref(list:TAsmList; var op: tasmop; size: tcgsize; oppostfix: toppostfix; var ref: treference; preferred_newbasereg: tregister);
  38. function getfpuregister(list: TAsmList; size: Tcgsize): Tregister; override;
  39. procedure handle_reg_imm12_reg(list: TAsmList; op: Tasmop; size: tcgsize; src: tregister; a: tcgint; dst: tregister; tmpreg: tregister; setflags, usedest: boolean);
  40. procedure init_register_allocators;override;
  41. procedure done_register_allocators;override;
  42. function getmmregister(list:TAsmList;size:tcgsize):tregister;override;
  43. function handle_load_store(list:TAsmList; op: tasmop; size: tcgsize; oppostfix: toppostfix; reg: tregister; ref: treference):treference;
  44. procedure a_call_name(list:TAsmList;const s:string; weak: boolean);override;
  45. procedure a_call_reg(list:TAsmList;Reg:tregister);override;
  46. { General purpose instructions }
  47. procedure maybeadjustresult(list: TAsmList; op: topcg; size: tcgsize; dst: tregister);
  48. procedure a_op_const_reg(list: TAsmList; op: topcg; size: tcgsize; a: tcgint; reg: tregister);override;
  49. procedure a_op_reg_reg(list: TAsmList; op: topcg; size: tcgsize; src, dst: tregister);override;
  50. procedure a_op_const_reg_reg(list: TAsmList; op: topcg; size: tcgsize; a: tcgint; src, dst: tregister);override;
  51. procedure a_op_reg_reg_reg(list: TAsmList; op: topcg; size: tcgsize; src1, src2, dst: tregister);override;
  52. procedure a_op_const_reg_reg_checkoverflow(list: TAsmList; op: topcg; size: tcgsize; a: tcgint; src, dst: tregister; setflags : boolean; var ovloc : tlocation);override;
  53. procedure a_op_reg_reg_reg_checkoverflow(list: TAsmList; op: topcg; size: tcgsize; src1, src2, dst: tregister; setflags : boolean; var ovloc : tlocation);override;
  54. { move instructions }
  55. procedure a_load_const_reg(list: TAsmList; size: tcgsize; a: tcgint; reg: tregister);override;
  56. procedure a_load_const_ref(list: TAsmList; size: tcgsize; a: tcgint; const ref: treference); override;
  57. procedure a_load_reg_ref(list: TAsmList; fromsize, tosize: tcgsize; reg: tregister;const ref: TReference);override;
  58. procedure a_load_reg_ref_unaligned(list: TAsmList; fromsize, tosize: tcgsize; register: tregister; const ref: treference); override;
  59. procedure a_load_ref_reg(list: TAsmList; fromsize, tosize: tcgsize; const ref: TReference; reg: tregister);override;
  60. procedure a_load_ref_reg_unaligned(list: TAsmList; fromsize, tosize: tcgsize; const ref: treference; register: tregister); override;
  61. procedure a_load_reg_reg(list: TAsmList; fromsize, tosize: tcgsize; reg1, reg2: tregister);override;
  62. procedure a_loadaddr_ref_reg(list: TAsmList; const ref: TReference; r: tregister);override;
  63. { fpu move instructions (not used, all floating point is vector unit-based) }
  64. procedure a_loadfpu_reg_reg(list: TAsmList; fromsize, tosize: tcgsize; reg1, reg2: tregister); override;
  65. procedure a_loadfpu_ref_reg(list: TAsmList; fromsize, tosize: tcgsize; const ref: treference; reg: tregister); override;
  66. procedure a_loadfpu_reg_ref(list: TAsmList; fromsize, tosize: tcgsize; reg: tregister; const ref: treference); override;
  67. procedure a_loadmm_reg_reg(list: TAsmList; fromsize, tosize: tcgsize; reg1, reg2: tregister;shuffle : pmmshuffle);override;
  68. procedure a_loadmm_ref_reg(list: TAsmList; fromsize, tosize: tcgsize; const ref: TReference; reg: tregister; shuffle: pmmshuffle);override;
  69. procedure a_loadmm_reg_ref(list: TAsmList; fromsize, tosize: tcgsize; reg: tregister; const ref: TReference; shuffle: pmmshuffle);override;
  70. procedure a_loadmm_intreg_reg(list: TAsmList; fromsize, tosize: tcgsize; intreg, mmreg: tregister; shuffle: pmmshuffle); override;
  71. procedure a_loadmm_reg_intreg(list: TAsmList; fromsize, tosize: tcgsize; mmreg, intreg: tregister; shuffle: pmmshuffle); override;
  72. procedure a_opmm_reg_reg(list: TAsmList; Op: TOpCG; size: tcgsize; src, dst: tregister; shuffle: pmmshuffle); override;
  73. procedure a_bit_scan_reg_reg(list: TAsmList; reverse: boolean; srcsize, dstsize: tcgsize; src, dst: TRegister); override;
  74. { comparison operations }
  75. procedure a_cmp_const_reg_label(list: TAsmList; size: tcgsize; cmp_op: topcmp; a: tcgint; reg: tregister; l: tasmlabel);override;
  76. procedure a_cmp_reg_reg_label(list: TAsmList; size: tcgsize; cmp_op: topcmp; reg1, reg2: tregister; l: tasmlabel);override;
  77. procedure a_jmp_always(list: TAsmList; l: TAsmLabel);override;
  78. procedure a_jmp_name(list: TAsmList; const s: string);override;
  79. procedure a_jmp_cond(list: TAsmList; cond: TOpCmp; l: tasmlabel);{ override;}
  80. procedure a_jmp_flags(list: TAsmList; const f: tresflags; l: tasmlabel);override;
  81. procedure g_flags2reg(list: TAsmList; size: tcgsize; const f:tresflags; reg: tregister);override;
  82. procedure g_overflowcheck(list: TAsmList; const loc: tlocation; def: tdef);override;
  83. procedure g_overflowcheck_loc(list: TAsmList; const loc: tlocation; def: tdef; ovloc: tlocation);override;
  84. procedure g_stackpointer_alloc(list: TAsmList; localsize: longint);override;
  85. procedure g_proc_entry(list: TAsmList; localsize: longint; nostackframe: boolean);override;
  86. procedure g_proc_exit(list: TAsmList; parasize: longint; nostackframe: boolean);override;
  87. procedure g_maybe_got_init(list: TAsmList); override;
  88. procedure g_restore_registers(list: TAsmList);override;
  89. procedure g_save_registers(list: TAsmList);override;
  90. procedure g_concatcopy_move(list: TAsmList; const source, dest: treference; len: tcgint);
  91. procedure g_concatcopy(list: TAsmList; const source, dest: treference; len: tcgint);override;
  92. procedure g_adjust_self_value(list: TAsmList; procdef: tprocdef; ioffset: tcgint);override;
  93. procedure g_check_for_fpu_exception(list: TAsmList; force, clear: boolean);override;
  94. procedure g_profilecode(list: TAsmList);override;
  95. private
  96. function save_regs(list: TAsmList; rt: tregistertype; lowsr, highsr: tsuperregister; sub: tsubregister): longint;
  97. procedure load_regs(list: TAsmList; rt: tregistertype; lowsr, highsr: tsuperregister; sub: tsubregister);
  98. end;
  99. procedure create_codegen;
  100. const
  101. TOpCG2AsmOpReg: array[topcg] of TAsmOp = (
  102. A_NONE,A_MOV,A_ADD,A_AND,A_UDIV,A_SDIV,A_MUL,A_MUL,A_NEG,A_MVN,A_ORR,A_ASRV,A_LSLV,A_LSRV,A_SUB,A_EOR,A_NONE,A_RORV
  103. );
  104. TOpCG2AsmOpImm: array[topcg] of TAsmOp = (
  105. A_NONE,A_MOV,A_ADD,A_AND,A_UDIV,A_SDIV,A_MUL,A_MUL,A_NEG,A_MVN,A_ORR,A_ASR,A_LSL,A_LSR,A_SUB,A_EOR,A_NONE,A_ROR
  106. );
  107. TOpCmp2AsmCond: array[topcmp] of TAsmCond = (C_NONE,C_EQ,C_GT,
  108. C_LT,C_GE,C_LE,C_NE,C_LS,C_CC,C_CS,C_HI
  109. );
  110. winstackpagesize = 4096;
  111. implementation
  112. uses
  113. globals,verbose,systems,cutils,cclasses,
  114. paramgr,fmodule,
  115. symtable,symsym,
  116. tgobj,
  117. ncgutil,
  118. procinfo,cpupi;
  119. procedure tcgaarch64.make_simple_ref(list:TAsmList; var op: tasmop; size: tcgsize; oppostfix: toppostfix; var ref: treference; preferred_newbasereg: tregister);
  120. var
  121. href: treference;
  122. so: tshifterop;
  123. accesssize: longint;
  124. begin
  125. if (ref.base=NR_NO) then
  126. begin
  127. if ref.shiftmode<>SM_None then
  128. internalerror(2014110701);
  129. ref.base:=ref.index;
  130. ref.index:=NR_NO;
  131. end;
  132. { no abitrary scale factor support (the generic code doesn't set it,
  133. AArch-specific code shouldn't either) }
  134. if not(ref.scalefactor in [0,1]) then
  135. internalerror(2014111002);
  136. case simple_ref_type(op,size,oppostfix,ref) of
  137. sr_simple:
  138. exit;
  139. sr_internal_illegal:
  140. internalerror(2014121702);
  141. sr_complex:
  142. { continue } ;
  143. end;
  144. if assigned(ref.symbol) then
  145. begin
  146. { internal "load symbol" instructions should already be valid }
  147. if assigned(ref.symboldata) or
  148. (ref.refaddr in [addr_pic,addr_gotpage,addr_gotpageoffset,addr_page,addr_pageoffset]) then
  149. internalerror(2014110802);
  150. { no relative symbol support (needed) yet }
  151. if assigned(ref.relsymbol) then
  152. internalerror(2014111001);
  153. { loading a symbol address (whether it's in the GOT or not) consists
  154. of two parts: first load the page on which it is located, then
  155. either the offset in the page or load the value at that offset in
  156. the page. This final GOT-load can be relaxed by the linker in case
  157. the variable itself can be stored directly in the GOT }
  158. if (preferred_newbasereg=NR_NO) or
  159. (ref.base=preferred_newbasereg) or
  160. (ref.index=preferred_newbasereg) then
  161. preferred_newbasereg:=getaddressregister(list);
  162. { load the (GOT) page }
  163. reference_reset_symbol(href,ref.symbol,0,8,[]);
  164. if ((ref.symbol.typ in [AT_FUNCTION,AT_LABEL]) and
  165. (ref.symbol.bind in [AB_LOCAL,AB_GLOBAL])) or
  166. ((ref.symbol.typ=AT_DATA) and
  167. (ref.symbol.bind=AB_LOCAL)) or
  168. (target_info.system=system_aarch64_win64) then
  169. href.refaddr:=addr_page
  170. else
  171. href.refaddr:=addr_gotpage;
  172. list.concat(taicpu.op_reg_ref(A_ADRP,preferred_newbasereg,href));
  173. { load the GOT entry (= address of the variable) }
  174. reference_reset_base(href,preferred_newbasereg,0,ctempposinvalid,sizeof(pint),[]);
  175. href.symbol:=ref.symbol;
  176. { code symbols defined in the current compilation unit do not
  177. have to be accessed via the GOT }
  178. if ((ref.symbol.typ in [AT_FUNCTION,AT_LABEL]) and
  179. (ref.symbol.bind in [AB_LOCAL,AB_GLOBAL])) or
  180. ((ref.symbol.typ=AT_DATA) and
  181. (ref.symbol.bind=AB_LOCAL)) or
  182. (target_info.system=system_aarch64_win64) then
  183. begin
  184. href.base:=NR_NO;
  185. href.refaddr:=addr_pageoffset;
  186. list.concat(taicpu.op_reg_reg_ref(A_ADD,preferred_newbasereg,preferred_newbasereg,href));
  187. end
  188. else
  189. begin
  190. href.refaddr:=addr_gotpageoffset;
  191. { use a_load_ref_reg() rather than directly encoding the LDR,
  192. so that we'll check the validity of the reference }
  193. a_load_ref_reg(list,OS_ADDR,OS_ADDR,href,preferred_newbasereg);
  194. end;
  195. { set as new base register }
  196. if ref.base=NR_NO then
  197. ref.base:=preferred_newbasereg
  198. else if ref.index=NR_NO then
  199. ref.index:=preferred_newbasereg
  200. else
  201. begin
  202. { make sure it's valid in case ref.base is SP -> make it
  203. the second operand}
  204. a_op_reg_reg_reg(list,OP_ADD,OS_ADDR,preferred_newbasereg,ref.base,preferred_newbasereg);
  205. ref.base:=preferred_newbasereg
  206. end;
  207. ref.symbol:=nil;
  208. end;
  209. { base & index }
  210. if (ref.base<>NR_NO) and
  211. (ref.index<>NR_NO) then
  212. begin
  213. case op of
  214. A_LDR, A_STR:
  215. begin
  216. if (ref.shiftmode=SM_None) and
  217. (ref.shiftimm<>0) then
  218. internalerror(2014110805);
  219. { wrong shift? (possible in case of something like
  220. array_of_2byte_rec[x].bytefield -> shift will be set 1, but
  221. the final load is a 1 byte -> can't use shift after all }
  222. if (ref.shiftmode in [SM_LSL,SM_UXTW,SM_SXTW]) and
  223. ((ref.shiftimm<>BsfDWord(tcgsizep2size[size])) or
  224. (ref.offset<>0)) then
  225. begin
  226. if preferred_newbasereg=NR_NO then
  227. preferred_newbasereg:=getaddressregister(list);
  228. { "add" supports a superset of the shift modes supported by
  229. load/store instructions }
  230. shifterop_reset(so);
  231. so.shiftmode:=ref.shiftmode;
  232. so.shiftimm:=ref.shiftimm;
  233. list.concat(taicpu.op_reg_reg_reg_shifterop(A_ADD,preferred_newbasereg,ref.base,ref.index,so));
  234. reference_reset_base(ref,preferred_newbasereg,ref.offset,ref.temppos,ref.alignment,ref.volatility);
  235. { possibly still an invalid offset -> fall through }
  236. end
  237. else if ref.offset<>0 then
  238. begin
  239. if (preferred_newbasereg=NR_NO) or
  240. { we keep ref.index, so it must not be overwritten }
  241. (ref.index=preferred_newbasereg) then
  242. preferred_newbasereg:=getaddressregister(list);
  243. { add to the base and not to the index, because the index
  244. may be scaled; this works even if the base is SP }
  245. a_op_const_reg_reg(list,OP_ADD,OS_ADDR,ref.offset,ref.base,preferred_newbasereg);
  246. ref.offset:=0;
  247. ref.base:=preferred_newbasereg;
  248. { finished }
  249. exit;
  250. end
  251. else
  252. { valid -> exit }
  253. exit;
  254. end;
  255. { todo }
  256. A_LD1,A_LD2,A_LD3,A_LD4,
  257. A_ST1,A_ST2,A_ST3,A_ST4:
  258. internalerror(2014110702);
  259. { these don't support base+index }
  260. A_LDUR,A_STUR,
  261. A_LDP,A_STP:
  262. begin
  263. { these either don't support pre-/post-indexing, or don't
  264. support it with base+index }
  265. if ref.addressmode<>AM_OFFSET then
  266. internalerror(2014110911);
  267. if preferred_newbasereg=NR_NO then
  268. preferred_newbasereg:=getaddressregister(list);
  269. if ref.shiftmode<>SM_None then
  270. begin
  271. { "add" supports a superset of the shift modes supported by
  272. load/store instructions }
  273. shifterop_reset(so);
  274. so.shiftmode:=ref.shiftmode;
  275. so.shiftimm:=ref.shiftimm;
  276. list.concat(taicpu.op_reg_reg_reg_shifterop(A_ADD,preferred_newbasereg,ref.base,ref.index,so));
  277. end
  278. else
  279. a_op_reg_reg_reg(list,OP_ADD,OS_ADDR,ref.index,ref.base,preferred_newbasereg);
  280. reference_reset_base(ref,preferred_newbasereg,ref.offset,ref.temppos,ref.alignment,ref.volatility);
  281. { fall through to the handling of base + offset, since the
  282. offset may still be too big }
  283. end;
  284. else
  285. internalerror(2014110903);
  286. end;
  287. end;
  288. { base + offset }
  289. if ref.base<>NR_NO then
  290. begin
  291. { valid offset for LDUR/STUR -> use that }
  292. if (ref.addressmode=AM_OFFSET) and
  293. (op in [A_LDR,A_STR]) and
  294. (ref.offset>=-256) and
  295. (ref.offset<=255) then
  296. begin
  297. if op=A_LDR then
  298. op:=A_LDUR
  299. else
  300. op:=A_STUR
  301. end
  302. { if it's not a valid LDUR/STUR, use LDR/STR }
  303. else if (op in [A_LDUR,A_STUR]) and
  304. ((ref.offset<-256) or
  305. (ref.offset>255) or
  306. (ref.addressmode<>AM_OFFSET)) then
  307. begin
  308. if op=A_LDUR then
  309. op:=A_LDR
  310. else
  311. op:=A_STR
  312. end;
  313. case op of
  314. A_LDR,A_STR:
  315. begin
  316. case ref.addressmode of
  317. AM_PREINDEXED:
  318. begin
  319. { since the loaded/stored register cannot be the same
  320. as the base register, we can safely add the
  321. offset to the base if it doesn't fit}
  322. if (ref.offset<-256) or
  323. (ref.offset>255) then
  324. begin
  325. a_op_const_reg(list,OP_ADD,OS_ADDR,ref.offset,ref.base);
  326. ref.offset:=0;
  327. end;
  328. end;
  329. AM_POSTINDEXED:
  330. begin
  331. { cannot emulate post-indexing if we have to fold the
  332. offset into the base register }
  333. if (ref.offset<-256) or
  334. (ref.offset>255) then
  335. internalerror(2014110909);
  336. { ok }
  337. end;
  338. AM_OFFSET:
  339. begin
  340. { unsupported offset -> fold into base register }
  341. accesssize:=1 shl tcgsizep2size[size];
  342. if (ref.offset<0) or
  343. (ref.offset>(((1 shl 12)-1)*accesssize)) or
  344. ((ref.offset mod accesssize)<>0) then
  345. begin
  346. if preferred_newbasereg=NR_NO then
  347. preferred_newbasereg:=getaddressregister(list);
  348. { can we split the offset beween an
  349. "add/sub (imm12 shl 12)" and the load (also an
  350. imm12)?
  351. -- the offset from the load will always be added,
  352. that's why the lower bound has a smaller range
  353. than the upper bound; it must also be a multiple
  354. of the access size }
  355. if (ref.offset>=-(((1 shl 12)-1) shl 12)) and
  356. (ref.offset<=((1 shl 12)-1) shl 12 + ((1 shl 12)-1)) and
  357. ((ref.offset mod accesssize)=0) then
  358. begin
  359. a_op_const_reg_reg(list,OP_ADD,OS_ADDR,(ref.offset shr 12) shl 12,ref.base,preferred_newbasereg);
  360. ref.offset:=ref.offset-(ref.offset shr 12) shl 12;
  361. end
  362. else
  363. begin
  364. a_op_const_reg_reg(list,OP_ADD,OS_ADDR,ref.offset,ref.base,preferred_newbasereg);
  365. ref.offset:=0;
  366. end;
  367. reference_reset_base(ref,preferred_newbasereg,ref.offset,ref.temppos,ref.alignment,ref.volatility);
  368. end;
  369. end
  370. end;
  371. end;
  372. A_LDP,A_STP:
  373. begin
  374. { unsupported offset -> fold into base register (these
  375. instructions support all addressmodes) }
  376. if (ref.offset<-(1 shl (6+tcgsizep2size[size]))) or
  377. (ref.offset>(1 shl (6+tcgsizep2size[size]))-1) then
  378. begin
  379. case ref.addressmode of
  380. AM_POSTINDEXED:
  381. { don't emulate post-indexing if we have to fold the
  382. offset into the base register }
  383. internalerror(2014110910);
  384. AM_PREINDEXED:
  385. { this means the offset must be added to the current
  386. base register }
  387. preferred_newbasereg:=ref.base;
  388. AM_OFFSET:
  389. if preferred_newbasereg=NR_NO then
  390. preferred_newbasereg:=getaddressregister(list);
  391. end;
  392. a_op_const_reg_reg(list,OP_ADD,OS_ADDR,ref.offset,ref.base,preferred_newbasereg);
  393. reference_reset_base(ref,preferred_newbasereg,0,ref.temppos,ref.alignment,ref.volatility);
  394. end
  395. end;
  396. A_LDUR,A_STUR:
  397. begin
  398. { valid, checked above }
  399. end;
  400. { todo }
  401. A_LD1,A_LD2,A_LD3,A_LD4,
  402. A_ST1,A_ST2,A_ST3,A_ST4:
  403. internalerror(2014110908);
  404. else
  405. internalerror(2014110708);
  406. end;
  407. { done }
  408. exit;
  409. end;
  410. { only an offset -> change to base (+ offset 0) }
  411. if preferred_newbasereg=NR_NO then
  412. preferred_newbasereg:=getaddressregister(list);
  413. a_load_const_reg(list,OS_ADDR,ref.offset,preferred_newbasereg);
  414. reference_reset_base(ref,preferred_newbasereg,0,ref.temppos,newalignment(8,ref.offset),ref.volatility);
  415. end;
  416. function tcgaarch64.makeregsize(reg: tregister; size: tcgsize): tregister;
  417. var
  418. subreg:Tsubregister;
  419. begin
  420. subreg:=cgsize2subreg(getregtype(reg),size);
  421. result:=reg;
  422. setsubreg(result,subreg);
  423. end;
  424. function tcgaarch64.getfpuregister(list: TAsmList; size: Tcgsize): Tregister;
  425. begin
  426. internalerror(2014122110);
  427. { squash warning }
  428. result:=NR_NO;
  429. end;
  430. function tcgaarch64.handle_load_store(list: TAsmList; op: tasmop; size: tcgsize; oppostfix: toppostfix; reg: tregister; ref: treference):treference;
  431. begin
  432. make_simple_ref(list,op,size,oppostfix,ref,NR_NO);
  433. list.concat(setoppostfix(taicpu.op_reg_ref(op,reg,ref),oppostfix));
  434. result:=ref;
  435. end;
  436. procedure tcgaarch64.handle_reg_imm12_reg(list: TAsmList; op: Tasmop; size: tcgsize; src: tregister; a: tcgint; dst: tregister; tmpreg: tregister; setflags, usedest: boolean);
  437. var
  438. instr: taicpu;
  439. so: tshifterop;
  440. hadtmpreg: boolean;
  441. begin
  442. { imm12 }
  443. if (a>=0) and
  444. (a<=((1 shl 12)-1)) then
  445. if usedest then
  446. instr:=taicpu.op_reg_reg_const(op,dst,src,a)
  447. else
  448. instr:=taicpu.op_reg_const(op,src,a)
  449. { imm12 lsl 12 }
  450. else if (a and not(((tcgint(1) shl 12)-1) shl 12))=0 then
  451. begin
  452. so.shiftmode:=SM_LSL;
  453. so.shiftimm:=12;
  454. if usedest then
  455. instr:=taicpu.op_reg_reg_const_shifterop(op,dst,src,a shr 12,so)
  456. else
  457. instr:=taicpu.op_reg_const_shifterop(op,src,a shr 12,so)
  458. end
  459. else
  460. begin
  461. { todo: other possible optimizations (e.g. load 16 bit constant in
  462. register and then add/sub/cmp/cmn shifted the rest) }
  463. if tmpreg=NR_NO then
  464. begin
  465. hadtmpreg:=false;
  466. tmpreg:=getintregister(list,size);
  467. end
  468. else
  469. begin
  470. hadtmpreg:=true;
  471. getcpuregister(list,tmpreg);
  472. end;
  473. a_load_const_reg(list,size,a,tmpreg);
  474. if usedest then
  475. instr:=taicpu.op_reg_reg_reg(op,dst,src,tmpreg)
  476. else
  477. instr:=taicpu.op_reg_reg(op,src,tmpreg);
  478. if hadtmpreg then
  479. ungetcpuregister(list,tmpreg);
  480. end;
  481. if setflags then
  482. setoppostfix(instr,PF_S);
  483. list.concat(instr);
  484. end;
  485. {****************************************************************************
  486. Assembler code
  487. ****************************************************************************}
  488. procedure tcgaarch64.init_register_allocators;
  489. begin
  490. inherited init_register_allocators;
  491. rg[R_INTREGISTER]:=trgintcpu.create(R_INTREGISTER,R_SUBWHOLE,
  492. [RS_X0,RS_X1,RS_X2,RS_X3,RS_X4,RS_X5,RS_X6,RS_X7,RS_X8,
  493. RS_X9,RS_X10,RS_X11,RS_X12,RS_X13,RS_X14,RS_X15,RS_X16,RS_X17,
  494. RS_X19,RS_X20,RS_X21,RS_X22,RS_X23,RS_X24,RS_X25,RS_X26,RS_X27,RS_X28
  495. { maybe we can enable this in the future for leaf functions (it's
  496. the frame pointer)
  497. ,RS_X29 }],
  498. first_int_imreg,[]);
  499. rg[R_MMREGISTER]:=trgcpu.create(R_MMREGISTER,R_SUBMMD,
  500. [RS_Q0,RS_Q1,RS_Q2,RS_Q3,RS_Q4,RS_Q5,RS_Q6,RS_Q7,
  501. RS_Q8,RS_Q9,RS_Q10,RS_Q11,RS_Q12,RS_Q13,RS_Q14,RS_Q15,
  502. RS_Q16,RS_Q17,RS_Q18,RS_Q19,RS_Q20,RS_Q21,RS_Q22,RS_Q23,
  503. RS_Q24,RS_Q25,RS_Q26,RS_Q27,RS_Q28,RS_Q29,RS_Q30,RS_Q31],
  504. first_mm_imreg,[]);
  505. end;
  506. procedure tcgaarch64.done_register_allocators;
  507. begin
  508. rg[R_INTREGISTER].free;
  509. rg[R_FPUREGISTER].free;
  510. rg[R_MMREGISTER].free;
  511. inherited done_register_allocators;
  512. end;
  513. function tcgaarch64.getmmregister(list: TAsmList; size: tcgsize):tregister;
  514. begin
  515. case size of
  516. OS_F32:
  517. result:=rg[R_MMREGISTER].getregister(list,R_SUBMMS);
  518. OS_F64:
  519. result:=rg[R_MMREGISTER].getregister(list,R_SUBMMD)
  520. else
  521. internalerror(2014102701);
  522. end;
  523. end;
  524. procedure tcgaarch64.a_call_name(list: TAsmList; const s: string; weak: boolean);
  525. begin
  526. if not weak then
  527. list.concat(taicpu.op_sym(A_BL,current_asmdata.RefAsmSymbol(s,AT_FUNCTION)))
  528. else
  529. list.concat(taicpu.op_sym(A_BL,current_asmdata.WeakRefAsmSymbol(s,AT_FUNCTION)));
  530. end;
  531. procedure tcgaarch64.a_call_reg(list:TAsmList;Reg:tregister);
  532. begin
  533. list.concat(taicpu.op_reg(A_BLR,reg));
  534. end;
  535. {********************** load instructions ********************}
  536. procedure tcgaarch64.a_load_const_reg(list: TAsmList; size: tcgsize; a: tcgint; reg : tregister);
  537. var
  538. opc: tasmop;
  539. shift: byte;
  540. so: tshifterop;
  541. reginited,doinverted,extendedsize: boolean;
  542. manipulated_a: tcgint;
  543. leftover_a: word;
  544. begin
  545. {$ifdef extdebug}
  546. list.concat(tai_comment.Create(strpnew('Generating constant ' + tostr(a) + ' / $' + hexstr(a, 16))));
  547. {$endif extdebug}
  548. extendedsize := (size in [OS_64,OS_S64]);
  549. case a of
  550. { Small positive number }
  551. $0..$FFFF:
  552. begin
  553. list.concat(taicpu.op_reg_const(A_MOVZ, reg, a));
  554. Exit;
  555. end;
  556. { Small negative number }
  557. -65536..-1:
  558. begin
  559. list.concat(taicpu.op_reg_const(A_MOVN, reg, Word(not a)));
  560. Exit;
  561. end;
  562. { Can be represented as a negative number more compactly }
  563. $FFFF0000..$FFFFFFFF:
  564. begin
  565. { if we load a value into a 32 bit register, it is automatically
  566. zero-extended to 64 bit }
  567. list.concat(taicpu.op_reg_const(A_MOVN, makeregsize(reg,OS_32), Word(not a)));
  568. Exit;
  569. end;
  570. else
  571. begin
  572. if not extendedsize then
  573. { Mostly so programmers don't get confused when they view the disassembly and
  574. 'a' is sign-extended to 64-bit, say, but also avoids potential problems with
  575. third-party assemblers if the number is out of bounds for a given size }
  576. a := Cardinal(a);
  577. { Check to see if a is a valid shifter constant that can be encoded in ORR as is }
  578. if is_shifter_const(a,size) then
  579. begin
  580. { Use synthetic "MOV" instruction instead of "ORR reg,wzr,#a" (an alias),
  581. since AArch64 conventions prefer this, and it's clearer in the
  582. disassembly }
  583. list.concat(taicpu.op_reg_const(A_MOV,reg,a));
  584. Exit;
  585. end;
  586. { If the value of a fits into 32 bits, it's fastest to use movz/movk regardless }
  587. if extendedsize and ((a shr 32) <> 0) then
  588. begin
  589. { This determines whether this write can be performed with an ORR followed by MOVK
  590. by copying the 3nd word to the 1st word for the ORR constant, then overwriting
  591. the 1st word. The alternative would require 4 instructions. This sequence is
  592. common when division reciprocals are calculated (e.g. 3 produces AAAAAAAAAAAAAAAB). }
  593. leftover_a := word(a and $FFFF);
  594. manipulated_a := (a and $FFFFFFFFFFFF0000) or ((a shr 32) and $FFFF);
  595. { if manipulated_a = a, don't check, because is_shifter_const was already
  596. called for a and it returned False. Reduces processing time. [Kit] }
  597. if (manipulated_a <> a) and is_shifter_const(manipulated_a, OS_64) then
  598. begin
  599. { Encode value as:
  600. orr reg,xzr,manipulated_a
  601. movk reg,#(leftover_a)
  602. Use "orr" instead of "mov" here for the assembly dump so it better
  603. implies that something special is happening with the number arrangement.
  604. }
  605. list.concat(taicpu.op_reg_reg_const(A_ORR, reg, NR_XZR, manipulated_a));
  606. list.concat(taicpu.op_reg_const(A_MOVK, reg, leftover_a));
  607. Exit;
  608. end;
  609. { This determines whether this write can be performed with an ORR followed by MOVK
  610. by copying the 2nd word to the 4th word for the ORR constant, then overwriting
  611. the 4th word. The alternative would require 3 instructions }
  612. leftover_a := word(a shr 48);
  613. manipulated_a := (a and $0000FFFFFFFFFFFF);
  614. if manipulated_a = $0000FFFFFFFFFFFF then
  615. begin
  616. { This is even better, as we can just use a single MOVN on the last word }
  617. shifterop_reset(so);
  618. so.shiftmode := SM_LSL;
  619. so.shiftimm := 48;
  620. list.concat(taicpu.op_reg_const_shifterop(A_MOVN, reg, word(not leftover_a), so));
  621. Exit;
  622. end;
  623. manipulated_a := manipulated_a or (((a shr 16) and $FFFF) shl 48);
  624. { if manipulated_a = a, don't check, because is_shifter_const was already
  625. called for a and it returned False. Reduces processing time. [Kit] }
  626. if (manipulated_a <> a) and is_shifter_const(manipulated_a, OS_64) then
  627. begin
  628. { Encode value as:
  629. orr reg,xzr,manipulated_a
  630. movk reg,#(leftover_a),lsl #48
  631. Use "orr" instead of "mov" here for the assembly dump so it better
  632. implies that something special is happening with the number arrangement.
  633. }
  634. list.concat(taicpu.op_reg_reg_const(A_ORR, reg, NR_XZR, manipulated_a));
  635. shifterop_reset(so);
  636. so.shiftmode := SM_LSL;
  637. so.shiftimm := 48;
  638. list.concat(taicpu.op_reg_const_shifterop(A_MOVK, reg, leftover_a, so));
  639. Exit;
  640. end;
  641. case a of
  642. { If a is in the given negative range, it can be stored
  643. more efficiently if it is inverted. }
  644. TCgInt($FFFF000000000000)..-65537:
  645. begin
  646. { NOTE: This excluded range can be more efficiently
  647. stored as the first 16 bits followed by a shifter constant }
  648. case a of
  649. TCgInt($FFFF0000FFFF0000)..TCgInt($FFFF0000FFFFFFFF):
  650. doinverted := False;
  651. else
  652. begin
  653. doinverted := True;
  654. a := not a;
  655. end;
  656. end;
  657. end;
  658. else
  659. doinverted := False;
  660. end;
  661. end
  662. else
  663. doinverted:=False;
  664. end;
  665. end;
  666. reginited:=false;
  667. shift:=0;
  668. if doinverted then
  669. opc:=A_MOVN
  670. else
  671. opc:=A_MOVZ;
  672. repeat
  673. { leftover is shifterconst? (don't check if we can represent it just
  674. as effectively with movz/movk, as this check is expensive) }
  675. if (word(a)<>0) then
  676. begin
  677. if not doinverted and
  678. ((shift<tcgsize2size[size]*(8 div 2)) and
  679. ((a shr 16)<>0)) and
  680. is_shifter_const(a shl shift,size) then
  681. begin
  682. if reginited then
  683. list.concat(taicpu.op_reg_reg_const(A_ORR,reg,reg,a shl shift))
  684. else
  685. list.concat(taicpu.op_reg_reg_const(A_ORR,reg,makeregsize(NR_XZR,size),a shl shift));
  686. exit;
  687. end;
  688. { set all 16 bit parts <> 0 }
  689. if shift=0 then
  690. begin
  691. list.concat(taicpu.op_reg_const(opc,reg,word(a)));
  692. reginited:=true;
  693. end
  694. else
  695. begin
  696. shifterop_reset(so);
  697. so.shiftmode:=SM_LSL;
  698. so.shiftimm:=shift;
  699. if not reginited then
  700. begin
  701. list.concat(taicpu.op_reg_const_shifterop(opc,reg,word(a),so));
  702. reginited:=true;
  703. end
  704. else
  705. begin
  706. if doinverted then
  707. list.concat(taicpu.op_reg_const_shifterop(A_MOVK,reg,word(not a),so))
  708. else
  709. list.concat(taicpu.op_reg_const_shifterop(A_MOVK,reg,word(a),so));
  710. end;
  711. end;
  712. end;
  713. a:=a shr 16;
  714. inc(shift,16);
  715. until a = 0;
  716. if not reginited then
  717. internalerror(2014102702);
  718. end;
  719. procedure tcgaarch64.a_load_const_ref(list: TAsmList; size: tcgsize; a: tcgint; const ref: treference);
  720. var
  721. reg: tregister;
  722. href: treference;
  723. i: Integer;
  724. begin
  725. { use the zero register if possible }
  726. if a=0 then
  727. begin
  728. href:=ref;
  729. inc(href.offset,tcgsize2size[size]-1);
  730. if (tcgsize2size[size]>1) and (ref.alignment=1) and (simple_ref_type(A_STUR,OS_8,PF_None,ref)=sr_simple) and
  731. (simple_ref_type(A_STUR,OS_8,PF_None,href)=sr_simple) then
  732. begin
  733. href:=ref;
  734. for i:=0 to tcgsize2size[size]-1 do
  735. begin
  736. a_load_const_ref(list,OS_8,0,href);
  737. inc(href.offset);
  738. end;
  739. end
  740. else
  741. begin
  742. if size in [OS_64,OS_S64] then
  743. reg:=NR_XZR
  744. else
  745. reg:=NR_WZR;
  746. a_load_reg_ref(list,size,size,reg,ref);
  747. end;
  748. end
  749. else
  750. inherited;
  751. end;
  752. procedure tcgaarch64.a_load_reg_ref(list: TAsmList; fromsize, tosize: tcgsize; reg: tregister; const ref: treference);
  753. var
  754. oppostfix:toppostfix;
  755. hreg: tregister;
  756. begin
  757. if tcgsize2Size[fromsize]>=tcgsize2Size[tosize] then
  758. begin
  759. fromsize:=tosize;
  760. reg:=makeregsize(list,reg,fromsize);
  761. end
  762. { have a 32 bit register but need a 64 bit one? }
  763. else if tosize in [OS_64,OS_S64] then
  764. begin
  765. { sign extend if necessary }
  766. if fromsize in [OS_S8,OS_S16,OS_S32] then
  767. begin
  768. { can't overwrite reg, may be a constant reg }
  769. hreg:=getintregister(list,tosize);
  770. a_load_reg_reg(list,fromsize,tosize,reg,hreg);
  771. reg:=hreg;
  772. end
  773. else
  774. { top 32 bit are zero by default }
  775. reg:=makeregsize(reg,OS_64);
  776. fromsize:=tosize;
  777. end;
  778. if not(target_info.system=system_aarch64_darwin) and (ref.alignment<>0) and
  779. (ref.alignment<tcgsize2size[tosize]) then
  780. begin
  781. a_load_reg_ref_unaligned(list,fromsize,tosize,reg,ref);
  782. end
  783. else
  784. begin
  785. case tosize of
  786. { signed integer registers }
  787. OS_8,
  788. OS_S8:
  789. oppostfix:=PF_B;
  790. OS_16,
  791. OS_S16:
  792. oppostfix:=PF_H;
  793. OS_32,
  794. OS_S32,
  795. OS_64,
  796. OS_S64:
  797. oppostfix:=PF_None;
  798. else
  799. InternalError(200308299);
  800. end;
  801. handle_load_store(list,A_STR,tosize,oppostfix,reg,ref);
  802. end;
  803. end;
  804. procedure tcgaarch64.a_load_ref_reg(list: TAsmList; fromsize, tosize: tcgsize; const ref: treference; reg: tregister);
  805. var
  806. oppostfix:toppostfix;
  807. begin
  808. if tcgsize2Size[fromsize]>=tcgsize2Size[tosize] then
  809. fromsize:=tosize;
  810. { ensure that all bits of the 32/64 register are always correctly set:
  811. * default behaviour is always to zero-extend to the entire (64 bit)
  812. register -> unsigned 8/16/32 bit loads only exist with a 32 bit
  813. target register, as the upper 32 bit will be zeroed implicitly
  814. -> always make target register 32 bit
  815. * signed loads exist both with 32 and 64 bit target registers,
  816. depending on whether the value should be sign extended to 32 or
  817. to 64 bit (if sign extended to 32 bit, the upper 32 bits of the
  818. corresponding 64 bit register are again zeroed) -> no need to
  819. change anything (we only have 32 and 64 bit registers), except that
  820. when loading an OS_S32 to a 32 bit register, we don't need/can't
  821. use sign extension
  822. }
  823. if fromsize in [OS_8,OS_16,OS_32] then
  824. reg:=makeregsize(reg,OS_32);
  825. if not(target_info.system=system_aarch64_darwin) and (ref.alignment<>0) and
  826. (ref.alignment<tcgsize2size[fromsize]) then
  827. begin
  828. a_load_ref_reg_unaligned(list,fromsize,tosize,ref,reg);
  829. exit;
  830. end;
  831. case fromsize of
  832. { signed integer registers }
  833. OS_8:
  834. oppostfix:=PF_B;
  835. OS_S8:
  836. oppostfix:=PF_SB;
  837. OS_16:
  838. oppostfix:=PF_H;
  839. OS_S16:
  840. oppostfix:=PF_SH;
  841. OS_S32:
  842. if getsubreg(reg)=R_SUBD then
  843. oppostfix:=PF_NONE
  844. else
  845. oppostfix:=PF_SW;
  846. OS_32,
  847. OS_64,
  848. OS_S64:
  849. oppostfix:=PF_None;
  850. else
  851. InternalError(200308297);
  852. end;
  853. handle_load_store(list,A_LDR,fromsize,oppostfix,reg,ref);
  854. { clear upper 16 bits if the value was negative }
  855. if (fromsize=OS_S8) and (tosize=OS_16) then
  856. a_load_reg_reg(list,fromsize,tosize,reg,reg);
  857. end;
  858. procedure tcgaarch64.a_load_ref_reg_unaligned(list: TAsmList; fromsize, tosize: tcgsize; const ref: treference; register: tregister);
  859. var
  860. href: treference;
  861. hreg1, hreg2, tmpreg,tmpreg2: tregister;
  862. i : Integer;
  863. begin
  864. case fromsize of
  865. OS_64,OS_S64:
  866. begin
  867. { split into two 32 bit loads }
  868. hreg1:=getintregister(list,OS_32);
  869. hreg2:=getintregister(list,OS_32);
  870. if target_info.endian=endian_big then
  871. begin
  872. tmpreg:=hreg1;
  873. hreg1:=hreg2;
  874. hreg2:=tmpreg;
  875. end;
  876. { can we use LDP? }
  877. if (ref.alignment=4) and
  878. (simple_ref_type(A_LDP,OS_32,PF_None,ref)=sr_simple) then
  879. list.concat(taicpu.op_reg_reg_ref(A_LDP,hreg1,hreg2,ref))
  880. else
  881. begin
  882. a_load_ref_reg(list,OS_32,OS_32,ref,hreg1);
  883. href:=ref;
  884. inc(href.offset,4);
  885. a_load_ref_reg(list,OS_32,OS_32,href,hreg2);
  886. end;
  887. a_load_reg_reg(list,OS_32,OS_64,hreg1,register);
  888. list.concat(taicpu.op_reg_reg_const_const(A_BFI,register,makeregsize(hreg2,OS_64),32,32));
  889. end;
  890. OS_16,OS_S16,
  891. OS_32,OS_S32:
  892. begin
  893. if ref.alignment=2 then
  894. begin
  895. href:=ref;
  896. if target_info.endian=endian_big then
  897. inc(href.offset,tcgsize2size[fromsize]-2);
  898. tmpreg:=getintregister(list,OS_32);
  899. a_load_ref_reg(list,OS_16,OS_32,href,tmpreg);
  900. tmpreg2:=getintregister(list,OS_32);
  901. for i:=1 to (tcgsize2size[fromsize]-1) div 2 do
  902. begin
  903. if target_info.endian=endian_big then
  904. dec(href.offset,2)
  905. else
  906. inc(href.offset,2);
  907. a_load_ref_reg(list,OS_16,OS_32,href,tmpreg2);
  908. list.concat(taicpu.op_reg_reg_const_const(A_BFI,tmpreg,tmpreg2,i*16,16));
  909. end;
  910. a_load_reg_reg(list,fromsize,tosize,tmpreg,register);
  911. end
  912. else
  913. begin
  914. href:=ref;
  915. if target_info.endian=endian_big then
  916. inc(href.offset,tcgsize2size[fromsize]-1);
  917. tmpreg:=getintregister(list,OS_32);
  918. a_load_ref_reg(list,OS_8,OS_32,href,tmpreg);
  919. tmpreg2:=getintregister(list,OS_32);
  920. for i:=1 to tcgsize2size[fromsize]-1 do
  921. begin
  922. if target_info.endian=endian_big then
  923. dec(href.offset)
  924. else
  925. inc(href.offset);
  926. a_load_ref_reg(list,OS_8,OS_32,href,tmpreg2);
  927. list.concat(taicpu.op_reg_reg_const_const(A_BFI,tmpreg,tmpreg2,i*8,8));
  928. end;
  929. a_load_reg_reg(list,fromsize,tosize,tmpreg,register);
  930. end;
  931. end;
  932. else
  933. inherited;
  934. end;
  935. end;
  936. procedure tcgaarch64.a_load_reg_reg(list:TAsmList;fromsize,tosize:tcgsize;reg1,reg2:tregister);
  937. var
  938. instr: taicpu;
  939. begin
  940. { we use both 32 and 64 bit registers -> insert conversion when when
  941. we have to truncate/sign extend inside the (32 or 64 bit) register
  942. holding the value, and when we sign extend from a 32 to a 64 bit
  943. register }
  944. if (tcgsize2size[fromsize]>tcgsize2size[tosize]) or
  945. ((tcgsize2size[fromsize]=tcgsize2size[tosize]) and
  946. (fromsize<>tosize) and
  947. not(fromsize in [OS_32,OS_S32,OS_64,OS_S64])) or
  948. ((fromsize in [OS_S8,OS_S16,OS_S32]) and
  949. (tosize in [OS_64,OS_S64])) or
  950. { needs to mask out the sign in the top 16 bits }
  951. ((fromsize=OS_S8) and
  952. (tosize=OS_16)) then
  953. begin
  954. case tosize of
  955. OS_8:
  956. list.concat(taicpu.op_reg_reg(A_UXTB,reg2,makeregsize(reg1,OS_32)));
  957. OS_16:
  958. list.concat(taicpu.op_reg_reg(A_UXTH,reg2,makeregsize(reg1,OS_32)));
  959. OS_S8:
  960. list.concat(taicpu.op_reg_reg(A_SXTB,reg2,makeregsize(reg1,OS_32)));
  961. OS_S16:
  962. list.concat(taicpu.op_reg_reg(A_SXTH,reg2,makeregsize(reg1,OS_32)));
  963. { while "mov wN, wM" automatically inserts a zero-extension and
  964. hence we could encode a 64->32 bit move like that, the problem
  965. is that we then can't distinguish 64->32 from 32->32 moves, and
  966. the 64->32 truncation could be removed altogether... So use a
  967. different instruction }
  968. OS_32,
  969. OS_S32:
  970. { in theory, reg1 should be 64 bit here (since fromsize>tosize),
  971. but because of the way location_force_register() tries to
  972. avoid superfluous zero/sign extensions, it's not always the
  973. case -> also force reg1 to to 64 bit }
  974. list.concat(taicpu.op_reg_reg_const_const(A_UBFIZ,makeregsize(reg2,OS_64),makeregsize(reg1,OS_64),0,32));
  975. OS_64,
  976. OS_S64:
  977. list.concat(taicpu.op_reg_reg(A_SXTW,reg2,makeregsize(reg1,OS_32)));
  978. else
  979. internalerror(2002090901);
  980. end;
  981. end
  982. else
  983. begin
  984. { 32 -> 32 bit move implies zero extension (sign extensions have
  985. been handled above) -> also use for 32 <-> 64 bit moves }
  986. if not(fromsize in [OS_64,OS_S64]) or
  987. not(tosize in [OS_64,OS_S64]) then
  988. instr:=taicpu.op_reg_reg(A_MOV,makeregsize(reg2,OS_32),makeregsize(reg1,OS_32))
  989. else
  990. instr:=taicpu.op_reg_reg(A_MOV,reg2,reg1);
  991. list.Concat(instr);
  992. { Notify the register allocator that we have written a move instruction so
  993. it can try to eliminate it. }
  994. add_move_instruction(instr);
  995. end;
  996. end;
  997. procedure tcgaarch64.a_loadaddr_ref_reg(list: TAsmList; const ref: treference; r: tregister);
  998. var
  999. href: treference;
  1000. so: tshifterop;
  1001. op: tasmop;
  1002. begin
  1003. op:=A_LDR;
  1004. href:=ref;
  1005. { simplify as if we're going to perform a regular 64 bit load, using
  1006. "r" as the new base register if possible/necessary }
  1007. make_simple_ref(list,op,OS_ADDR,PF_None,href,r);
  1008. { load literal? }
  1009. if assigned(href.symbol) then
  1010. begin
  1011. if (href.base<>NR_NO) or
  1012. (href.index<>NR_NO) or
  1013. not assigned(href.symboldata) then
  1014. internalerror(2014110912);
  1015. list.concat(taicpu.op_reg_sym_ofs(A_ADR,r,href.symbol,href.offset));
  1016. end
  1017. else
  1018. begin
  1019. if href.index<>NR_NO then
  1020. begin
  1021. if href.shiftmode<>SM_None then
  1022. begin
  1023. { "add" supports a supperset of the shift modes supported by
  1024. load/store instructions }
  1025. shifterop_reset(so);
  1026. so.shiftmode:=href.shiftmode;
  1027. so.shiftimm:=href.shiftimm;
  1028. list.concat(taicpu.op_reg_reg_reg_shifterop(A_ADD,r,href.base,href.index,so));
  1029. end
  1030. else
  1031. a_op_reg_reg_reg(list,OP_ADD,OS_ADDR,href.index,href.base,r);
  1032. end
  1033. else if href.offset<>0 then
  1034. a_op_const_reg_reg(list,OP_ADD,OS_ADDR,href.offset,href.base,r)
  1035. else
  1036. a_load_reg_reg(list,OS_ADDR,OS_ADDR,href.base,r);
  1037. end;
  1038. end;
  1039. procedure tcgaarch64.a_loadfpu_reg_reg(list: TAsmList; fromsize, tosize: tcgsize; reg1, reg2: tregister);
  1040. begin
  1041. internalerror(2014122107)
  1042. end;
  1043. procedure tcgaarch64.a_loadfpu_ref_reg(list: TAsmList; fromsize, tosize: tcgsize; const ref: treference; reg: tregister);
  1044. begin
  1045. internalerror(2014122108)
  1046. end;
  1047. procedure tcgaarch64.a_loadfpu_reg_ref(list: TAsmList; fromsize, tosize: tcgsize; reg: tregister; const ref: treference);
  1048. begin
  1049. internalerror(2014122109)
  1050. end;
  1051. procedure tcgaarch64.a_loadmm_reg_reg(list: TAsmList; fromsize, tosize: tcgsize; reg1, reg2: tregister; shuffle: pmmshuffle);
  1052. var
  1053. instr: taicpu;
  1054. begin
  1055. if assigned(shuffle) and
  1056. not shufflescalar(shuffle) then
  1057. internalerror(2014122104);
  1058. if fromsize=tosize then
  1059. begin
  1060. instr:=taicpu.op_reg_reg(A_FMOV,reg2,reg1);
  1061. { Notify the register allocator that we have written a move
  1062. instruction so it can try to eliminate it. }
  1063. add_move_instruction(instr);
  1064. { FMOV cannot generate a floating point exception }
  1065. end
  1066. else
  1067. begin
  1068. if (reg_cgsize(reg1)<>fromsize) or
  1069. (reg_cgsize(reg2)<>tosize) then
  1070. internalerror(2014110913);
  1071. instr:=taicpu.op_reg_reg(A_FCVT,reg2,reg1);
  1072. maybe_check_for_fpu_exception(list);
  1073. end;
  1074. list.Concat(instr);
  1075. end;
  1076. procedure tcgaarch64.a_loadmm_ref_reg(list: TAsmList; fromsize, tosize: tcgsize; const ref: treference; reg: tregister; shuffle: pmmshuffle);
  1077. var
  1078. tmpreg: tregister;
  1079. begin
  1080. if assigned(shuffle) and
  1081. not shufflescalar(shuffle) then
  1082. internalerror(2014122105);
  1083. tmpreg:=NR_NO;
  1084. if (fromsize<>tosize) then
  1085. begin
  1086. tmpreg:=reg;
  1087. reg:=getmmregister(list,fromsize);
  1088. end;
  1089. handle_load_store(list,A_LDR,fromsize,PF_None,reg,ref);
  1090. if (fromsize<>tosize) then
  1091. a_loadmm_reg_reg(list,fromsize,tosize,reg,tmpreg,nil);
  1092. end;
  1093. procedure tcgaarch64.a_loadmm_reg_ref(list: TAsmList; fromsize, tosize: tcgsize; reg: tregister; const ref: treference; shuffle: pmmshuffle);
  1094. var
  1095. tmpreg: tregister;
  1096. begin
  1097. if assigned(shuffle) and
  1098. not shufflescalar(shuffle) then
  1099. internalerror(2014122106);
  1100. if (fromsize<>tosize) then
  1101. begin
  1102. tmpreg:=getmmregister(list,tosize);
  1103. a_loadmm_reg_reg(list,fromsize,tosize,reg,tmpreg,nil);
  1104. reg:=tmpreg;
  1105. end;
  1106. handle_load_store(list,A_STR,tosize,PF_NONE,reg,ref);
  1107. end;
  1108. procedure tcgaarch64.a_loadmm_intreg_reg(list: TAsmList; fromsize, tosize: tcgsize; intreg, mmreg: tregister; shuffle: pmmshuffle);
  1109. begin
  1110. if not shufflescalar(shuffle) then
  1111. internalerror(2014122801);
  1112. if tcgsize2size[fromsize]<>tcgsize2size[tosize] then
  1113. internalerror(2014122803);
  1114. case tcgsize2size[tosize] of
  1115. 4:
  1116. setsubreg(mmreg,R_SUBMMS);
  1117. 8:
  1118. setsubreg(mmreg,R_SUBMMD);
  1119. else
  1120. internalerror(2020101310);
  1121. end;
  1122. list.concat(taicpu.op_indexedreg_reg(A_INS,mmreg,0,intreg));
  1123. end;
  1124. procedure tcgaarch64.a_loadmm_reg_intreg(list: TAsmList; fromsize, tosize: tcgsize; mmreg, intreg: tregister; shuffle: pmmshuffle);
  1125. var
  1126. r : tregister;
  1127. begin
  1128. if not shufflescalar(shuffle) then
  1129. internalerror(2014122802);
  1130. if tcgsize2size[fromsize]>tcgsize2size[tosize] then
  1131. internalerror(2014122804);
  1132. case tcgsize2size[fromsize] of
  1133. 4:
  1134. setsubreg(mmreg,R_SUBMMS);
  1135. 8:
  1136. setsubreg(mmreg,R_SUBMMD);
  1137. else
  1138. internalerror(2020101311);
  1139. end;
  1140. if tcgsize2size[fromsize]<tcgsize2size[tosize] then
  1141. r:=makeregsize(intreg,fromsize)
  1142. else
  1143. r:=intreg;
  1144. list.concat(taicpu.op_reg_indexedreg(A_UMOV,r,mmreg,0));
  1145. end;
  1146. procedure tcgaarch64.a_opmm_reg_reg(list: TAsmList; Op: TOpCG; size: tcgsize; src, dst: tregister; shuffle: pmmshuffle);
  1147. begin
  1148. case op of
  1149. { "xor Vx,Vx" is used to initialize global regvars to 0 }
  1150. OP_XOR:
  1151. begin
  1152. if shuffle=nil then
  1153. begin
  1154. dst:=newreg(R_MMREGISTER,getsupreg(dst),R_SUBMM16B);
  1155. src:=newreg(R_MMREGISTER,getsupreg(src),R_SUBMM16B);
  1156. list.concat(taicpu.op_reg_reg_reg(A_EOR,dst,dst,src))
  1157. end
  1158. else if (src<>dst) or
  1159. (reg_cgsize(src)<>size) or
  1160. assigned(shuffle) then
  1161. internalerror(2015011401)
  1162. else
  1163. case size of
  1164. OS_F32,
  1165. OS_F64:
  1166. list.concat(taicpu.op_reg_const(A_MOVI,makeregsize(dst,OS_F64),0));
  1167. else
  1168. internalerror(2015011402);
  1169. end;
  1170. end
  1171. else
  1172. internalerror(2015011403);
  1173. end;
  1174. end;
  1175. procedure tcgaarch64.a_bit_scan_reg_reg(list: TAsmList; reverse: boolean; srcsize, dstsize: tcgsize; src, dst: TRegister);
  1176. var
  1177. bitsize: longint;
  1178. begin
  1179. if srcsize in [OS_64,OS_S64] then
  1180. begin
  1181. bitsize:=64;
  1182. end
  1183. else
  1184. begin
  1185. bitsize:=32;
  1186. end;
  1187. { source is 0 -> dst will have to become 255 }
  1188. list.concat(taicpu.op_reg_const(A_CMP,src,0));
  1189. if reverse then
  1190. begin
  1191. list.Concat(taicpu.op_reg_reg(A_CLZ,makeregsize(dst,srcsize),src));
  1192. { xor 31/63 is the same as setting the lower 5/6 bits to
  1193. "31/63-(lower 5/6 bits of dst)" }
  1194. list.Concat(taicpu.op_reg_reg_const(A_EOR,dst,dst,bitsize-1));
  1195. end
  1196. else
  1197. begin
  1198. list.Concat(taicpu.op_reg_reg(A_RBIT,makeregsize(dst,srcsize),src));
  1199. list.Concat(taicpu.op_reg_reg(A_CLZ,dst,dst));
  1200. end;
  1201. { set dst to -1 if src was 0 }
  1202. list.Concat(taicpu.op_reg_reg_reg_cond(A_CSINV,dst,dst,makeregsize(NR_XZR,dstsize),C_NE));
  1203. { mask the -1 to 255 if src was 0 (anyone find a two-instruction
  1204. branch-free version? All of mine are 3...) }
  1205. list.Concat(taicpu.op_reg_reg(A_UXTB,makeregsize(dst,OS_32),makeregsize(dst,OS_32)));
  1206. end;
  1207. procedure tcgaarch64.a_load_reg_ref_unaligned(list: TAsmList; fromsize, tosize: tcgsize; register: tregister; const ref: treference);
  1208. var
  1209. href: treference;
  1210. hreg1, hreg2, tmpreg: tregister;
  1211. begin
  1212. if fromsize in [OS_64,OS_S64] then
  1213. begin
  1214. { split into two 32 bit stores }
  1215. hreg1:=getintregister(list,OS_32);
  1216. hreg2:=getintregister(list,OS_32);
  1217. a_load_reg_reg(list,OS_32,OS_32,makeregsize(register,OS_32),hreg1);
  1218. a_op_const_reg_reg(list,OP_SHR,OS_64,32,register,makeregsize(hreg2,OS_64));
  1219. if target_info.endian=endian_big then
  1220. begin
  1221. tmpreg:=hreg1;
  1222. hreg1:=hreg2;
  1223. hreg2:=tmpreg;
  1224. end;
  1225. { can we use STP? }
  1226. if (ref.alignment=4) and
  1227. (simple_ref_type(A_STP,OS_32,PF_None,ref)=sr_simple) then
  1228. list.concat(taicpu.op_reg_reg_ref(A_STP,hreg1,hreg2,ref))
  1229. else
  1230. begin
  1231. a_load_reg_ref(list,OS_32,OS_32,hreg1,ref);
  1232. href:=ref;
  1233. inc(href.offset,4);
  1234. a_load_reg_ref(list,OS_32,OS_32,hreg2,href);
  1235. end;
  1236. end
  1237. else
  1238. inherited;
  1239. end;
  1240. procedure tcgaarch64.maybeadjustresult(list: TAsmList; op: topcg; size: tcgsize; dst: tregister);
  1241. const
  1242. overflowops = [OP_MUL,OP_IMUL,OP_SHL,OP_ADD,OP_SUB,OP_NOT,OP_NEG];
  1243. begin
  1244. if (op in overflowops) and
  1245. (size in [OS_8,OS_S8,OS_16,OS_S16]) then
  1246. a_load_reg_reg(list,OS_32,size,makeregsize(dst,OS_32),makeregsize(dst,OS_32))
  1247. end;
  1248. procedure tcgaarch64.a_op_const_reg(list: TAsmList; op: topcg; size: tcgsize; a: tcgint; reg: tregister);
  1249. begin
  1250. optimize_op_const(size,op,a);
  1251. case op of
  1252. OP_NONE:
  1253. exit;
  1254. OP_MOVE:
  1255. a_load_const_reg(list,size,a,reg);
  1256. OP_NEG,OP_NOT:
  1257. internalerror(200306011);
  1258. else
  1259. a_op_const_reg_reg(list,op,size,a,reg,reg);
  1260. end;
  1261. end;
  1262. procedure tcgaarch64.a_op_reg_reg(list:TAsmList;op:topcg;size:tcgsize;src,dst:tregister);
  1263. begin
  1264. Case op of
  1265. OP_NEG,
  1266. OP_NOT:
  1267. begin
  1268. if (op=OP_NOT) and (size in [OS_8,OS_S8]) then
  1269. list.concat(taicpu.op_reg_reg_const(A_EOR,dst,src,255))
  1270. else
  1271. begin
  1272. list.concat(taicpu.op_reg_reg(TOpCG2AsmOpReg[op],dst,src));
  1273. maybeadjustresult(list,op,size,dst);
  1274. end;
  1275. end
  1276. else
  1277. a_op_reg_reg_reg(list,op,size,src,dst,dst);
  1278. end;
  1279. end;
  1280. procedure tcgaarch64.a_op_const_reg_reg(list: TAsmList; op: topcg; size: tcgsize; a: tcgint; src, dst: tregister);
  1281. var
  1282. l: tlocation;
  1283. begin
  1284. a_op_const_reg_reg_checkoverflow(list,op,size,a,src,dst,false,l);
  1285. end;
  1286. procedure tcgaarch64.a_op_reg_reg_reg(list: TAsmList; op: topcg; size: tcgsize; src1, src2, dst: tregister);
  1287. var
  1288. hreg: tregister;
  1289. begin
  1290. { no ROLV opcode... }
  1291. if op=OP_ROL then
  1292. begin
  1293. case size of
  1294. OS_32,OS_S32,
  1295. OS_64,OS_S64:
  1296. begin
  1297. hreg:=getintregister(list,size);
  1298. a_load_const_reg(list,size,tcgsize2size[size]*8,hreg);
  1299. a_op_reg_reg(list,OP_SUB,size,src1,hreg);
  1300. a_op_reg_reg_reg(list,OP_ROR,size,hreg,src2,dst);
  1301. exit;
  1302. end;
  1303. else
  1304. internalerror(2014111005);
  1305. end;
  1306. end
  1307. else if (op=OP_ROR) and
  1308. not(size in [OS_32,OS_S32,OS_64,OS_S64]) then
  1309. internalerror(2014111006);
  1310. if TOpCG2AsmOpReg[op]=A_NONE then
  1311. internalerror(2014111007);
  1312. list.concat(taicpu.op_reg_reg_reg(TOpCG2AsmOpReg[op],dst,src2,src1));
  1313. maybeadjustresult(list,op,size,dst);
  1314. end;
  1315. procedure tcgaarch64.a_op_const_reg_reg_checkoverflow(list: TAsmList; op: topcg; size: tcgsize; a: tcgint; src, dst: tregister; setflags : boolean; var ovloc : tlocation);
  1316. var
  1317. shiftcountmask: longint;
  1318. constreg: tregister;
  1319. begin
  1320. { add/sub instructions have only positive immediate operands }
  1321. if (op in [OP_ADD,OP_SUB]) and
  1322. (a<0) then
  1323. begin
  1324. if op=OP_ADD then
  1325. op:=op_SUB
  1326. else
  1327. op:=OP_ADD;
  1328. { avoid range/overflow error in case a = low(tcgint) }
  1329. {$push}{$r-}{$q-}
  1330. a:=-a;
  1331. {$pop}
  1332. end;
  1333. ovloc.loc:=LOC_VOID;
  1334. optimize_op_const(size,op,a);
  1335. case op of
  1336. OP_NONE:
  1337. begin
  1338. a_load_reg_reg(list,size,size,src,dst);
  1339. exit;
  1340. end;
  1341. OP_MOVE:
  1342. begin
  1343. a_load_const_reg(list,size,a,dst);
  1344. exit;
  1345. end;
  1346. else
  1347. ;
  1348. end;
  1349. case op of
  1350. OP_ADD,
  1351. OP_SUB:
  1352. begin
  1353. handle_reg_imm12_reg(list,TOpCG2AsmOpImm[op],size,src,a,dst,NR_NO,setflags,true);
  1354. { on a 64 bit target, overflows with smaller data types
  1355. are handled via range errors }
  1356. if setflags and
  1357. (size in [OS_64,OS_S64]) then
  1358. begin
  1359. location_reset(ovloc,LOC_FLAGS,OS_8);
  1360. if size=OS_64 then
  1361. if op=OP_ADD then
  1362. ovloc.resflags:=F_CS
  1363. else
  1364. ovloc.resflags:=F_CC
  1365. else
  1366. ovloc.resflags:=F_VS;
  1367. end;
  1368. end;
  1369. OP_OR,
  1370. OP_AND,
  1371. OP_XOR:
  1372. begin
  1373. if not(size in [OS_64,OS_S64]) then
  1374. a:=cardinal(a);
  1375. if is_shifter_const(a,size) then
  1376. list.concat(taicpu.op_reg_reg_const(TOpCG2AsmOpReg[op],dst,src,a))
  1377. else
  1378. begin
  1379. constreg:=getintregister(list,size);
  1380. a_load_const_reg(list,size,a,constreg);
  1381. a_op_reg_reg_reg(list,op,size,constreg,src,dst);
  1382. end;
  1383. end;
  1384. OP_SHL,
  1385. OP_SHR,
  1386. OP_SAR:
  1387. begin
  1388. if size in [OS_64,OS_S64] then
  1389. shiftcountmask:=63
  1390. else
  1391. shiftcountmask:=31;
  1392. if (a and shiftcountmask)<>0 Then
  1393. list.concat(taicpu.op_reg_reg_const(
  1394. TOpCG2AsmOpImm[Op],dst,src,a and shiftcountmask))
  1395. else
  1396. a_load_reg_reg(list,size,size,src,dst);
  1397. if (a and not(tcgint(shiftcountmask)))<>0 then
  1398. internalError(2014112101);
  1399. end;
  1400. OP_ROL,
  1401. OP_ROR:
  1402. begin
  1403. case size of
  1404. OS_32,OS_S32:
  1405. if (a and not(tcgint(31)))<>0 then
  1406. internalError(2014112102);
  1407. OS_64,OS_S64:
  1408. if (a and not(tcgint(63)))<>0 then
  1409. internalError(2014112103);
  1410. else
  1411. internalError(2014112104);
  1412. end;
  1413. { there's only a ror opcode }
  1414. if op=OP_ROL then
  1415. a:=(tcgsize2size[size]*8)-a;
  1416. list.concat(taicpu.op_reg_reg_const(A_ROR,dst,src,a));
  1417. end;
  1418. OP_MUL,
  1419. OP_IMUL,
  1420. OP_DIV,
  1421. OP_IDIV:
  1422. begin
  1423. constreg:=getintregister(list,size);
  1424. a_load_const_reg(list,size,a,constreg);
  1425. a_op_reg_reg_reg_checkoverflow(list,op,size,constreg,src,dst,setflags,ovloc);
  1426. end;
  1427. else
  1428. internalerror(2014111403);
  1429. end;
  1430. maybeadjustresult(list,op,size,dst);
  1431. end;
  1432. procedure tcgaarch64.a_op_reg_reg_reg_checkoverflow(list: TAsmList; op: topcg; size: tcgsize; src1, src2, dst: tregister; setflags : boolean; var ovloc : tlocation);
  1433. var
  1434. tmpreg1, tmpreg2: tregister;
  1435. begin
  1436. ovloc.loc:=LOC_VOID;
  1437. { overflow can only occur with 64 bit calculations on 64 bit cpus }
  1438. if setflags and
  1439. (size in [OS_64,OS_S64]) then
  1440. begin
  1441. case op of
  1442. OP_ADD,
  1443. OP_SUB:
  1444. begin
  1445. list.concat(setoppostfix(taicpu.op_reg_reg_reg(TOpCG2AsmOpReg[op],dst,src2,src1),PF_S));
  1446. ovloc.loc:=LOC_FLAGS;
  1447. if size=OS_64 then
  1448. if op=OP_ADD then
  1449. ovloc.resflags:=F_CS
  1450. else
  1451. ovloc.resflags:=F_CC
  1452. else
  1453. ovloc.resflags:=F_VS;
  1454. { finished }
  1455. exit;
  1456. end;
  1457. OP_MUL:
  1458. begin
  1459. { check whether the upper 64 bit of the 128 bit product is 0 }
  1460. tmpreg1:=getintregister(list,OS_64);
  1461. list.concat(taicpu.op_reg_reg_reg(A_UMULH,tmpreg1,src2,src1));
  1462. list.concat(taicpu.op_reg_const(A_CMP,tmpreg1,0));
  1463. ovloc.loc:=LOC_FLAGS;
  1464. ovloc.resflags:=F_NE;
  1465. { still have to perform the actual multiplication }
  1466. end;
  1467. OP_IMUL:
  1468. begin
  1469. { check whether the upper 64 bits of the 128 bit multiplication
  1470. result have the same value as the replicated sign bit of the
  1471. lower 64 bits }
  1472. tmpreg1:=getintregister(list,OS_64);
  1473. list.concat(taicpu.op_reg_reg_reg(A_SMULH,tmpreg1,src2,src1));
  1474. { calculate lower 64 bits (afterwards, because dst may be
  1475. equal to src1 or src2) }
  1476. a_op_reg_reg_reg(list,op,size,src1,src2,dst);
  1477. { replicate sign bit }
  1478. tmpreg2:=getintregister(list,OS_64);
  1479. a_op_const_reg_reg(list,OP_SAR,OS_S64,63,dst,tmpreg2);
  1480. list.concat(taicpu.op_reg_reg(A_CMP,tmpreg1,tmpreg2));
  1481. ovloc.loc:=LOC_FLAGS;
  1482. ovloc.resflags:=F_NE;
  1483. { finished }
  1484. exit;
  1485. end;
  1486. OP_IDIV,
  1487. OP_DIV:
  1488. begin
  1489. { not handled here, needs div-by-zero check (dividing by zero
  1490. just gives a 0 result on aarch64), and low(int64) div -1
  1491. check for overflow) }
  1492. internalerror(2014122101);
  1493. end;
  1494. else
  1495. internalerror(2019050936);
  1496. end;
  1497. end;
  1498. a_op_reg_reg_reg(list,op,size,src1,src2,dst);
  1499. end;
  1500. {*************** compare instructructions ****************}
  1501. procedure tcgaarch64.a_cmp_const_reg_label(list: TAsmList; size: tcgsize; cmp_op: topcmp; a: tcgint; reg: tregister; l: tasmlabel);
  1502. var
  1503. op: tasmop;
  1504. begin
  1505. if a>=0 then
  1506. op:=A_CMP
  1507. else
  1508. op:=A_CMN;
  1509. { avoid range/overflow error in case a=low(tcgint) }
  1510. {$push}{$r-}{$q-}
  1511. handle_reg_imm12_reg(list,op,size,reg,abs(a),NR_XZR,NR_NO,false,false);
  1512. {$pop}
  1513. a_jmp_cond(list,cmp_op,l);
  1514. end;
  1515. procedure tcgaarch64.a_cmp_reg_reg_label(list: TAsmList; size: tcgsize; cmp_op: topcmp; reg1,reg2: tregister; l: tasmlabel);
  1516. begin
  1517. list.concat(taicpu.op_reg_reg(A_CMP,reg2,reg1));
  1518. a_jmp_cond(list,cmp_op,l);
  1519. end;
  1520. procedure tcgaarch64.a_jmp_always(list: TAsmList; l: TAsmLabel);
  1521. var
  1522. ai: taicpu;
  1523. begin
  1524. ai:=TAiCpu.op_sym(A_B,current_asmdata.RefAsmSymbol(l.name,AT_FUNCTION));
  1525. ai.is_jmp:=true;
  1526. list.Concat(ai);
  1527. end;
  1528. procedure tcgaarch64.a_jmp_name(list: TAsmList; const s: string);
  1529. var
  1530. ai: taicpu;
  1531. begin
  1532. ai:=TAiCpu.op_sym(A_B,current_asmdata.RefAsmSymbol(s,AT_FUNCTION));
  1533. ai.is_jmp:=true;
  1534. list.Concat(ai);
  1535. end;
  1536. procedure tcgaarch64.a_jmp_cond(list: TAsmList; cond: TOpCmp; l: TAsmLabel);
  1537. var
  1538. ai: taicpu;
  1539. begin
  1540. ai:=TAiCpu.op_sym(A_B,l);
  1541. ai.is_jmp:=true;
  1542. ai.SetCondition(TOpCmp2AsmCond[cond]);
  1543. list.Concat(ai);
  1544. end;
  1545. procedure tcgaarch64.a_jmp_flags(list: TAsmList; const f: tresflags; l: tasmlabel);
  1546. var
  1547. ai : taicpu;
  1548. begin
  1549. ai:=Taicpu.op_sym(A_B,l);
  1550. ai.is_jmp:=true;
  1551. ai.SetCondition(flags_to_cond(f));
  1552. list.Concat(ai);
  1553. end;
  1554. procedure tcgaarch64.g_flags2reg(list: TAsmList; size: tcgsize; const f: tresflags; reg: tregister);
  1555. begin
  1556. list.concat(taicpu.op_reg_cond(A_CSET,reg,flags_to_cond(f)));
  1557. end;
  1558. procedure tcgaarch64.g_overflowcheck(list: TAsmList; const loc: tlocation; def: tdef);
  1559. begin
  1560. { we need an explicit overflow location, because there are many
  1561. possibilities (not just the overflow flag, which is only used for
  1562. signed add/sub) }
  1563. internalerror(2014112303);
  1564. end;
  1565. procedure tcgaarch64.g_overflowcheck_loc(list: TAsmList; const loc: tlocation; def: tdef; ovloc : tlocation);
  1566. var
  1567. hl : tasmlabel;
  1568. hflags : tresflags;
  1569. begin
  1570. if not(cs_check_overflow in current_settings.localswitches) then
  1571. exit;
  1572. current_asmdata.getjumplabel(hl);
  1573. case ovloc.loc of
  1574. LOC_FLAGS:
  1575. begin
  1576. hflags:=ovloc.resflags;
  1577. inverse_flags(hflags);
  1578. cg.a_jmp_flags(list,hflags,hl);
  1579. end;
  1580. else
  1581. internalerror(2014112304);
  1582. end;
  1583. a_call_name(list,'FPC_OVERFLOW',false);
  1584. a_label(list,hl);
  1585. end;
  1586. { *********** entry/exit code and address loading ************ }
  1587. function tcgaarch64.save_regs(list: TAsmList; rt: tregistertype; lowsr, highsr: tsuperregister; sub: tsubregister): longint;
  1588. var
  1589. ref: treference;
  1590. sr: tsuperregister;
  1591. pairreg: tregister;
  1592. sehreg,sehregp : TAsmSehDirective;
  1593. begin
  1594. result:=0;
  1595. reference_reset_base(ref,NR_SP,-16,ctempposinvalid,16,[]);
  1596. ref.addressmode:=AM_PREINDEXED;
  1597. pairreg:=NR_NO;
  1598. { for SEH on Win64 we can only store consecutive register pairs, others
  1599. need to be stored with STR }
  1600. if target_info.system=system_aarch64_win64 then
  1601. begin
  1602. if rt=R_INTREGISTER then
  1603. begin
  1604. sehreg:=ash_savereg_x;
  1605. sehregp:=ash_saveregp_x;
  1606. end
  1607. else if rt=R_MMREGISTER then
  1608. begin
  1609. sehreg:=ash_savefreg_x;
  1610. sehregp:=ash_savefregp_x;
  1611. end
  1612. else
  1613. internalerror(2020041304);
  1614. for sr:=lowsr to highsr do
  1615. if sr in rg[rt].used_in_proc then
  1616. if pairreg=NR_NO then
  1617. pairreg:=newreg(rt,sr,sub)
  1618. else
  1619. begin
  1620. inc(result,16);
  1621. if getsupreg(pairreg)=sr-1 then
  1622. begin
  1623. list.concat(taicpu.op_reg_reg_ref(A_STP,pairreg,newreg(rt,sr,sub),ref));
  1624. list.concat(cai_seh_directive.create_reg_offset(sehregp,pairreg,16));
  1625. pairreg:=NR_NO;
  1626. end
  1627. else
  1628. begin
  1629. list.concat(taicpu.op_reg_ref(A_STR,pairreg,ref));
  1630. list.concat(cai_seh_directive.create_reg_offset(sehreg,pairreg,16));
  1631. pairreg:=newreg(rt,sr,sub);
  1632. end;
  1633. end;
  1634. if pairreg<>NR_NO then
  1635. begin
  1636. inc(result,16);
  1637. list.concat(taicpu.op_reg_ref(A_STR,pairreg,ref));
  1638. list.concat(cai_seh_directive.create_reg_offset(sehreg,pairreg,16));
  1639. end;
  1640. end
  1641. else
  1642. begin
  1643. { store all used registers pairwise }
  1644. for sr:=lowsr to highsr do
  1645. if sr in rg[rt].used_in_proc then
  1646. if pairreg=NR_NO then
  1647. pairreg:=newreg(rt,sr,sub)
  1648. else
  1649. begin
  1650. inc(result,16);
  1651. list.concat(taicpu.op_reg_reg_ref(A_STP,pairreg,newreg(rt,sr,sub),ref));
  1652. pairreg:=NR_NO
  1653. end;
  1654. { one left -> store twice (stack must be 16 bytes aligned) }
  1655. if pairreg<>NR_NO then
  1656. begin
  1657. list.concat(taicpu.op_reg_reg_ref(A_STP,pairreg,pairreg,ref));
  1658. inc(result,16);
  1659. end;
  1660. end;
  1661. end;
  1662. procedure FixupOffsets(p:TObject;arg:pointer);
  1663. var
  1664. sym: tabstractnormalvarsym absolute p;
  1665. begin
  1666. if (tsym(p).typ in [paravarsym,localvarsym]) and
  1667. (sym.localloc.loc=LOC_REFERENCE) and
  1668. (sym.localloc.reference.base=NR_STACK_POINTER_REG) then
  1669. begin
  1670. sym.localloc.reference.base:=NR_FRAME_POINTER_REG;
  1671. dec(sym.localloc.reference.offset,PLongint(arg)^);
  1672. end;
  1673. end;
  1674. procedure tcgaarch64.g_stackpointer_alloc(list : TAsmList;localsize : longint);
  1675. var
  1676. href : treference;
  1677. i : integer;
  1678. again : tasmlabel;
  1679. begin
  1680. if localsize>0 then
  1681. begin
  1682. { windows guards only a few pages for stack growing,
  1683. so we have to access every page first }
  1684. if (target_info.system=system_aarch64_win64) and
  1685. (localsize>=winstackpagesize) then
  1686. begin
  1687. if localsize div winstackpagesize<=4 then
  1688. begin
  1689. handle_reg_imm12_reg(list,A_SUB,OS_ADDR,NR_SP,localsize,NR_SP,NR_IP0,false,true);
  1690. for i:=1 to localsize div winstackpagesize do
  1691. begin
  1692. reference_reset_base(href,NR_SP,localsize-i*winstackpagesize+4,ctempposinvalid,4,[]);
  1693. list.concat(Taicpu.op_reg_ref(A_STR,NR_WZR,href));
  1694. end;
  1695. reference_reset_base(href,NR_SP,0,ctempposinvalid,4,[]);
  1696. list.concat(Taicpu.op_reg_ref(A_STR,NR_WZR,href));
  1697. end
  1698. else
  1699. begin
  1700. current_asmdata.getjumplabel(again);
  1701. getcpuregister(list,NR_IP0);
  1702. a_load_const_reg(list,OS_ADDR,localsize div winstackpagesize,NR_IP0);
  1703. a_label(list,again);
  1704. handle_reg_imm12_reg(list,A_SUB,OS_ADDR,NR_SP,winstackpagesize,NR_SP,NR_IP1,false,true);
  1705. reference_reset_base(href,NR_SP,0,ctempposinvalid,4,[]);
  1706. list.concat(Taicpu.op_reg_ref(A_STR,NR_WZR,href));
  1707. list.concat(setoppostfix(Taicpu.op_reg_reg_const(A_SUB,NR_IP0,NR_IP0,1),PF_S));
  1708. a_jmp_cond(list,OC_NE,again);
  1709. handle_reg_imm12_reg(list,A_SUB,OS_ADDR,NR_SP,localsize mod winstackpagesize,NR_SP,NR_IP1,false,true);
  1710. ungetcpuregister(list,NR_IP0);
  1711. end
  1712. end
  1713. else
  1714. begin
  1715. handle_reg_imm12_reg(list,A_SUB,OS_ADDR,NR_SP,localsize,NR_SP,NR_IP0,false,true);
  1716. if target_info.system=system_aarch64_win64 then
  1717. list.concat(cai_seh_directive.create_offset(ash_stackalloc,localsize));
  1718. end;
  1719. end;
  1720. end;
  1721. procedure tcgaarch64.g_proc_entry(list: TAsmList; localsize: longint; nostackframe: boolean);
  1722. var
  1723. hitem: tlinkedlistitem;
  1724. seh_proc: tai_seh_directive;
  1725. templist: TAsmList;
  1726. suppress_endprologue: boolean;
  1727. ref: treference;
  1728. totalstackframesize: longint;
  1729. begin
  1730. { on aarch64, we need to store the link register and the generate a frame pointer if the subroutine either
  1731. - receives parameters on the stack
  1732. - is not a leaf procedure
  1733. - has nested procedures
  1734. - helpers retrieve the stack pointer
  1735. }
  1736. hitem:=list.last;
  1737. { pi_has_unwind_info may already be set at this point if there are
  1738. SEH directives in assembler body. In this case, .seh_endprologue
  1739. is expected to be one of those directives, and not generated here. }
  1740. suppress_endprologue:=(pi_has_unwind_info in current_procinfo.flags);
  1741. if not nostackframe then
  1742. begin
  1743. { stack pointer has to be aligned to 16 bytes at all times }
  1744. localsize:=align(localsize,16);
  1745. if target_info.system=system_aarch64_win64 then
  1746. include(current_procinfo.flags,pi_has_unwind_info);
  1747. if not(pi_no_framepointer_needed in current_procinfo.flags) then
  1748. begin
  1749. { save stack pointer and return address }
  1750. reference_reset_base(ref,NR_SP,-16,ctempposinvalid,16,[]);
  1751. ref.addressmode:=AM_PREINDEXED;
  1752. list.concat(taicpu.op_reg_reg_ref(A_STP,NR_FP,NR_LR,ref));
  1753. current_asmdata.asmcfi.cfa_def_cfa_offset(list,2*sizeof(pint));
  1754. current_asmdata.asmcfi.cfa_offset(list,NR_FP,-16);
  1755. current_asmdata.asmcfi.cfa_offset(list,NR_LR,-8);
  1756. if target_info.system=system_aarch64_win64 then
  1757. list.concat(cai_seh_directive.create_offset(ash_savefplr_x,16));
  1758. { initialise frame pointer }
  1759. if current_procinfo.procdef.proctypeoption<>potype_exceptfilter then
  1760. begin
  1761. a_load_reg_reg(list,OS_ADDR,OS_ADDR,NR_SP,NR_FP);
  1762. current_asmdata.asmcfi.cfa_def_cfa_register(list,NR_FP);
  1763. if target_info.system=system_aarch64_win64 then
  1764. list.concat(cai_seh_directive.create(ash_setfp));
  1765. end
  1766. else
  1767. begin
  1768. gen_load_frame_for_exceptfilter(list);
  1769. localsize:=current_procinfo.maxpushedparasize;
  1770. end;
  1771. end;
  1772. totalstackframesize:=localsize;
  1773. { save modified integer registers }
  1774. inc(totalstackframesize,
  1775. save_regs(list,R_INTREGISTER,RS_X19,RS_X28,R_SUBWHOLE));
  1776. { only the lower 64 bits of the modified vector registers need to be
  1777. saved; if the caller needs the upper 64 bits, it has to save them
  1778. itself }
  1779. inc(totalstackframesize,
  1780. save_regs(list,R_MMREGISTER,RS_D8,RS_D15,R_SUBMMD));
  1781. { allocate stack space }
  1782. if localsize<>0 then
  1783. begin
  1784. localsize:=align(localsize,16);
  1785. current_procinfo.final_localsize:=localsize;
  1786. g_stackpointer_alloc(list,localsize);
  1787. end;
  1788. { By default, we use the frame pointer to access parameters passed via
  1789. the stack and the stack pointer to address local variables and temps
  1790. because
  1791. a) we can use bigger positive than negative offsets (so accessing
  1792. locals via negative offsets from the frame pointer would be less
  1793. efficient)
  1794. b) we don't know the local size while generating the code, so
  1795. accessing the parameters via the stack pointer is not possible
  1796. without copying them
  1797. The problem with this is the get_frame() intrinsic:
  1798. a) it must return the same value as what we pass as parentfp
  1799. parameter, since that's how it's used in the TP-style objects unit
  1800. b) its return value must usable to access all local data from a
  1801. routine (locals and parameters), since it's all the nested
  1802. routines have access to
  1803. c) its return value must be usable to construct a backtrace, as it's
  1804. also used by the exception handling routines
  1805. The solution we use here, based on something similar that's done in
  1806. the MIPS port, is to generate all accesses to locals in the routine
  1807. itself SP-relative, and then after the code is generated and the local
  1808. size is known (namely, here), we change all SP-relative variables/
  1809. parameters into FP-relative ones. This means that they'll be accessed
  1810. less efficiently from nested routines, but those accesses are indirect
  1811. anyway and at least this way they can be accessed at all
  1812. }
  1813. if current_procinfo.has_nestedprocs or
  1814. (
  1815. (target_info.system=system_aarch64_win64) and
  1816. (current_procinfo.flags*[pi_has_implicit_finally,pi_needs_implicit_finally,pi_uses_exceptions]<>[])
  1817. ) then
  1818. begin
  1819. current_procinfo.procdef.localst.SymList.ForEachCall(@FixupOffsets,@totalstackframesize);
  1820. current_procinfo.procdef.parast.SymList.ForEachCall(@FixupOffsets,@totalstackframesize);
  1821. end;
  1822. end;
  1823. if not (pi_has_unwind_info in current_procinfo.flags) then
  1824. exit;
  1825. { Generate unwind data for aarch64-win64 }
  1826. seh_proc:=cai_seh_directive.create_name(ash_proc,current_procinfo.procdef.mangledname);
  1827. if assigned(hitem) then
  1828. list.insertafter(seh_proc,hitem)
  1829. else
  1830. list.insert(seh_proc);
  1831. { the directive creates another section }
  1832. inc(list.section_count);
  1833. templist:=TAsmList.Create;
  1834. if not suppress_endprologue then
  1835. begin
  1836. templist.concat(cai_seh_directive.create(ash_endprologue));
  1837. end;
  1838. if assigned(current_procinfo.endprologue_ai) then
  1839. current_procinfo.aktproccode.insertlistafter(current_procinfo.endprologue_ai,templist)
  1840. else
  1841. list.concatlist(templist);
  1842. templist.free;
  1843. end;
  1844. procedure tcgaarch64.g_maybe_got_init(list : TAsmList);
  1845. begin
  1846. { nothing to do on Darwin or Linux }
  1847. end;
  1848. procedure tcgaarch64.g_restore_registers(list:TAsmList);
  1849. begin
  1850. { done in g_proc_exit }
  1851. end;
  1852. procedure tcgaarch64.load_regs(list: TAsmList; rt: tregistertype; lowsr, highsr: tsuperregister; sub: tsubregister);
  1853. var
  1854. ref: treference;
  1855. sr, highestsetsr: tsuperregister;
  1856. pairreg: tregister;
  1857. i,
  1858. regcount: longint;
  1859. aiarr : array of tai;
  1860. begin
  1861. reference_reset_base(ref,NR_SP,16,ctempposinvalid,16,[]);
  1862. ref.addressmode:=AM_POSTINDEXED;
  1863. regcount:=0;
  1864. { due to SEH on Win64 we can only load consecutive registers and single
  1865. ones are done using LDR, so we need to handle this differently there }
  1866. if target_info.system=system_aarch64_win64 then
  1867. begin
  1868. setlength(aiarr,highsr-lowsr+1);
  1869. pairreg:=NR_NO;
  1870. for sr:=lowsr to highsr do
  1871. if sr in rg[rt].used_in_proc then
  1872. begin
  1873. if pairreg=NR_NO then
  1874. pairreg:=newreg(rt,sr,sub)
  1875. else
  1876. begin
  1877. if getsupreg(pairreg)=sr-1 then
  1878. begin
  1879. aiarr[regcount]:=taicpu.op_reg_reg_ref(A_LDP,pairreg,newreg(rt,sr,sub),ref);
  1880. inc(regcount);
  1881. pairreg:=NR_NO;
  1882. end
  1883. else
  1884. begin
  1885. aiarr[regcount]:=taicpu.op_reg_ref(A_LDR,pairreg,ref);
  1886. inc(regcount);
  1887. pairreg:=newreg(rt,sr,sub);
  1888. end;
  1889. end;
  1890. end;
  1891. if pairreg<>NR_NO then
  1892. begin
  1893. aiarr[regcount]:=taicpu.op_reg_ref(A_LDR,pairreg,ref);
  1894. inc(regcount);
  1895. pairreg:=NR_NO;
  1896. end;
  1897. for i:=regcount-1 downto 0 do
  1898. list.concat(aiarr[i]);
  1899. end
  1900. else
  1901. begin
  1902. { highest reg stored twice? }
  1903. highestsetsr:=RS_NO;
  1904. for sr:=lowsr to highsr do
  1905. if sr in rg[rt].used_in_proc then
  1906. begin
  1907. inc(regcount);
  1908. highestsetsr:=sr;
  1909. end;
  1910. if odd(regcount) then
  1911. begin
  1912. list.concat(taicpu.op_reg_ref(A_LDR,newreg(rt,highestsetsr,sub),ref));
  1913. highestsetsr:=pred(highestsetsr);
  1914. end;
  1915. { load all (other) used registers pairwise }
  1916. pairreg:=NR_NO;
  1917. for sr:=highestsetsr downto lowsr do
  1918. if sr in rg[rt].used_in_proc then
  1919. if pairreg=NR_NO then
  1920. pairreg:=newreg(rt,sr,sub)
  1921. else
  1922. begin
  1923. list.concat(taicpu.op_reg_reg_ref(A_LDP,newreg(rt,sr,sub),pairreg,ref));
  1924. pairreg:=NR_NO
  1925. end;
  1926. end;
  1927. { There can't be any register left }
  1928. if pairreg<>NR_NO then
  1929. internalerror(2014112602);
  1930. end;
  1931. procedure tcgaarch64.g_proc_exit(list : TAsmList;parasize:longint;nostackframe:boolean);
  1932. var
  1933. ref: treference;
  1934. regsstored: boolean;
  1935. sr: tsuperregister;
  1936. begin
  1937. if not(nostackframe) and
  1938. { we do not need an exit stack frame when we never return
  1939. * the final ret is left so the peephole optimizer can easily do call/ret -> jmp or call conversions
  1940. * the entry stack frame must be normally generated because the subroutine could be still left by
  1941. an exception and then the unwinding code might need to restore the registers stored by the entry code
  1942. }
  1943. not(po_noreturn in current_procinfo.procdef.procoptions) then
  1944. begin
  1945. { if no registers have been stored, we don't have to subtract the
  1946. allocated temp space from the stack pointer }
  1947. regsstored:=false;
  1948. for sr:=RS_X19 to RS_X28 do
  1949. if sr in rg[R_INTREGISTER].used_in_proc then
  1950. begin
  1951. regsstored:=true;
  1952. break;
  1953. end;
  1954. if not regsstored then
  1955. for sr:=RS_D8 to RS_D15 do
  1956. if sr in rg[R_MMREGISTER].used_in_proc then
  1957. begin
  1958. regsstored:=true;
  1959. break;
  1960. end;
  1961. { restore registers (and stack pointer) }
  1962. if regsstored then
  1963. begin
  1964. if current_procinfo.final_localsize<>0 then
  1965. handle_reg_imm12_reg(list,A_ADD,OS_ADDR,NR_SP,current_procinfo.final_localsize,NR_SP,NR_IP0,false,true);
  1966. load_regs(list,R_MMREGISTER,RS_D8,RS_D15,R_SUBMMD);
  1967. load_regs(list,R_INTREGISTER,RS_X19,RS_X28,R_SUBWHOLE);
  1968. end
  1969. else if current_procinfo.final_localsize<>0 then
  1970. begin
  1971. { restore stack pointer }
  1972. if pi_no_framepointer_needed in current_procinfo.flags then
  1973. handle_reg_imm12_reg(list,A_ADD,OS_ADDR,current_procinfo.framepointer,current_procinfo.final_localsize,
  1974. current_procinfo.framepointer,NR_IP0,false,true)
  1975. else
  1976. a_load_reg_reg(list,OS_ADDR,OS_ADDR,NR_FP,NR_SP);
  1977. end;
  1978. if not(pi_no_framepointer_needed in current_procinfo.flags) then
  1979. begin
  1980. { restore framepointer and return address }
  1981. reference_reset_base(ref,NR_SP,16,ctempposinvalid,16,[]);
  1982. ref.addressmode:=AM_POSTINDEXED;
  1983. list.concat(taicpu.op_reg_reg_ref(A_LDP,NR_FP,NR_LR,ref));
  1984. end;
  1985. end;
  1986. { return }
  1987. list.concat(taicpu.op_none(A_RET));
  1988. if (pi_has_unwind_info in current_procinfo.flags) then
  1989. begin
  1990. tcpuprocinfo(current_procinfo).dump_scopes(list);
  1991. list.concat(cai_seh_directive.create(ash_endproc));
  1992. end;
  1993. end;
  1994. procedure tcgaarch64.g_save_registers(list : TAsmList);
  1995. begin
  1996. { done in g_proc_entry }
  1997. end;
  1998. { ************* concatcopy ************ }
  1999. procedure tcgaarch64.g_concatcopy_move(list : TAsmList;const source,dest : treference;len : tcgint);
  2000. var
  2001. paraloc1,paraloc2,paraloc3 : TCGPara;
  2002. pd : tprocdef;
  2003. begin
  2004. pd:=search_system_proc('MOVE');
  2005. paraloc1.init;
  2006. paraloc2.init;
  2007. paraloc3.init;
  2008. paramanager.getcgtempparaloc(list,pd,1,paraloc1);
  2009. paramanager.getcgtempparaloc(list,pd,2,paraloc2);
  2010. paramanager.getcgtempparaloc(list,pd,3,paraloc3);
  2011. a_load_const_cgpara(list,OS_SINT,len,paraloc3);
  2012. a_loadaddr_ref_cgpara(list,dest,paraloc2);
  2013. a_loadaddr_ref_cgpara(list,source,paraloc1);
  2014. paramanager.freecgpara(list,paraloc3);
  2015. paramanager.freecgpara(list,paraloc2);
  2016. paramanager.freecgpara(list,paraloc1);
  2017. alloccpuregisters(list,R_INTREGISTER,paramanager.get_volatile_registers_int(pocall_default));
  2018. alloccpuregisters(list,R_MMREGISTER,paramanager.get_volatile_registers_mm(pocall_default));
  2019. a_call_name(list,'FPC_MOVE',false);
  2020. dealloccpuregisters(list,R_MMREGISTER,paramanager.get_volatile_registers_mm(pocall_default));
  2021. dealloccpuregisters(list,R_INTREGISTER,paramanager.get_volatile_registers_int(pocall_default));
  2022. paraloc3.done;
  2023. paraloc2.done;
  2024. paraloc1.done;
  2025. end;
  2026. procedure tcgaarch64.g_concatcopy(list: TAsmList; const source, dest: treference; len: tcgint);
  2027. var
  2028. sourcebasereplaced, destbasereplaced: boolean;
  2029. { get optimal memory operation to use for loading/storing data
  2030. in an unrolled loop }
  2031. procedure getmemop(scaledop, unscaledop: tasmop; const startref, endref: treference; opsize: tcgsize; postfix: toppostfix; out memop: tasmop; out needsimplify: boolean);
  2032. begin
  2033. if (simple_ref_type(scaledop,opsize,postfix,startref)=sr_simple) and
  2034. (simple_ref_type(scaledop,opsize,postfix,endref)=sr_simple) then
  2035. begin
  2036. memop:=unscaledop;
  2037. needsimplify:=true;
  2038. end
  2039. else if (unscaledop<>A_NONE) and
  2040. (simple_ref_type(unscaledop,opsize,postfix,startref)=sr_simple) and
  2041. (simple_ref_type(unscaledop,opsize,postfix,endref)=sr_simple) then
  2042. begin
  2043. memop:=unscaledop;
  2044. needsimplify:=false;
  2045. end
  2046. else
  2047. begin
  2048. memop:=scaledop;
  2049. needsimplify:=true;
  2050. end;
  2051. end;
  2052. { adjust the offset and/or addressing mode after a load/store so it's
  2053. correct for the next one of the same size }
  2054. procedure updaterefafterloadstore(var ref: treference; oplen: longint);
  2055. begin
  2056. case ref.addressmode of
  2057. AM_OFFSET:
  2058. inc(ref.offset,oplen);
  2059. AM_POSTINDEXED:
  2060. { base register updated by instruction, next offset can remain
  2061. the same }
  2062. ;
  2063. AM_PREINDEXED:
  2064. begin
  2065. { base register updated by instruction -> next instruction can
  2066. use post-indexing with offset = sizeof(operation) }
  2067. ref.offset:=0;
  2068. ref.addressmode:=AM_OFFSET;
  2069. end;
  2070. end;
  2071. end;
  2072. { generate a load/store and adjust the reference offset to the next
  2073. memory location if necessary }
  2074. procedure genloadstore(list: TAsmList; op: tasmop; reg: tregister; var ref: treference; postfix: toppostfix; opsize: tcgsize);
  2075. begin
  2076. list.concat(setoppostfix(taicpu.op_reg_ref(op,reg,ref),postfix));
  2077. updaterefafterloadstore(ref,tcgsize2size[opsize]);
  2078. end;
  2079. { generate a dual load/store (ldp/stp) and adjust the reference offset to
  2080. the next memory location if necessary }
  2081. procedure gendualloadstore(list: TAsmList; op: tasmop; reg1, reg2: tregister; var ref: treference; postfix: toppostfix; opsize: tcgsize);
  2082. begin
  2083. list.concat(setoppostfix(taicpu.op_reg_reg_ref(op,reg1,reg2,ref),postfix));
  2084. updaterefafterloadstore(ref,tcgsize2size[opsize]*2);
  2085. end;
  2086. { turn a reference into a pre- or post-indexed reference for use in a
  2087. load/store of a particular size }
  2088. procedure makesimpleforcopy(list: TAsmList; var scaledop: tasmop; opsize: tcgsize; postfix: toppostfix; forcepostindexing: boolean; var ref: treference; var basereplaced: boolean);
  2089. var
  2090. tmpreg: tregister;
  2091. scaledoffset: longint;
  2092. orgaddressmode: taddressmode;
  2093. begin
  2094. scaledoffset:=tcgsize2size[opsize];
  2095. if scaledop in [A_LDP,A_STP] then
  2096. scaledoffset:=scaledoffset*2;
  2097. { can we use the reference as post-indexed without changes? }
  2098. if forcepostindexing then
  2099. begin
  2100. orgaddressmode:=ref.addressmode;
  2101. ref.addressmode:=AM_POSTINDEXED;
  2102. if (orgaddressmode=AM_POSTINDEXED) or
  2103. ((ref.offset=0) and
  2104. (simple_ref_type(scaledop,opsize,postfix,ref)=sr_simple)) then
  2105. begin
  2106. { just change the post-indexed offset to the access size }
  2107. ref.offset:=scaledoffset;
  2108. { and replace the base register if that didn't happen yet
  2109. (could be sp or a regvar) }
  2110. if not basereplaced then
  2111. begin
  2112. tmpreg:=getaddressregister(list);
  2113. a_load_reg_reg(list,OS_ADDR,OS_ADDR,ref.base,tmpreg);
  2114. ref.base:=tmpreg;
  2115. basereplaced:=true;
  2116. end;
  2117. exit;
  2118. end;
  2119. ref.addressmode:=orgaddressmode;
  2120. end;
  2121. {$ifdef dummy}
  2122. This could in theory be useful in case you have a concatcopy from
  2123. e.g. x1+255 to x1+267 *and* the reference is aligned, but this seems
  2124. very unlikely. Disabled because it still needs fixes, as it
  2125. also generates pre-indexed loads right now at the very end for the
  2126. left-over gencopies
  2127. { can we turn it into a pre-indexed reference for free? (after the
  2128. first operation, it will be turned into an offset one) }
  2129. if not forcepostindexing and
  2130. (ref.offset<>0) then
  2131. begin
  2132. orgaddressmode:=ref.addressmode;
  2133. ref.addressmode:=AM_PREINDEXED;
  2134. tmpreg:=ref.base;
  2135. if not basereplaced and
  2136. (ref.base=tmpreg) then
  2137. begin
  2138. tmpreg:=getaddressregister(list);
  2139. a_load_reg_reg(list,OS_ADDR,OS_ADDR,ref.base,tmpreg);
  2140. ref.base:=tmpreg;
  2141. basereplaced:=true;
  2142. end;
  2143. if simple_ref_type(scaledop,opsize,postfix,ref)<>sr_simple then
  2144. make_simple_ref(list,scaledop,opsize,postfix,ref,NR_NO);
  2145. exit;
  2146. end;
  2147. {$endif dummy}
  2148. if not forcepostindexing then
  2149. begin
  2150. ref.addressmode:=AM_OFFSET;
  2151. make_simple_ref(list,scaledop,opsize,postfix,ref,NR_NO);
  2152. { this may still cause problems if the final offset is no longer
  2153. a simple ref; it's a bit complicated to pass all information
  2154. through at all places and check that here, so play safe: we
  2155. currently never generate unrolled copies for more than 64
  2156. bytes (32 with non-double-register copies) }
  2157. if ref.index=NR_NO then
  2158. begin
  2159. if ((scaledop in [A_LDP,A_STP]) and
  2160. (ref.offset<((64-8)*tcgsize2size[opsize]))) or
  2161. ((scaledop in [A_LDUR,A_STUR]) and
  2162. (ref.offset<(255-8*tcgsize2size[opsize]))) or
  2163. ((scaledop in [A_LDR,A_STR]) and
  2164. (ref.offset<((4096-8)*tcgsize2size[opsize]))) then
  2165. exit;
  2166. end;
  2167. end;
  2168. tmpreg:=getaddressregister(list);
  2169. a_loadaddr_ref_reg(list,ref,tmpreg);
  2170. basereplaced:=true;
  2171. if forcepostindexing then
  2172. begin
  2173. reference_reset_base(ref,tmpreg,scaledoffset,ref.temppos,ref.alignment,ref.volatility);
  2174. ref.addressmode:=AM_POSTINDEXED;
  2175. end
  2176. else
  2177. begin
  2178. reference_reset_base(ref,tmpreg,0,ref.temppos,ref.alignment,ref.volatility);
  2179. ref.addressmode:=AM_OFFSET;
  2180. end
  2181. end;
  2182. { prepare a reference for use by gencopy. This is done both after the
  2183. unrolled and regular copy loop -> get rid of post-indexing mode, make
  2184. sure ref is valid }
  2185. procedure preparecopy(list: tasmlist; scaledop, unscaledop: tasmop; var ref: treference; opsize: tcgsize; postfix: toppostfix; out op: tasmop; var basereplaced: boolean);
  2186. var
  2187. simplify: boolean;
  2188. begin
  2189. if ref.addressmode=AM_POSTINDEXED then
  2190. ref.offset:=tcgsize2size[opsize];
  2191. getmemop(scaledop,scaledop,ref,ref,opsize,postfix,op,simplify);
  2192. if simplify then
  2193. begin
  2194. makesimpleforcopy(list,scaledop,opsize,postfix,false,ref,basereplaced);
  2195. op:=scaledop;
  2196. end;
  2197. end;
  2198. { generate a copy from source to dest of size opsize/postfix }
  2199. procedure gencopy(list: TAsmList; var source, dest: treference; postfix: toppostfix; opsize: tcgsize);
  2200. var
  2201. reg: tregister;
  2202. loadop, storeop: tasmop;
  2203. begin
  2204. preparecopy(list,A_LDR,A_LDUR,source,opsize,postfix,loadop,sourcebasereplaced);
  2205. preparecopy(list,A_STR,A_STUR,dest,opsize,postfix,storeop,destbasereplaced);
  2206. reg:=getintregister(list,opsize);
  2207. genloadstore(list,loadop,reg,source,postfix,opsize);
  2208. genloadstore(list,storeop,reg,dest,postfix,opsize);
  2209. end;
  2210. { copy the leftovers after an unrolled or regular copy loop }
  2211. procedure gencopyleftovers(list: TAsmList; var source, dest: treference; len: longint);
  2212. begin
  2213. { stop post-indexing if we did so in the loop, since in that case all
  2214. offsets definitely can be represented now }
  2215. if source.addressmode=AM_POSTINDEXED then
  2216. begin
  2217. source.addressmode:=AM_OFFSET;
  2218. source.offset:=0;
  2219. end;
  2220. if dest.addressmode=AM_POSTINDEXED then
  2221. begin
  2222. dest.addressmode:=AM_OFFSET;
  2223. dest.offset:=0;
  2224. end;
  2225. { transfer the leftovers }
  2226. if len>=8 then
  2227. begin
  2228. dec(len,8);
  2229. gencopy(list,source,dest,PF_NONE,OS_64);
  2230. end;
  2231. if len>=4 then
  2232. begin
  2233. dec(len,4);
  2234. gencopy(list,source,dest,PF_NONE,OS_32);
  2235. end;
  2236. if len>=2 then
  2237. begin
  2238. dec(len,2);
  2239. gencopy(list,source,dest,PF_H,OS_16);
  2240. end;
  2241. if len>=1 then
  2242. begin
  2243. dec(len);
  2244. gencopy(list,source,dest,PF_B,OS_8);
  2245. end;
  2246. end;
  2247. const
  2248. { load_length + loop dec + cbnz }
  2249. loopoverhead=12;
  2250. { loop overhead + load + store }
  2251. totallooplen=loopoverhead + 8;
  2252. var
  2253. totalalign: longint;
  2254. maxlenunrolled: tcgint;
  2255. loadop, storeop: tasmop;
  2256. opsize: tcgsize;
  2257. postfix: toppostfix;
  2258. tmpsource, tmpdest: treference;
  2259. scaledstoreop, unscaledstoreop,
  2260. scaledloadop, unscaledloadop: tasmop;
  2261. regs: array[1..8] of tregister;
  2262. countreg: tregister;
  2263. i, regcount: longint;
  2264. hl: tasmlabel;
  2265. simplifysource, simplifydest: boolean;
  2266. begin
  2267. if len=0 then
  2268. exit;
  2269. sourcebasereplaced:=false;
  2270. destbasereplaced:=false;
  2271. { maximum common alignment }
  2272. totalalign:=max(1,newalignment(source.alignment,dest.alignment));
  2273. { use a simple load/store? }
  2274. if (len in [1,2,4,8]) and
  2275. ((totalalign>=(len div 2)) or
  2276. (source.alignment=len) or
  2277. (dest.alignment=len)) then
  2278. begin
  2279. opsize:=int_cgsize(len);
  2280. a_load_ref_ref(list,opsize,opsize,source,dest);
  2281. exit;
  2282. end;
  2283. { alignment > length is not useful, and would break some checks below }
  2284. while totalalign>len do
  2285. totalalign:=totalalign div 2;
  2286. { operation sizes to use based on common alignment }
  2287. case totalalign of
  2288. 1:
  2289. begin
  2290. postfix:=PF_B;
  2291. opsize:=OS_8;
  2292. end;
  2293. 2:
  2294. begin
  2295. postfix:=PF_H;
  2296. opsize:=OS_16;
  2297. end;
  2298. 4:
  2299. begin
  2300. postfix:=PF_None;
  2301. opsize:=OS_32;
  2302. end
  2303. else
  2304. begin
  2305. totalalign:=8;
  2306. postfix:=PF_None;
  2307. opsize:=OS_64;
  2308. end;
  2309. end;
  2310. { maximum length to handled with an unrolled loop (4 loads + 4 stores) }
  2311. maxlenunrolled:=min(totalalign,8)*4;
  2312. { ldp/stp -> 2 registers per instruction }
  2313. if (totalalign>=4) and
  2314. (len>=totalalign*2) then
  2315. begin
  2316. maxlenunrolled:=maxlenunrolled*2;
  2317. scaledstoreop:=A_STP;
  2318. scaledloadop:=A_LDP;
  2319. unscaledstoreop:=A_NONE;
  2320. unscaledloadop:=A_NONE;
  2321. end
  2322. else
  2323. begin
  2324. scaledstoreop:=A_STR;
  2325. scaledloadop:=A_LDR;
  2326. unscaledstoreop:=A_STUR;
  2327. unscaledloadop:=A_LDUR;
  2328. end;
  2329. { we only need 4 instructions extra to call FPC_MOVE }
  2330. if cs_opt_size in current_settings.optimizerswitches then
  2331. maxlenunrolled:=maxlenunrolled div 2;
  2332. if (len>maxlenunrolled) and
  2333. (len>totalalign*8) and
  2334. (pi_do_call in current_procinfo.flags) then
  2335. begin
  2336. g_concatcopy_move(list,source,dest,len);
  2337. exit;
  2338. end;
  2339. simplifysource:=true;
  2340. simplifydest:=true;
  2341. tmpsource:=source;
  2342. tmpdest:=dest;
  2343. { can we directly encode all offsets in an unrolled loop? }
  2344. if len<=maxlenunrolled then
  2345. begin
  2346. {$ifdef extdebug}
  2347. list.concat(tai_comment.Create(strpnew('concatcopy unrolled loop; len/opsize/align: '+tostr(len)+'/'+tostr(tcgsize2size[opsize])+'/'+tostr(totalalign))));
  2348. {$endif extdebug}
  2349. { the leftovers will be handled separately -> -(len mod opsize) }
  2350. inc(tmpsource.offset,len-(len mod tcgsize2size[opsize]));
  2351. { additionally, the last regular load/store will be at
  2352. offset+len-opsize (if len-(len mod opsize)>len) }
  2353. if tmpsource.offset>source.offset then
  2354. dec(tmpsource.offset,tcgsize2size[opsize]);
  2355. getmemop(scaledloadop,unscaledloadop,source,tmpsource,opsize,postfix,loadop,simplifysource);
  2356. inc(tmpdest.offset,len-(len mod tcgsize2size[opsize]));
  2357. if tmpdest.offset>dest.offset then
  2358. dec(tmpdest.offset,tcgsize2size[opsize]);
  2359. getmemop(scaledstoreop,unscaledstoreop,dest,tmpdest,opsize,postfix,storeop,simplifydest);
  2360. tmpsource:=source;
  2361. tmpdest:=dest;
  2362. { if we can't directly encode all offsets, simplify }
  2363. if simplifysource then
  2364. begin
  2365. loadop:=scaledloadop;
  2366. makesimpleforcopy(list,loadop,opsize,postfix,false,tmpsource,sourcebasereplaced);
  2367. end;
  2368. if simplifydest then
  2369. begin
  2370. storeop:=scaledstoreop;
  2371. makesimpleforcopy(list,storeop,opsize,postfix,false,tmpdest,destbasereplaced);
  2372. end;
  2373. regcount:=len div tcgsize2size[opsize];
  2374. { in case we transfer two registers at a time, we copy an even
  2375. number of registers }
  2376. if loadop=A_LDP then
  2377. regcount:=regcount and not(1);
  2378. { initialise for dfa }
  2379. regs[low(regs)]:=NR_NO;
  2380. { max 4 loads/stores -> max 8 registers (in case of ldp/stdp) }
  2381. for i:=1 to regcount do
  2382. regs[i]:=getintregister(list,opsize);
  2383. if loadop=A_LDP then
  2384. begin
  2385. { load registers }
  2386. for i:=1 to (regcount div 2) do
  2387. gendualloadstore(list,loadop,regs[i*2-1],regs[i*2],tmpsource,postfix,opsize);
  2388. { store registers }
  2389. for i:=1 to (regcount div 2) do
  2390. gendualloadstore(list,storeop,regs[i*2-1],regs[i*2],tmpdest,postfix,opsize);
  2391. end
  2392. else
  2393. begin
  2394. for i:=1 to regcount do
  2395. genloadstore(list,loadop,regs[i],tmpsource,postfix,opsize);
  2396. for i:=1 to regcount do
  2397. genloadstore(list,storeop,regs[i],tmpdest,postfix,opsize);
  2398. end;
  2399. { leftover }
  2400. len:=len-regcount*tcgsize2size[opsize];
  2401. {$ifdef extdebug}
  2402. list.concat(tai_comment.Create(strpnew('concatcopy unrolled loop leftover: '+tostr(len))));
  2403. {$endif extdebug}
  2404. end
  2405. else
  2406. begin
  2407. {$ifdef extdebug}
  2408. list.concat(tai_comment.Create(strpnew('concatcopy regular loop; len/align: '+tostr(len)+'/'+tostr(totalalign))));
  2409. {$endif extdebug}
  2410. { regular loop -> definitely use post-indexing }
  2411. loadop:=scaledloadop;
  2412. makesimpleforcopy(list,loadop,opsize,postfix,true,tmpsource,sourcebasereplaced);
  2413. storeop:=scaledstoreop;
  2414. makesimpleforcopy(list,storeop,opsize,postfix,true,tmpdest,destbasereplaced);
  2415. current_asmdata.getjumplabel(hl);
  2416. countreg:=getintregister(list,OS_32);
  2417. if loadop=A_LDP then
  2418. a_load_const_reg(list,OS_32,len div (tcgsize2size[opsize]*2),countreg)
  2419. else
  2420. a_load_const_reg(list,OS_32,len div tcgsize2size[opsize],countreg);
  2421. a_label(list,hl);
  2422. a_op_const_reg(list,OP_SUB,OS_32,1,countreg);
  2423. if loadop=A_LDP then
  2424. begin
  2425. regs[1]:=getintregister(list,opsize);
  2426. regs[2]:=getintregister(list,opsize);
  2427. gendualloadstore(list,loadop,regs[1],regs[2],tmpsource,postfix,opsize);
  2428. gendualloadstore(list,storeop,regs[1],regs[2],tmpdest,postfix,opsize);
  2429. end
  2430. else
  2431. begin
  2432. regs[1]:=getintregister(list,opsize);
  2433. genloadstore(list,loadop,regs[1],tmpsource,postfix,opsize);
  2434. genloadstore(list,storeop,regs[1],tmpdest,postfix,opsize);
  2435. end;
  2436. list.concat(taicpu.op_reg_sym_ofs(A_CBNZ,countreg,hl,0));
  2437. if loadop=A_LDP then
  2438. len:=len mod (tcgsize2size[opsize]*2)
  2439. else
  2440. len:=len mod tcgsize2size[opsize];
  2441. end;
  2442. gencopyleftovers(list,tmpsource,tmpdest,len);
  2443. end;
  2444. procedure tcgaarch64.g_adjust_self_value(list:TAsmList;procdef: tprocdef;ioffset: tcgint);
  2445. begin
  2446. { This method is integrated into g_intf_wrapper and shouldn't be called separately }
  2447. InternalError(2013020102);
  2448. end;
  2449. procedure tcgaarch64.g_check_for_fpu_exception(list: TAsmList;force,clear : boolean);
  2450. var
  2451. r, tmpreg: TRegister;
  2452. ai: taicpu;
  2453. l1,l2: TAsmLabel;
  2454. begin
  2455. { so far, we assume all flavours of AArch64 need explicit floating point exception checking }
  2456. if ((cs_check_fpu_exceptions in current_settings.localswitches) and
  2457. (force or current_procinfo.FPUExceptionCheckNeeded)) then
  2458. begin
  2459. r:=getintregister(list,OS_INT);
  2460. tmpreg:=getintregister(list,OS_INT);
  2461. list.concat(taicpu.op_reg_reg(A_MRS,r,NR_FPSR));
  2462. list.concat(taicpu.op_reg_reg_const(A_AND,tmpreg,r,$1f));
  2463. current_asmdata.getjumplabel(l1);
  2464. current_asmdata.getjumplabel(l2);
  2465. ai:=taicpu.op_reg_sym_ofs(A_CBNZ,tmpreg,l1,0);
  2466. ai.is_jmp:=true;
  2467. list.concat(ai);
  2468. list.concat(taicpu.op_reg_reg_const(A_AND,tmpreg,r,$80));
  2469. ai:=taicpu.op_reg_sym_ofs(A_CBZ,tmpreg,l2,0);
  2470. ai.is_jmp:=true;
  2471. list.concat(ai);
  2472. a_label(list,l1);
  2473. alloccpuregisters(list,R_INTREGISTER,paramanager.get_volatile_registers_int(pocall_default));
  2474. cg.a_call_name(list,'FPC_THROWFPUEXCEPTION',false);
  2475. dealloccpuregisters(list,R_INTREGISTER,paramanager.get_volatile_registers_int(pocall_default));
  2476. a_label(list,l2);
  2477. if clear then
  2478. current_procinfo.FPUExceptionCheckNeeded:=false;
  2479. end;
  2480. end;
  2481. procedure tcgaarch64.g_profilecode(list : TAsmList);
  2482. begin
  2483. if target_info.system = system_aarch64_linux then
  2484. begin
  2485. list.concat(taicpu.op_reg_reg(A_MOV,NR_X0,NR_X30));
  2486. a_call_name(list,'_mcount',false);
  2487. end
  2488. else
  2489. internalerror(2020021901);
  2490. end;
  2491. procedure create_codegen;
  2492. begin
  2493. cg:=tcgaarch64.Create;
  2494. cg128:=tcg128.Create;
  2495. end;
  2496. end.