aoptx86.pas 673 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584135851358613587135881358913590135911359213593135941359513596135971359813599136001360113602136031360413605136061360713608136091361013611136121361313614136151361613617136181361913620136211362213623136241362513626136271362813629136301363113632136331363413635136361363713638136391364013641136421364313644136451364613647136481364913650136511365213653136541365513656136571365813659136601366113662136631366413665136661366713668136691367013671136721367313674136751367613677136781367913680136811368213683136841368513686136871368813689136901369113692136931369413695136961369713698136991370013701137021370313704137051370613707137081370913710137111371213713137141371513716137171371813719137201372113722137231372413725137261372713728137291373013731137321373313734137351373613737137381373913740137411374213743137441374513746137471374813749137501375113752137531375413755137561375713758137591376013761137621376313764137651376613767137681376913770137711377213773137741377513776137771377813779137801378113782137831378413785137861378713788137891379013791137921379313794137951379613797137981379913800138011380213803138041380513806138071380813809138101381113812138131381413815138161381713818138191382013821138221382313824138251382613827138281382913830138311383213833138341383513836138371383813839138401384113842138431384413845138461384713848138491385013851138521385313854138551385613857138581385913860138611386213863138641386513866138671386813869138701387113872138731387413875138761387713878138791388013881138821388313884138851388613887138881388913890138911389213893138941389513896138971389813899139001390113902139031390413905139061390713908139091391013911139121391313914139151391613917139181391913920139211392213923139241392513926139271392813929139301393113932139331393413935139361393713938139391394013941139421394313944139451394613947139481394913950139511395213953139541395513956139571395813959139601396113962139631396413965139661396713968139691397013971139721397313974139751397613977139781397913980139811398213983139841398513986139871398813989139901399113992139931399413995139961399713998139991400014001140021400314004140051400614007140081400914010140111401214013140141401514016140171401814019140201402114022140231402414025140261402714028140291403014031140321403314034140351403614037140381403914040140411404214043140441404514046140471404814049140501405114052140531405414055140561405714058140591406014061140621406314064140651406614067140681406914070140711407214073140741407514076140771407814079140801408114082140831408414085140861408714088140891409014091140921409314094140951409614097140981409914100141011410214103141041410514106141071410814109141101411114112141131411414115141161411714118141191412014121141221412314124141251412614127141281412914130141311413214133141341413514136141371413814139141401414114142141431414414145141461414714148141491415014151141521415314154141551415614157141581415914160141611416214163141641416514166141671416814169141701417114172141731417414175141761417714178141791418014181141821418314184141851418614187141881418914190141911419214193141941419514196141971419814199142001420114202142031420414205142061420714208142091421014211142121421314214142151421614217142181421914220142211422214223142241422514226142271422814229142301423114232142331423414235142361423714238142391424014241142421424314244142451424614247142481424914250142511425214253142541425514256142571425814259142601426114262142631426414265142661426714268142691427014271142721427314274142751427614277142781427914280142811428214283142841428514286142871428814289142901429114292142931429414295142961429714298142991430014301143021430314304143051430614307143081430914310143111431214313143141431514316143171431814319143201432114322143231432414325143261432714328143291433014331143321433314334143351433614337143381433914340143411434214343143441434514346143471434814349143501435114352143531435414355143561435714358143591436014361143621436314364143651436614367143681436914370143711437214373143741437514376143771437814379143801438114382143831438414385143861438714388143891439014391143921439314394143951439614397143981439914400144011440214403144041440514406144071440814409144101441114412144131441414415144161441714418144191442014421144221442314424144251442614427144281442914430144311443214433144341443514436144371443814439144401444114442144431444414445144461444714448144491445014451144521445314454144551445614457144581445914460144611446214463144641446514466144671446814469144701447114472144731447414475144761447714478144791448014481144821448314484144851448614487144881448914490144911449214493144941449514496144971449814499145001450114502145031450414505145061450714508145091451014511145121451314514145151451614517145181451914520145211452214523145241452514526145271452814529145301453114532145331453414535145361453714538145391454014541145421454314544145451454614547145481454914550145511455214553145541455514556145571455814559145601456114562145631456414565145661456714568145691457014571145721457314574145751457614577145781457914580145811458214583145841458514586145871458814589145901459114592145931459414595145961459714598145991460014601146021460314604146051460614607146081460914610146111461214613146141461514616146171461814619146201462114622146231462414625146261462714628146291463014631146321463314634146351463614637146381463914640146411464214643146441464514646146471464814649146501465114652146531465414655146561465714658146591466014661146621466314664146651466614667146681466914670146711467214673146741467514676146771467814679146801468114682146831468414685146861468714688146891469014691146921469314694146951469614697146981469914700147011470214703147041470514706147071470814709147101471114712147131471414715147161471714718147191472014721147221472314724147251472614727147281472914730147311473214733147341473514736147371473814739147401474114742147431474414745147461474714748147491475014751147521475314754147551475614757147581475914760147611476214763147641476514766147671476814769147701477114772147731477414775147761477714778147791478014781147821478314784147851478614787147881478914790147911479214793147941479514796147971479814799148001480114802148031480414805148061480714808148091481014811148121481314814148151481614817148181481914820148211482214823148241482514826148271482814829148301483114832148331483414835148361483714838148391484014841148421484314844148451484614847148481484914850148511485214853148541485514856148571485814859148601486114862148631486414865148661486714868148691487014871148721487314874148751487614877148781487914880148811488214883148841488514886148871488814889148901489114892148931489414895148961489714898148991490014901149021490314904149051490614907149081490914910149111491214913149141491514916149171491814919149201492114922149231492414925149261492714928149291493014931149321493314934149351493614937149381493914940149411494214943149441494514946149471494814949149501495114952149531495414955149561495714958149591496014961149621496314964149651496614967149681496914970149711497214973149741497514976149771497814979149801498114982149831498414985149861498714988149891499014991149921499314994149951499614997149981499915000150011500215003150041500515006150071500815009150101501115012150131501415015150161501715018150191502015021150221502315024150251502615027150281502915030150311503215033150341503515036150371503815039150401504115042150431504415045150461504715048150491505015051150521505315054150551505615057150581505915060150611506215063150641506515066150671506815069150701507115072150731507415075150761507715078150791508015081150821508315084150851508615087150881508915090150911509215093150941509515096150971509815099151001510115102151031510415105151061510715108151091511015111151121511315114151151511615117151181511915120151211512215123151241512515126151271512815129151301513115132151331513415135151361513715138151391514015141151421514315144151451514615147151481514915150151511515215153151541515515156151571515815159151601516115162151631516415165151661516715168151691517015171151721517315174151751517615177151781517915180151811518215183151841518515186151871518815189151901519115192151931519415195151961519715198151991520015201152021520315204152051520615207152081520915210152111521215213152141521515216152171521815219152201522115222152231522415225152261522715228152291523015231152321523315234152351523615237152381523915240152411524215243152441524515246152471524815249152501525115252152531525415255152561525715258152591526015261152621526315264152651526615267152681526915270152711527215273152741527515276152771527815279152801528115282152831528415285152861528715288152891529015291152921529315294152951529615297152981529915300153011530215303153041530515306153071530815309153101531115312153131531415315153161531715318153191532015321153221532315324153251532615327153281532915330153311533215333153341533515336153371533815339153401534115342153431534415345153461534715348153491535015351153521535315354153551535615357153581535915360153611536215363153641536515366153671536815369153701537115372153731537415375153761537715378153791538015381153821538315384153851538615387153881538915390153911539215393153941539515396153971539815399154001540115402154031540415405154061540715408154091541015411154121541315414154151541615417154181541915420154211542215423154241542515426154271542815429154301543115432154331543415435154361543715438154391544015441154421544315444154451544615447154481544915450154511545215453154541545515456154571545815459154601546115462154631546415465154661546715468154691547015471154721547315474154751547615477154781547915480154811548215483154841548515486154871548815489154901549115492154931549415495154961549715498154991550015501155021550315504155051550615507155081550915510155111551215513155141551515516155171551815519155201552115522155231552415525155261552715528155291553015531155321553315534155351553615537155381553915540155411554215543155441554515546155471554815549155501555115552155531555415555155561555715558155591556015561155621556315564155651556615567155681556915570155711557215573155741557515576155771557815579155801558115582155831558415585155861558715588155891559015591155921559315594155951559615597155981559915600156011560215603156041560515606156071560815609156101561115612156131561415615156161561715618156191562015621156221562315624156251562615627156281562915630156311563215633156341563515636156371563815639156401564115642156431564415645156461564715648156491565015651156521565315654156551565615657156581565915660156611566215663156641566515666156671566815669
  1. {
  2. Copyright (c) 1998-2002 by Florian Klaempfl and Jonas Maebe
  3. This unit contains the peephole optimizer.
  4. This program is free software; you can redistribute it and/or modify
  5. it under the terms of the GNU General Public License as published by
  6. the Free Software Foundation; either version 2 of the License, or
  7. (at your option) any later version.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. GNU General Public License for more details.
  12. You should have received a copy of the GNU General Public License
  13. along with this program; if not, write to the Free Software
  14. Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  15. ****************************************************************************
  16. }
  17. unit aoptx86;
  18. {$i fpcdefs.inc}
  19. { $define DEBUG_AOPTCPU}
  20. {$ifdef EXTDEBUG}
  21. {$define DEBUG_AOPTCPU}
  22. {$endif EXTDEBUG}
  23. interface
  24. uses
  25. globtype,cclasses,
  26. cpubase,
  27. aasmtai,aasmcpu,
  28. cgbase,cgutils,
  29. aopt,aoptobj;
  30. type
  31. TOptsToCheck = (
  32. aoc_MovAnd2Mov_3,
  33. aoc_ForceNewIteration
  34. );
  35. TX86AsmOptimizer = class(TAsmOptimizer)
  36. { some optimizations are very expensive to check, so the
  37. pre opt pass can be used to set some flags, depending on the found
  38. instructions if it is worth to check a certain optimization }
  39. OptsToCheck : set of TOptsToCheck;
  40. function RegLoadedWithNewValue(reg : tregister; hp : tai) : boolean; override;
  41. function InstructionLoadsFromReg(const reg : TRegister; const hp : tai) : boolean; override;
  42. class function RegReadByInstruction(reg : TRegister; hp : tai) : boolean; static;
  43. function RegInInstruction(Reg: TRegister; p1: tai): Boolean;override;
  44. function GetNextInstructionUsingReg(Current: tai; out Next: tai; reg: TRegister): Boolean;
  45. { Identical to GetNextInstructionUsingReg, but returns a value indicating
  46. how many instructions away that Next is from Current is.
  47. 0 = failure, equivalent to False in GetNextInstructionUsingReg }
  48. function GetNextInstructionUsingRegCount(Current: tai; out Next: tai; reg: TRegister): Cardinal;
  49. { This version of GetNextInstructionUsingReg will look across conditional jumps,
  50. potentially allowing further optimisation (although it might need to know if
  51. it crossed a conditional jump. }
  52. function GetNextInstructionUsingRegCond(Current: tai; out Next: tai; reg: TRegister; var JumpTracking: TLinkedList; var CrossJump: Boolean): Boolean;
  53. {
  54. In comparison with GetNextInstructionUsingReg, GetNextInstructionUsingRegTrackingUse tracks
  55. the use of a register by allocs/dealloc, so it can ignore calls.
  56. In the following example, GetNextInstructionUsingReg will return the second movq,
  57. GetNextInstructionUsingRegTrackingUse won't.
  58. movq %rdi,%rax
  59. # Register rdi released
  60. # Register rdi allocated
  61. movq %rax,%rdi
  62. While in this example:
  63. movq %rdi,%rax
  64. call proc
  65. movq %rdi,%rax
  66. GetNextInstructionUsingRegTrackingUse will return the second instruction while GetNextInstructionUsingReg
  67. won't.
  68. }
  69. function GetNextInstructionUsingRegTrackingUse(Current: tai; out Next: tai; reg: TRegister): Boolean;
  70. function RegModifiedByInstruction(Reg: TRegister; p1: tai): boolean; override;
  71. private
  72. function SkipSimpleInstructions(var hp1: tai): Boolean;
  73. protected
  74. class function IsMOVZXAcceptable: Boolean; static; inline;
  75. { Attempts to allocate a volatile integer register for use between p and hp,
  76. using AUsedRegs for the current register usage information. Returns NR_NO
  77. if no free register could be found }
  78. function GetIntRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  79. { Attempts to allocate a volatile MM register for use between p and hp,
  80. using AUsedRegs for the current register usage information. Returns NR_NO
  81. if no free register could be found }
  82. function GetMMRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  83. { checks whether loading a new value in reg1 overwrites the entirety of reg2 }
  84. class function Reg1WriteOverwritesReg2Entirely(reg1, reg2: tregister): boolean; static;
  85. { checks whether reading the value in reg1 depends on the value of reg2. This
  86. is very similar to SuperRegisterEquals, except it takes into account that
  87. R_SUBH and R_SUBL are independendent (e.g. reading from AL does not
  88. depend on the value in AH). }
  89. class function Reg1ReadDependsOnReg2(reg1, reg2: tregister): boolean; static;
  90. { Replaces all references to AOldReg in a memory reference to ANewReg }
  91. class function ReplaceRegisterInRef(var ref: TReference; const AOldReg, ANewReg: TRegister): Boolean; static;
  92. { Replaces all references to AOldReg in an operand to ANewReg }
  93. class function ReplaceRegisterInOper(const p: taicpu; const OperIdx: Integer; const AOldReg, ANewReg: TRegister): Boolean; static;
  94. { Replaces all references to AOldReg in an instruction to ANewReg,
  95. except where the register is being written }
  96. class function ReplaceRegisterInInstruction(const p: taicpu; const AOldReg, ANewReg: TRegister): Boolean; static;
  97. { Returns true if the reference only refers to ESP or EBP (or their 64-bit equivalents),
  98. or writes to a global symbol }
  99. class function IsRefSafe(const ref: PReference): Boolean; static;
  100. { Returns true if the given MOV instruction can be safely converted to CMOV }
  101. class function CanBeCMOV(p, cond_p: tai) : boolean; static;
  102. { Like UpdateUsedRegs, but ignores deallocations }
  103. class procedure UpdateIntRegsNoDealloc(var AUsedRegs: TAllUsedRegs; p: Tai); static;
  104. { Returns true if the given logic instruction can be converted into a BTx instruction (BT not included) }
  105. class function IsBTXAcceptable(p : tai) : boolean; static;
  106. { Converts the LEA instruction to ADD/INC/SUB/DEC. Returns True if the
  107. conversion was successful }
  108. function ConvertLEA(const p : taicpu): Boolean;
  109. function DeepMOVOpt(const p_mov: taicpu; const hp: taicpu): Boolean;
  110. function FuncMov2Func(var p: tai; const hp1: tai): Boolean;
  111. procedure DebugMsg(const s : string; p : tai);inline;
  112. class function IsExitCode(p : tai) : boolean; static;
  113. class function isFoldableArithOp(hp1 : taicpu; reg : tregister) : boolean; static;
  114. class function IsShrMovZFoldable(shr_size, movz_size: topsize; Shift: TCGInt): Boolean; static;
  115. procedure RemoveLastDeallocForFuncRes(p : tai);
  116. function DoArithCombineOpt(var p : tai) : Boolean;
  117. function DoMovCmpMemOpt(var p : tai; const hp1: tai; UpdateTmpUsedRegs: Boolean) : Boolean;
  118. function DoSETccLblRETOpt(var p: tai; const hp_label: tai_label) : Boolean;
  119. function PrePeepholeOptSxx(var p : tai) : boolean;
  120. function PrePeepholeOptIMUL(var p : tai) : boolean;
  121. function PrePeepholeOptAND(var p : tai) : boolean;
  122. function OptPass1Test(var p: tai): boolean;
  123. function OptPass1Add(var p: tai): boolean;
  124. function OptPass1AND(var p : tai) : boolean;
  125. function OptPass1_V_MOVAP(var p : tai) : boolean;
  126. function OptPass1VOP(var p : tai) : boolean;
  127. function OptPass1MOV(var p : tai) : boolean;
  128. function OptPass1Movx(var p : tai) : boolean;
  129. function OptPass1MOVXX(var p : tai) : boolean;
  130. function OptPass1OP(var p : tai) : boolean;
  131. function OptPass1LEA(var p : tai) : boolean;
  132. function OptPass1Sub(var p : tai) : boolean;
  133. function OptPass1SHLSAL(var p : tai) : boolean;
  134. function OptPass1SHR(var p : tai) : boolean;
  135. function OptPass1FSTP(var p : tai) : boolean;
  136. function OptPass1FLD(var p : tai) : boolean;
  137. function OptPass1Cmp(var p : tai) : boolean;
  138. function OptPass1PXor(var p : tai) : boolean;
  139. function OptPass1VPXor(var p: tai): boolean;
  140. function OptPass1Imul(var p : tai) : boolean;
  141. function OptPass1Jcc(var p : tai) : boolean;
  142. function OptPass1SHXX(var p: tai): boolean;
  143. function OptPass1VMOVDQ(var p: tai): Boolean;
  144. function OptPass1_V_Cvtss2sd(var p: tai): boolean;
  145. function OptPass2Movx(var p : tai): Boolean;
  146. function OptPass2MOV(var p : tai) : boolean;
  147. function OptPass2Imul(var p : tai) : boolean;
  148. function OptPass2Jmp(var p : tai) : boolean;
  149. function OptPass2Jcc(var p : tai) : boolean;
  150. function OptPass2Lea(var p: tai): Boolean;
  151. function OptPass2SUB(var p: tai): Boolean;
  152. function OptPass2ADD(var p : tai): Boolean;
  153. function OptPass2SETcc(var p : tai) : boolean;
  154. function CheckMemoryWrite(var first_mov, second_mov: taicpu): Boolean;
  155. function PostPeepholeOptMov(var p : tai) : Boolean;
  156. function PostPeepholeOptMovzx(var p : tai) : Boolean;
  157. function PostPeepholeOptXor(var p : tai) : Boolean;
  158. function PostPeepholeOptAnd(var p : tai) : boolean;
  159. function PostPeepholeOptMOVSX(var p : tai) : boolean;
  160. function PostPeepholeOptCmp(var p : tai) : Boolean;
  161. function PostPeepholeOptTestOr(var p : tai) : Boolean;
  162. function PostPeepholeOptCall(var p : tai) : Boolean;
  163. function PostPeepholeOptLea(var p : tai) : Boolean;
  164. function PostPeepholeOptPush(var p: tai): Boolean;
  165. function PostPeepholeOptShr(var p : tai) : boolean;
  166. function PostPeepholeOptADDSUB(var p : tai) : Boolean;
  167. function PostPeepholeOptVPXOR(var p: tai): Boolean;
  168. procedure ConvertJumpToRET(const p: tai; const ret_p: tai);
  169. function CheckJumpMovTransferOpt(var p: tai; hp1: tai; LoopCount: Integer; out Count: Integer): Boolean;
  170. function TrySwapMovOp(var p, hp1: tai): Boolean;
  171. function TrySwapMovCmp(var p, hp1: tai): Boolean;
  172. { Processor-dependent reference optimisation }
  173. class procedure OptimizeRefs(var p: taicpu); static;
  174. end;
  175. function MatchInstruction(const instr: tai; const op: TAsmOp; const opsize: topsizes): boolean;
  176. function MatchInstruction(const instr: tai; const op1,op2: TAsmOp; const opsize: topsizes): boolean;
  177. function MatchInstruction(const instr: tai; const op1,op2,op3: TAsmOp; const opsize: topsizes): boolean;
  178. function MatchInstruction(const instr: tai; const ops: array of TAsmOp; const opsize: topsizes): boolean;
  179. function MatchOperand(const oper: TOper; const reg: TRegister): boolean; inline;
  180. function MatchOperand(const oper: TOper; const a: tcgint): boolean; inline;
  181. function MatchOperand(const oper1: TOper; const oper2: TOper): boolean;
  182. {$if max_operands>2}
  183. function MatchOperand(const oper1: TOper; const oper2: TOper; const oper3: TOper): boolean;
  184. {$endif max_operands>2}
  185. function RefsEqual(const r1, r2: treference): boolean;
  186. { Note that Result is set to True if the references COULD overlap but the
  187. compiler cannot be sure (e.g. "(%reg1)" and "4(%reg2)" with a range of 4
  188. might still overlap because %reg2 could be equal to %reg1-4 }
  189. function RefsMightOverlap(const r1, r2: treference; const Range: asizeint): boolean;
  190. function MatchReference(const ref : treference;base,index : TRegister) : Boolean;
  191. { returns true, if ref is a reference using only the registers passed as base and index
  192. and having an offset }
  193. function MatchReferenceWithOffset(const ref : treference;base,index : TRegister) : Boolean;
  194. implementation
  195. uses
  196. cutils,verbose,
  197. systems,
  198. globals,
  199. cpuinfo,
  200. procinfo,
  201. paramgr,
  202. aasmbase,
  203. aoptbase,aoptutils,
  204. symconst,symsym,
  205. cgx86,
  206. itcpugas;
  207. {$ifdef DEBUG_AOPTCPU}
  208. const
  209. SPeepholeOptimization: shortstring = 'Peephole Optimization: ';
  210. {$else DEBUG_AOPTCPU}
  211. { Empty strings help the optimizer to remove string concatenations that won't
  212. ever appear to the user on release builds. [Kit] }
  213. const
  214. SPeepholeOptimization = '';
  215. {$endif DEBUG_AOPTCPU}
  216. LIST_STEP_SIZE = 4;
  217. {$ifndef 8086}
  218. MAX_CMOV_INSTRUCTIONS = 4;
  219. MAX_CMOV_REGISTERS = 8;
  220. {$endif 8086}
  221. type
  222. TJumpTrackingItem = class(TLinkedListItem)
  223. private
  224. FSymbol: TAsmSymbol;
  225. FRefs: LongInt;
  226. public
  227. constructor Create(ASymbol: TAsmSymbol);
  228. procedure IncRefs; {$ifdef USEINLINE}inline;{$endif USEINLINE}
  229. property Symbol: TAsmSymbol read FSymbol;
  230. property Refs: LongInt read FRefs;
  231. end;
  232. constructor TJumpTrackingItem.Create(ASymbol: TAsmSymbol);
  233. begin
  234. inherited Create;
  235. FSymbol := ASymbol;
  236. FRefs := 0;
  237. end;
  238. procedure TJumpTrackingItem.IncRefs; {$ifdef USEINLINE}inline;{$endif USEINLINE}
  239. begin
  240. Inc(FRefs);
  241. end;
  242. function MatchInstruction(const instr: tai; const op: TAsmOp; const opsize: topsizes): boolean;
  243. begin
  244. result :=
  245. (instr.typ = ait_instruction) and
  246. (taicpu(instr).opcode = op) and
  247. ((opsize = []) or (taicpu(instr).opsize in opsize));
  248. end;
  249. function MatchInstruction(const instr: tai; const op1,op2: TAsmOp; const opsize: topsizes): boolean;
  250. begin
  251. result :=
  252. (instr.typ = ait_instruction) and
  253. ((taicpu(instr).opcode = op1) or
  254. (taicpu(instr).opcode = op2)
  255. ) and
  256. ((opsize = []) or (taicpu(instr).opsize in opsize));
  257. end;
  258. function MatchInstruction(const instr: tai; const op1,op2,op3: TAsmOp; const opsize: topsizes): boolean;
  259. begin
  260. result :=
  261. (instr.typ = ait_instruction) and
  262. ((taicpu(instr).opcode = op1) or
  263. (taicpu(instr).opcode = op2) or
  264. (taicpu(instr).opcode = op3)
  265. ) and
  266. ((opsize = []) or (taicpu(instr).opsize in opsize));
  267. end;
  268. function MatchInstruction(const instr : tai;const ops : array of TAsmOp;
  269. const opsize : topsizes) : boolean;
  270. var
  271. op : TAsmOp;
  272. begin
  273. result:=false;
  274. if (instr.typ <> ait_instruction) or
  275. ((opsize <> []) and not(taicpu(instr).opsize in opsize)) then
  276. exit;
  277. for op in ops do
  278. begin
  279. if taicpu(instr).opcode = op then
  280. begin
  281. result:=true;
  282. exit;
  283. end;
  284. end;
  285. end;
  286. function MatchOperand(const oper: TOper; const reg: TRegister): boolean; inline;
  287. begin
  288. result := (oper.typ = top_reg) and (oper.reg = reg);
  289. end;
  290. function MatchOperand(const oper: TOper; const a: tcgint): boolean; inline;
  291. begin
  292. result := (oper.typ = top_const) and (oper.val = a);
  293. end;
  294. function MatchOperand(const oper1: TOper; const oper2: TOper): boolean;
  295. begin
  296. result := oper1.typ = oper2.typ;
  297. if result then
  298. case oper1.typ of
  299. top_const:
  300. Result:=oper1.val = oper2.val;
  301. top_reg:
  302. Result:=oper1.reg = oper2.reg;
  303. top_ref:
  304. Result:=RefsEqual(oper1.ref^, oper2.ref^);
  305. else
  306. internalerror(2013102801);
  307. end
  308. end;
  309. function MatchOperand(const oper1: TOper; const oper2: TOper; const oper3: TOper): boolean;
  310. begin
  311. result := (oper1.typ = oper2.typ) and (oper1.typ = oper3.typ);
  312. if result then
  313. case oper1.typ of
  314. top_const:
  315. Result:=(oper1.val = oper2.val) and (oper1.val = oper3.val);
  316. top_reg:
  317. Result:=(oper1.reg = oper2.reg) and (oper1.reg = oper3.reg);
  318. top_ref:
  319. Result:=RefsEqual(oper1.ref^, oper2.ref^) and RefsEqual(oper1.ref^, oper3.ref^);
  320. else
  321. internalerror(2020052401);
  322. end
  323. end;
  324. function RefsEqual(const r1, r2: treference): boolean;
  325. begin
  326. RefsEqual :=
  327. (r1.symbol=r2.symbol) and (r1.refaddr = r2.refaddr) and
  328. (r1.relsymbol = r2.relsymbol) and
  329. (r1.segment = r2.segment) and (r1.base = r2.base) and
  330. (r1.index = r2.index) and (r1.scalefactor = r2.scalefactor) and
  331. (r1.offset = r2.offset) and
  332. (r1.volatility + r2.volatility = []);
  333. end;
  334. function RefsMightOverlap(const r1, r2: treference; const Range: asizeint): boolean;
  335. begin
  336. if (r1.symbol<>r2.symbol) then
  337. { If the index registers are different, there's a chance one could
  338. be set so it equals the other symbol }
  339. Exit((r1.index<>r2.index) or (r1.scalefactor<>r2.scalefactor));
  340. if (r1.symbol=r2.symbol) and (r1.refaddr = r2.refaddr) and
  341. (r1.relsymbol = r2.relsymbol) and
  342. (r1.segment = r2.segment) and (r1.base = r2.base) and
  343. (r1.index = r2.index) and (r1.scalefactor = r2.scalefactor) and
  344. (r1.volatility + r2.volatility = []) then
  345. { In this case, it all depends on the offsets }
  346. Exit(abs(r1.offset - r2.offset) < Range);
  347. { There's a chance things MIGHT overlap, so take no chances }
  348. Result := True;
  349. end;
  350. function MatchReference(const ref : treference;base,index : TRegister) : Boolean;
  351. begin
  352. Result:=(ref.offset=0) and
  353. (ref.scalefactor in [0,1]) and
  354. (ref.segment=NR_NO) and
  355. (ref.symbol=nil) and
  356. (ref.relsymbol=nil) and
  357. ((base=NR_INVALID) or
  358. (ref.base=base)) and
  359. ((index=NR_INVALID) or
  360. (ref.index=index)) and
  361. (ref.volatility=[]);
  362. end;
  363. function MatchReferenceWithOffset(const ref : treference;base,index : TRegister) : Boolean;
  364. begin
  365. Result:=(ref.scalefactor in [0,1]) and
  366. (ref.segment=NR_NO) and
  367. (ref.symbol=nil) and
  368. (ref.relsymbol=nil) and
  369. ((base=NR_INVALID) or
  370. (ref.base=base)) and
  371. ((index=NR_INVALID) or
  372. (ref.index=index)) and
  373. (ref.volatility=[]);
  374. end;
  375. function InstrReadsFlags(p: tai): boolean;
  376. begin
  377. InstrReadsFlags := true;
  378. case p.typ of
  379. ait_instruction:
  380. if InsProp[taicpu(p).opcode].Ch*
  381. [Ch_RCarryFlag,Ch_RParityFlag,Ch_RAuxiliaryFlag,Ch_RZeroFlag,Ch_RSignFlag,Ch_ROverflowFlag,
  382. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag,
  383. Ch_RFlags,Ch_RWFlags,Ch_RFLAGScc,Ch_All]<>[] then
  384. exit;
  385. ait_label:
  386. exit;
  387. else
  388. ;
  389. end;
  390. InstrReadsFlags := false;
  391. end;
  392. function TX86AsmOptimizer.GetNextInstructionUsingReg(Current: tai; out Next: tai; reg: TRegister): Boolean;
  393. begin
  394. Next:=Current;
  395. repeat
  396. Result:=GetNextInstruction(Next,Next);
  397. until not (Result) or
  398. not(cs_opt_level3 in current_settings.optimizerswitches) or
  399. (Next.typ<>ait_instruction) or
  400. RegInInstruction(reg,Next) or
  401. is_calljmp(taicpu(Next).opcode);
  402. end;
  403. function TX86AsmOptimizer.GetNextInstructionUsingRegCount(Current: tai; out Next: tai; reg: TRegister): Cardinal;
  404. var
  405. GetNextResult: Boolean;
  406. begin
  407. Result:=0;
  408. Next:=Current;
  409. repeat
  410. GetNextResult := GetNextInstruction(Next,Next);
  411. if GetNextResult then
  412. Inc(Result)
  413. else
  414. { Must return zero upon hitting the end of the linked list without a match }
  415. Result := 0;
  416. until not (GetNextResult) or
  417. not(cs_opt_level3 in current_settings.optimizerswitches) or
  418. (Next.typ<>ait_instruction) or
  419. RegInInstruction(reg,Next) or
  420. is_calljmp(taicpu(Next).opcode);
  421. end;
  422. function TX86AsmOptimizer.GetNextInstructionUsingRegCond(Current: tai; out Next: tai; reg: TRegister; var JumpTracking: TLinkedList; var CrossJump: Boolean): Boolean;
  423. procedure TrackJump(Symbol: TAsmSymbol);
  424. var
  425. Search: TJumpTrackingItem;
  426. begin
  427. { See if an entry already exists in our jump tracking list
  428. (faster to search backwards due to the higher chance of
  429. matching destinations) }
  430. Search := TJumpTrackingItem(JumpTracking.Last);
  431. while Assigned(Search) do
  432. begin
  433. if Search.Symbol = Symbol then
  434. begin
  435. { Found it - remove it so it can be pushed to the front }
  436. JumpTracking.Remove(Search);
  437. Break;
  438. end;
  439. Search := TJumpTrackingItem(Search.Previous);
  440. end;
  441. if not Assigned(Search) then
  442. Search := TJumpTrackingItem.Create(JumpTargetOp(taicpu(Next))^.ref^.symbol);
  443. JumpTracking.Concat(Search);
  444. Search.IncRefs;
  445. end;
  446. function LabelAccountedFor(Symbol: TAsmSymbol): Boolean;
  447. var
  448. Search: TJumpTrackingItem;
  449. begin
  450. Result := False;
  451. { See if this label appears in the tracking list }
  452. Search := TJumpTrackingItem(JumpTracking.Last);
  453. while Assigned(Search) do
  454. begin
  455. if Search.Symbol = Symbol then
  456. begin
  457. { Found it - let's see what we can discover }
  458. if Search.Symbol.getrefs = Search.Refs then
  459. begin
  460. { Success - all the references are accounted for }
  461. JumpTracking.Remove(Search);
  462. Search.Free;
  463. { It is logically impossible for CrossJump to be false here
  464. because we must have run into a conditional jump for
  465. this label at some point }
  466. if not CrossJump then
  467. InternalError(2022041710);
  468. if JumpTracking.First = nil then
  469. { Tracking list is now empty - no more cross jumps }
  470. CrossJump := False;
  471. Result := True;
  472. Exit;
  473. end;
  474. { If the references don't match, it's possible to enter
  475. this label through other means, so drop out }
  476. Exit;
  477. end;
  478. Search := TJumpTrackingItem(Search.Previous);
  479. end;
  480. end;
  481. var
  482. Next_Label: tai;
  483. begin
  484. { Note, CrossJump keeps its input value if a conditional jump is not found - it doesn't get set to False }
  485. Next := Current;
  486. repeat
  487. Result := GetNextInstruction(Next,Next);
  488. if not Result then
  489. Break;
  490. if Next.typ = ait_align then
  491. Result := SkipAligns(Next, Next);
  492. if (Next.typ=ait_instruction) and is_calljmp(taicpu(Next).opcode) then
  493. if is_calljmpuncondret(taicpu(Next).opcode) then
  494. begin
  495. if (taicpu(Next).opcode = A_JMP) and
  496. { Remove dead code now to save time }
  497. RemoveDeadCodeAfterJump(taicpu(Next)) then
  498. { A jump was removed, but not the current instruction, and
  499. Result doesn't necessarily translate into an optimisation
  500. routine's Result, so use the "Force New Iteration" flag so
  501. mark a new pass }
  502. Include(OptsToCheck, aoc_ForceNewIteration);
  503. if not Assigned(JumpTracking) then
  504. begin
  505. { Cross-label optimisations often causes other optimisations
  506. to perform worse because they're not given the chance to
  507. optimise locally. In this case, don't do the cross-label
  508. optimisations yet, but flag them as a potential possibility
  509. for the next iteration of Pass 1 }
  510. if not NotFirstIteration then
  511. Include(OptsToCheck, aoc_ForceNewIteration);
  512. end
  513. else if IsJumpToLabel(taicpu(Next)) and
  514. GetNextInstruction(Next, Next_Label) and
  515. SkipAligns(Next_Label, Next_Label) then
  516. begin
  517. { If we have JMP .lbl, and the label after it has all of its
  518. references tracked, then this is probably an if-else style of
  519. block and we can keep tracking. If the label for this jump
  520. then appears later and is fully tracked, then it's the end
  521. of the if-else blocks and the code paths converge (thus
  522. marking the end of the cross-jump) }
  523. if (Next_Label.typ = ait_label) then
  524. begin
  525. if LabelAccountedFor(tai_label(Next_Label).labsym) then
  526. begin
  527. TrackJump(JumpTargetOp(taicpu(Next))^.ref^.symbol);
  528. Next := Next_Label;
  529. { CrossJump gets set to false by LabelAccountedFor if the
  530. list is completely emptied (as it indicates that all
  531. code paths have converged). We could avoid this nuance
  532. by moving the TrackJump call to before the
  533. LabelAccountedFor call, but this is slower in situations
  534. where LabelAccountedFor would return False due to the
  535. creation of a new object that is not used and destroyed
  536. soon after. }
  537. CrossJump := True;
  538. Continue;
  539. end;
  540. end
  541. else if (Next_Label.typ <> ait_marker) then
  542. { We just did a RemoveDeadCodeAfterJump, so either we find
  543. a label, the end of the procedure or some kind of marker}
  544. InternalError(2022041720);
  545. end;
  546. Result := False;
  547. Exit;
  548. end
  549. else
  550. begin
  551. if not Assigned(JumpTracking) then
  552. begin
  553. { Cross-label optimisations often causes other optimisations
  554. to perform worse because they're not given the chance to
  555. optimise locally. In this case, don't do the cross-label
  556. optimisations yet, but flag them as a potential possibility
  557. for the next iteration of Pass 1 }
  558. if not NotFirstIteration then
  559. Include(OptsToCheck, aoc_ForceNewIteration);
  560. end
  561. else if IsJumpToLabel(taicpu(Next)) then
  562. TrackJump(JumpTargetOp(taicpu(Next))^.ref^.symbol)
  563. else
  564. { Conditional jumps should always be a jump to label }
  565. InternalError(2022041701);
  566. CrossJump := True;
  567. Continue;
  568. end;
  569. if Next.typ = ait_label then
  570. begin
  571. if not Assigned(JumpTracking) then
  572. begin
  573. { Cross-label optimisations often causes other optimisations
  574. to perform worse because they're not given the chance to
  575. optimise locally. In this case, don't do the cross-label
  576. optimisations yet, but flag them as a potential possibility
  577. for the next iteration of Pass 1 }
  578. if not NotFirstIteration then
  579. Include(OptsToCheck, aoc_ForceNewIteration);
  580. end
  581. else if LabelAccountedFor(tai_label(Next).labsym) then
  582. Continue;
  583. { If we reach here, we're at a label that hasn't been seen before
  584. (or JumpTracking was nil) }
  585. Break;
  586. end;
  587. until not Result or
  588. not (cs_opt_level3 in current_settings.optimizerswitches) or
  589. not (Next.typ in [ait_label, ait_instruction]) or
  590. RegInInstruction(reg,Next);
  591. end;
  592. function TX86AsmOptimizer.GetNextInstructionUsingRegTrackingUse(Current: tai; out Next: tai; reg: TRegister): Boolean;
  593. begin
  594. if not(cs_opt_level3 in current_settings.optimizerswitches) then
  595. begin
  596. Result:=GetNextInstruction(Current,Next);
  597. exit;
  598. end;
  599. Next:=tai(Current.Next);
  600. Result:=false;
  601. while assigned(Next) do
  602. begin
  603. if ((Next.typ=ait_instruction) and is_calljmp(taicpu(Next).opcode) and not(taicpu(Next).opcode=A_CALL)) or
  604. ((Next.typ=ait_regalloc) and (getsupreg(tai_regalloc(Next).reg)=getsupreg(reg))) or
  605. ((Next.typ=ait_label) and not(labelCanBeSkipped(Tai_Label(Next)))) then
  606. exit
  607. else if (Next.typ=ait_instruction) and RegInInstruction(reg,Next) and not(taicpu(Next).opcode=A_CALL) then
  608. begin
  609. Result:=true;
  610. exit;
  611. end;
  612. Next:=tai(Next.Next);
  613. end;
  614. end;
  615. function TX86AsmOptimizer.InstructionLoadsFromReg(const reg: TRegister;const hp: tai): boolean;
  616. begin
  617. Result:=RegReadByInstruction(reg,hp);
  618. end;
  619. class function TX86AsmOptimizer.RegReadByInstruction(reg: TRegister; hp: tai): boolean;
  620. var
  621. p: taicpu;
  622. opcount: longint;
  623. begin
  624. RegReadByInstruction := false;
  625. if hp.typ <> ait_instruction then
  626. exit;
  627. p := taicpu(hp);
  628. case p.opcode of
  629. A_CALL:
  630. regreadbyinstruction := true;
  631. A_IMUL:
  632. case p.ops of
  633. 1:
  634. regReadByInstruction := RegInOp(reg,p.oper[0]^) or
  635. (
  636. ((getregtype(reg)=R_INTREGISTER) and (getsupreg(reg)=RS_EAX)) and
  637. ((getsubreg(reg)<>R_SUBH) or (p.opsize<>S_B))
  638. );
  639. 2,3:
  640. regReadByInstruction :=
  641. reginop(reg,p.oper[0]^) or
  642. reginop(reg,p.oper[1]^);
  643. else
  644. InternalError(2019112801);
  645. end;
  646. A_MUL:
  647. begin
  648. regReadByInstruction := RegInOp(reg,p.oper[0]^) or
  649. (
  650. ((getregtype(reg)=R_INTREGISTER) and (getsupreg(reg)=RS_EAX)) and
  651. ((getsubreg(reg)<>R_SUBH) or (p.opsize<>S_B))
  652. );
  653. end;
  654. A_IDIV,A_DIV:
  655. begin
  656. regReadByInstruction := RegInOp(reg,p.oper[0]^) or
  657. (
  658. (getregtype(reg)=R_INTREGISTER) and
  659. (
  660. (getsupreg(reg)=RS_EAX) or ((getsupreg(reg)=RS_EDX) and (p.opsize<>S_B))
  661. )
  662. );
  663. end;
  664. else
  665. begin
  666. if (p.opcode=A_LEA) and is_segment_reg(reg) then
  667. begin
  668. RegReadByInstruction := false;
  669. exit;
  670. end;
  671. for opcount := 0 to p.ops-1 do
  672. if (p.oper[opCount]^.typ = top_ref) and
  673. RegInRef(reg,p.oper[opcount]^.ref^) then
  674. begin
  675. RegReadByInstruction := true;
  676. exit
  677. end;
  678. { special handling for SSE MOVSD }
  679. if (p.opcode=A_MOVSD) and (p.ops>0) then
  680. begin
  681. if p.ops<>2 then
  682. internalerror(2017042702);
  683. regReadByInstruction := reginop(reg,p.oper[0]^) or
  684. (
  685. (p.oper[1]^.typ=top_reg) and (p.oper[0]^.typ=top_reg) and reginop(reg, p.oper[1]^)
  686. );
  687. exit;
  688. end;
  689. with insprop[p.opcode] do
  690. begin
  691. case getregtype(reg) of
  692. R_INTREGISTER:
  693. begin
  694. case getsupreg(reg) of
  695. RS_EAX:
  696. if [Ch_REAX,Ch_RWEAX,Ch_MEAX]*Ch<>[] then
  697. begin
  698. RegReadByInstruction := true;
  699. exit
  700. end;
  701. RS_ECX:
  702. if [Ch_RECX,Ch_RWECX,Ch_MECX]*Ch<>[] then
  703. begin
  704. RegReadByInstruction := true;
  705. exit
  706. end;
  707. RS_EDX:
  708. if [Ch_REDX,Ch_RWEDX,Ch_MEDX]*Ch<>[] then
  709. begin
  710. RegReadByInstruction := true;
  711. exit
  712. end;
  713. RS_EBX:
  714. if [Ch_REBX,Ch_RWEBX,Ch_MEBX]*Ch<>[] then
  715. begin
  716. RegReadByInstruction := true;
  717. exit
  718. end;
  719. RS_ESP:
  720. if [Ch_RESP,Ch_RWESP,Ch_MESP]*Ch<>[] then
  721. begin
  722. RegReadByInstruction := true;
  723. exit
  724. end;
  725. RS_EBP:
  726. if [Ch_REBP,Ch_RWEBP,Ch_MEBP]*Ch<>[] then
  727. begin
  728. RegReadByInstruction := true;
  729. exit
  730. end;
  731. RS_ESI:
  732. if [Ch_RESI,Ch_RWESI,Ch_MESI]*Ch<>[] then
  733. begin
  734. RegReadByInstruction := true;
  735. exit
  736. end;
  737. RS_EDI:
  738. if [Ch_REDI,Ch_RWEDI,Ch_MEDI]*Ch<>[] then
  739. begin
  740. RegReadByInstruction := true;
  741. exit
  742. end;
  743. end;
  744. end;
  745. R_MMREGISTER:
  746. begin
  747. case getsupreg(reg) of
  748. RS_XMM0:
  749. if [Ch_RXMM0,Ch_RWXMM0,Ch_MXMM0]*Ch<>[] then
  750. begin
  751. RegReadByInstruction := true;
  752. exit
  753. end;
  754. end;
  755. end;
  756. else
  757. ;
  758. end;
  759. if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  760. begin
  761. if (Ch_RFLAGScc in Ch) and not(getsubreg(reg) in [R_SUBW,R_SUBD,R_SUBQ]) then
  762. begin
  763. case p.condition of
  764. C_A,C_NBE, { CF=0 and ZF=0 }
  765. C_BE,C_NA: { CF=1 or ZF=1 }
  766. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGCARRY,R_SUBFLAGZERO];
  767. C_AE,C_NB,C_NC, { CF=0 }
  768. C_B,C_NAE,C_C: { CF=1 }
  769. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGCARRY];
  770. C_NE,C_NZ, { ZF=0 }
  771. C_E,C_Z: { ZF=1 }
  772. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGZERO];
  773. C_G,C_NLE, { ZF=0 and SF=OF }
  774. C_LE,C_NG: { ZF=1 or SF<>OF }
  775. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGZERO,R_SUBFLAGSIGN,R_SUBFLAGOVERFLOW];
  776. C_GE,C_NL, { SF=OF }
  777. C_L,C_NGE: { SF<>OF }
  778. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGSIGN,R_SUBFLAGOVERFLOW];
  779. C_NO, { OF=0 }
  780. C_O: { OF=1 }
  781. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGOVERFLOW];
  782. C_NP,C_PO, { PF=0 }
  783. C_P,C_PE: { PF=1 }
  784. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGPARITY];
  785. C_NS, { SF=0 }
  786. C_S: { SF=1 }
  787. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGSIGN];
  788. else
  789. internalerror(2017042701);
  790. end;
  791. if RegReadByInstruction then
  792. exit;
  793. end;
  794. case getsubreg(reg) of
  795. R_SUBW,R_SUBD,R_SUBQ:
  796. RegReadByInstruction :=
  797. [Ch_RCarryFlag,Ch_RParityFlag,Ch_RAuxiliaryFlag,Ch_RZeroFlag,Ch_RSignFlag,Ch_ROverflowFlag,
  798. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag,
  799. Ch_RDirFlag,Ch_RFlags,Ch_RWFlags,Ch_RFLAGScc]*Ch<>[];
  800. R_SUBFLAGCARRY:
  801. RegReadByInstruction:=[Ch_RCarryFlag,Ch_RWCarryFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  802. R_SUBFLAGPARITY:
  803. RegReadByInstruction:=[Ch_RParityFlag,Ch_RWParityFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  804. R_SUBFLAGAUXILIARY:
  805. RegReadByInstruction:=[Ch_RAuxiliaryFlag,Ch_RWAuxiliaryFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  806. R_SUBFLAGZERO:
  807. RegReadByInstruction:=[Ch_RZeroFlag,Ch_RWZeroFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  808. R_SUBFLAGSIGN:
  809. RegReadByInstruction:=[Ch_RSignFlag,Ch_RWSignFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  810. R_SUBFLAGOVERFLOW:
  811. RegReadByInstruction:=[Ch_ROverflowFlag,Ch_RWOverflowFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  812. R_SUBFLAGINTERRUPT:
  813. RegReadByInstruction:=[Ch_RFlags,Ch_RWFlags]*Ch<>[];
  814. R_SUBFLAGDIRECTION:
  815. RegReadByInstruction:=[Ch_RDirFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  816. else
  817. internalerror(2017042601);
  818. end;
  819. exit;
  820. end;
  821. if (Ch_NoReadIfEqualRegs in Ch) and (p.ops=2) and
  822. (p.oper[0]^.typ=top_reg) and (p.oper[1]^.typ=top_reg) and
  823. (p.oper[0]^.reg=p.oper[1]^.reg) then
  824. exit;
  825. if ([CH_RWOP1,CH_ROP1,CH_MOP1]*Ch<>[]) and reginop(reg,p.oper[0]^) then
  826. begin
  827. RegReadByInstruction := true;
  828. exit
  829. end;
  830. if ([Ch_RWOP2,Ch_ROP2,Ch_MOP2]*Ch<>[]) and reginop(reg,p.oper[1]^) then
  831. begin
  832. RegReadByInstruction := true;
  833. exit
  834. end;
  835. if ([Ch_RWOP3,Ch_ROP3,Ch_MOP3]*Ch<>[]) and reginop(reg,p.oper[2]^) then
  836. begin
  837. RegReadByInstruction := true;
  838. exit
  839. end;
  840. if ([Ch_RWOP4,Ch_ROP4,Ch_MOP4]*Ch<>[]) and reginop(reg,p.oper[3]^) then
  841. begin
  842. RegReadByInstruction := true;
  843. exit
  844. end;
  845. end;
  846. end;
  847. end;
  848. end;
  849. function TX86AsmOptimizer.RegInInstruction(Reg: TRegister; p1: tai): Boolean;
  850. begin
  851. result:=false;
  852. if p1.typ<>ait_instruction then
  853. exit;
  854. if (Ch_All in insprop[taicpu(p1).opcode].Ch) then
  855. exit(true);
  856. if (getregtype(reg)=R_INTREGISTER) and
  857. { change information for xmm movsd are not correct }
  858. ((taicpu(p1).opcode<>A_MOVSD) or (taicpu(p1).ops=0)) then
  859. begin
  860. case getsupreg(reg) of
  861. { RS_EAX = RS_RAX on x86-64 }
  862. RS_EAX:
  863. result:=([Ch_REAX,Ch_RRAX,Ch_WEAX,Ch_WRAX,Ch_RWEAX,Ch_RWRAX,Ch_MEAX,Ch_MRAX]*insprop[taicpu(p1).opcode].Ch)<>[];
  864. RS_ECX:
  865. result:=([Ch_RECX,Ch_RRCX,Ch_WECX,Ch_WRCX,Ch_RWECX,Ch_RWRCX,Ch_MECX,Ch_MRCX]*insprop[taicpu(p1).opcode].Ch)<>[];
  866. RS_EDX:
  867. result:=([Ch_REDX,Ch_RRDX,Ch_WEDX,Ch_WRDX,Ch_RWEDX,Ch_RWRDX,Ch_MEDX,Ch_MRDX]*insprop[taicpu(p1).opcode].Ch)<>[];
  868. RS_EBX:
  869. result:=([Ch_REBX,Ch_RRBX,Ch_WEBX,Ch_WRBX,Ch_RWEBX,Ch_RWRBX,Ch_MEBX,Ch_MRBX]*insprop[taicpu(p1).opcode].Ch)<>[];
  870. RS_ESP:
  871. result:=([Ch_RESP,Ch_RRSP,Ch_WESP,Ch_WRSP,Ch_RWESP,Ch_RWRSP,Ch_MESP,Ch_MRSP]*insprop[taicpu(p1).opcode].Ch)<>[];
  872. RS_EBP:
  873. result:=([Ch_REBP,Ch_RRBP,Ch_WEBP,Ch_WRBP,Ch_RWEBP,Ch_RWRBP,Ch_MEBP,Ch_MRBP]*insprop[taicpu(p1).opcode].Ch)<>[];
  874. RS_ESI:
  875. result:=([Ch_RESI,Ch_RRSI,Ch_WESI,Ch_WRSI,Ch_RWESI,Ch_RWRSI,Ch_MESI,Ch_MRSI,Ch_RMemEDI]*insprop[taicpu(p1).opcode].Ch)<>[];
  876. RS_EDI:
  877. result:=([Ch_REDI,Ch_RRDI,Ch_WEDI,Ch_WRDI,Ch_RWEDI,Ch_RWRDI,Ch_MEDI,Ch_MRDI,Ch_WMemEDI]*insprop[taicpu(p1).opcode].Ch)<>[];
  878. else
  879. ;
  880. end;
  881. if result then
  882. exit;
  883. end
  884. else if getregtype(reg)=R_MMREGISTER then
  885. begin
  886. case getsupreg(reg) of
  887. RS_XMM0:
  888. result:=([Ch_RXMM0,Ch_WXMM0,Ch_RWXMM0,Ch_MXMM0]*insprop[taicpu(p1).opcode].Ch)<>[];
  889. else
  890. ;
  891. end;
  892. if result then
  893. exit;
  894. end
  895. else if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  896. begin
  897. if ([Ch_RFlags,Ch_WFlags,Ch_RWFlags,Ch_RFLAGScc]*insprop[taicpu(p1).opcode].Ch)<>[] then
  898. exit(true);
  899. case getsubreg(reg) of
  900. R_SUBFLAGCARRY:
  901. Result:=([Ch_RCarryFlag,Ch_RWCarryFlag,Ch_W0CarryFlag,Ch_W1CarryFlag,Ch_WCarryFlag,Ch_WUCarryFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  902. R_SUBFLAGPARITY:
  903. Result:=([Ch_RParityFlag,Ch_RWParityFlag,Ch_W0ParityFlag,Ch_W1ParityFlag,Ch_WParityFlag,Ch_WUParityFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  904. R_SUBFLAGAUXILIARY:
  905. Result:=([Ch_RAuxiliaryFlag,Ch_RWAuxiliaryFlag,Ch_W0AuxiliaryFlag,Ch_W1AuxiliaryFlag,Ch_WAuxiliaryFlag,Ch_WUAuxiliaryFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  906. R_SUBFLAGZERO:
  907. Result:=([Ch_RZeroFlag,Ch_RWZeroFlag,Ch_W0ZeroFlag,Ch_W1ZeroFlag,Ch_WZeroFlag,Ch_WUZeroFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  908. R_SUBFLAGSIGN:
  909. Result:=([Ch_RSignFlag,Ch_RWSignFlag,Ch_W0SignFlag,Ch_W1SignFlag,Ch_WSignFlag,Ch_WUSignFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  910. R_SUBFLAGOVERFLOW:
  911. Result:=([Ch_ROverflowFlag,Ch_RWOverflowFlag,Ch_W0OverflowFlag,Ch_W1OverflowFlag,Ch_WOverflowFlag,Ch_WUOverflowFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  912. R_SUBFLAGINTERRUPT:
  913. Result:=([Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags]*insprop[taicpu(p1).opcode].Ch)<>[];
  914. R_SUBFLAGDIRECTION:
  915. Result:=([Ch_RDirFlag,Ch_W0DirFlag,Ch_W1DirFlag,Ch_WFlags]*insprop[taicpu(p1).opcode].Ch)<>[];
  916. R_SUBW,R_SUBD,R_SUBQ:
  917. { Everything except the direction bits }
  918. Result:=
  919. ([Ch_RCarryFlag,Ch_RParityFlag,Ch_RAuxiliaryFlag,Ch_RZeroFlag,Ch_RSignFlag,Ch_ROverflowFlag,
  920. Ch_WCarryFlag,Ch_WParityFlag,Ch_WAuxiliaryFlag,Ch_WZeroFlag,Ch_WSignFlag,Ch_WOverflowFlag,
  921. Ch_W0CarryFlag,Ch_W0ParityFlag,Ch_W0AuxiliaryFlag,Ch_W0ZeroFlag,Ch_W0SignFlag,Ch_W0OverflowFlag,
  922. Ch_W1CarryFlag,Ch_W1ParityFlag,Ch_W1AuxiliaryFlag,Ch_W1ZeroFlag,Ch_W1SignFlag,Ch_W1OverflowFlag,
  923. Ch_WUCarryFlag,Ch_WUParityFlag,Ch_WUAuxiliaryFlag,Ch_WUZeroFlag,Ch_WUSignFlag,Ch_WUOverflowFlag,
  924. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag
  925. ]*insprop[taicpu(p1).opcode].Ch)<>[];
  926. else
  927. ;
  928. end;
  929. if result then
  930. exit;
  931. end
  932. else if (getregtype(reg)=R_FPUREGISTER) and (Ch_FPU in insprop[taicpu(p1).opcode].Ch) then
  933. exit(true);
  934. Result:=inherited RegInInstruction(Reg, p1);
  935. end;
  936. function TX86AsmOptimizer.RegModifiedByInstruction(Reg: TRegister; p1: tai): boolean;
  937. const
  938. WriteOps: array[0..3] of set of TInsChange =
  939. ([CH_RWOP1,CH_WOP1,CH_MOP1],
  940. [Ch_RWOP2,Ch_WOP2,Ch_MOP2],
  941. [Ch_RWOP3,Ch_WOP3,Ch_MOP3],
  942. [Ch_RWOP4,Ch_WOP4,Ch_MOP4]);
  943. var
  944. OperIdx: Integer;
  945. begin
  946. Result := False;
  947. if p1.typ <> ait_instruction then
  948. exit;
  949. with insprop[taicpu(p1).opcode] do
  950. if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  951. begin
  952. case getsubreg(reg) of
  953. R_SUBW,R_SUBD,R_SUBQ:
  954. Result :=
  955. [Ch_WCarryFlag,Ch_WParityFlag,Ch_WAuxiliaryFlag,Ch_WZeroFlag,Ch_WSignFlag,Ch_WOverflowFlag,
  956. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag,
  957. Ch_W0DirFlag,Ch_W1DirFlag,Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  958. R_SUBFLAGCARRY:
  959. Result:=[Ch_WCarryFlag,Ch_RWCarryFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  960. R_SUBFLAGPARITY:
  961. Result:=[Ch_WParityFlag,Ch_RWParityFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  962. R_SUBFLAGAUXILIARY:
  963. Result:=[Ch_WAuxiliaryFlag,Ch_RWAuxiliaryFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  964. R_SUBFLAGZERO:
  965. Result:=[Ch_WZeroFlag,Ch_RWZeroFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  966. R_SUBFLAGSIGN:
  967. Result:=[Ch_WSignFlag,Ch_RWSignFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  968. R_SUBFLAGOVERFLOW:
  969. Result:=[Ch_WOverflowFlag,Ch_RWOverflowFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  970. R_SUBFLAGINTERRUPT:
  971. Result:=[Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  972. R_SUBFLAGDIRECTION:
  973. Result:=[Ch_W0DirFlag,Ch_W1DirFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  974. else
  975. internalerror(2017042602);
  976. end;
  977. exit;
  978. end;
  979. case taicpu(p1).opcode of
  980. A_CALL:
  981. { We could potentially set Result to False if the register in
  982. question is non-volatile for the subroutine's calling convention,
  983. but this would require detecting the calling convention in use and
  984. also assuming that the routine doesn't contain malformed assembly
  985. language, for example... so it could only be done under -O4 as it
  986. would be considered a side-effect. [Kit] }
  987. Result := True;
  988. A_MOVSD:
  989. { special handling for SSE MOVSD }
  990. if (taicpu(p1).ops>0) then
  991. begin
  992. if taicpu(p1).ops<>2 then
  993. internalerror(2017042703);
  994. Result := (taicpu(p1).oper[1]^.typ=top_reg) and RegInOp(reg,taicpu(p1).oper[1]^);
  995. end;
  996. { VMOVSS and VMOVSD has two and three operand flavours, this cannot modelled by x86ins.dat
  997. so fix it here (FK)
  998. }
  999. A_VMOVSS,
  1000. A_VMOVSD:
  1001. begin
  1002. Result := (taicpu(p1).ops=3) and (taicpu(p1).oper[2]^.typ=top_reg) and RegInOp(reg,taicpu(p1).oper[2]^);
  1003. exit;
  1004. end;
  1005. A_IMUL:
  1006. Result := (taicpu(p1).oper[taicpu(p1).ops-1]^.typ=top_reg) and RegInOp(reg,taicpu(p1).oper[taicpu(p1).ops-1]^);
  1007. else
  1008. ;
  1009. end;
  1010. if Result then
  1011. exit;
  1012. with insprop[taicpu(p1).opcode] do
  1013. begin
  1014. if getregtype(reg)=R_INTREGISTER then
  1015. begin
  1016. case getsupreg(reg) of
  1017. RS_EAX:
  1018. if [Ch_WEAX,Ch_RWEAX,Ch_MEAX,Ch_WRAX,Ch_RWRAX,Ch_MRAX]*Ch<>[] then
  1019. begin
  1020. Result := True;
  1021. exit
  1022. end;
  1023. RS_ECX:
  1024. if [Ch_WECX,Ch_RWECX,Ch_MECX,Ch_WRCX,Ch_RWRCX,Ch_MRCX]*Ch<>[] then
  1025. begin
  1026. Result := True;
  1027. exit
  1028. end;
  1029. RS_EDX:
  1030. if [Ch_WEDX,Ch_RWEDX,Ch_MEDX,Ch_WRDX,Ch_RWRDX,Ch_MRDX]*Ch<>[] then
  1031. begin
  1032. Result := True;
  1033. exit
  1034. end;
  1035. RS_EBX:
  1036. if [Ch_WEBX,Ch_RWEBX,Ch_MEBX,Ch_WRBX,Ch_RWRBX,Ch_MRBX]*Ch<>[] then
  1037. begin
  1038. Result := True;
  1039. exit
  1040. end;
  1041. RS_ESP:
  1042. if [Ch_WESP,Ch_RWESP,Ch_MESP,Ch_WRSP,Ch_RWRSP,Ch_MRSP]*Ch<>[] then
  1043. begin
  1044. Result := True;
  1045. exit
  1046. end;
  1047. RS_EBP:
  1048. if [Ch_WEBP,Ch_RWEBP,Ch_MEBP,Ch_WRBP,Ch_RWRBP,Ch_MRBP]*Ch<>[] then
  1049. begin
  1050. Result := True;
  1051. exit
  1052. end;
  1053. RS_ESI:
  1054. if [Ch_WESI,Ch_RWESI,Ch_MESI,Ch_WRSI,Ch_RWRSI,Ch_MRSI]*Ch<>[] then
  1055. begin
  1056. Result := True;
  1057. exit
  1058. end;
  1059. RS_EDI:
  1060. if [Ch_WEDI,Ch_RWEDI,Ch_MEDI,Ch_WRDI,Ch_RWRDI,Ch_MRDI]*Ch<>[] then
  1061. begin
  1062. Result := True;
  1063. exit
  1064. end;
  1065. end;
  1066. end;
  1067. for OperIdx := 0 to taicpu(p1).ops - 1 do
  1068. if (WriteOps[OperIdx]*Ch<>[]) and
  1069. { The register doesn't get modified inside a reference }
  1070. (taicpu(p1).oper[OperIdx]^.typ = top_reg) and
  1071. SuperRegistersEqual(reg,taicpu(p1).oper[OperIdx]^.reg) then
  1072. begin
  1073. Result := true;
  1074. exit
  1075. end;
  1076. end;
  1077. end;
  1078. {$ifdef DEBUG_AOPTCPU}
  1079. procedure TX86AsmOptimizer.DebugMsg(const s: string;p : tai);
  1080. begin
  1081. asml.insertbefore(tai_comment.Create(strpnew(s)), p);
  1082. end;
  1083. function debug_tostr(i: tcgint): string; inline;
  1084. begin
  1085. Result := tostr(i);
  1086. end;
  1087. function debug_regname(r: TRegister): string; inline;
  1088. begin
  1089. Result := '%' + std_regname(r);
  1090. end;
  1091. { Debug output function - creates a string representation of an operator }
  1092. function debug_operstr(oper: TOper): string;
  1093. begin
  1094. case oper.typ of
  1095. top_const:
  1096. Result := '$' + debug_tostr(oper.val);
  1097. top_reg:
  1098. Result := debug_regname(oper.reg);
  1099. top_ref:
  1100. begin
  1101. if oper.ref^.offset <> 0 then
  1102. Result := debug_tostr(oper.ref^.offset) + '('
  1103. else
  1104. Result := '(';
  1105. if (oper.ref^.base <> NR_INVALID) and (oper.ref^.base <> NR_NO) then
  1106. begin
  1107. Result := Result + debug_regname(oper.ref^.base);
  1108. if (oper.ref^.index <> NR_INVALID) and (oper.ref^.index <> NR_NO) then
  1109. Result := Result + ',' + debug_regname(oper.ref^.index);
  1110. end
  1111. else
  1112. if (oper.ref^.index <> NR_INVALID) and (oper.ref^.index <> NR_NO) then
  1113. Result := Result + debug_regname(oper.ref^.index);
  1114. if (oper.ref^.scalefactor > 1) then
  1115. Result := Result + ',' + debug_tostr(oper.ref^.scalefactor) + ')'
  1116. else
  1117. Result := Result + ')';
  1118. end;
  1119. else
  1120. Result := '[UNKNOWN]';
  1121. end;
  1122. end;
  1123. function debug_op2str(opcode: tasmop): string; inline;
  1124. begin
  1125. Result := std_op2str[opcode];
  1126. end;
  1127. function debug_opsize2str(opsize: topsize): string; inline;
  1128. begin
  1129. Result := gas_opsize2str[opsize];
  1130. end;
  1131. {$else DEBUG_AOPTCPU}
  1132. procedure TX86AsmOptimizer.DebugMsg(const s: string;p : tai);inline;
  1133. begin
  1134. end;
  1135. function debug_tostr(i: tcgint): string; inline;
  1136. begin
  1137. Result := '';
  1138. end;
  1139. function debug_regname(r: TRegister): string; inline;
  1140. begin
  1141. Result := '';
  1142. end;
  1143. function debug_operstr(oper: TOper): string; inline;
  1144. begin
  1145. Result := '';
  1146. end;
  1147. function debug_op2str(opcode: tasmop): string; inline;
  1148. begin
  1149. Result := '';
  1150. end;
  1151. function debug_opsize2str(opsize: topsize): string; inline;
  1152. begin
  1153. Result := '';
  1154. end;
  1155. {$endif DEBUG_AOPTCPU}
  1156. class function TX86AsmOptimizer.IsMOVZXAcceptable: Boolean; inline;
  1157. begin
  1158. {$ifdef x86_64}
  1159. { Always fine on x86-64 }
  1160. Result := True;
  1161. {$else x86_64}
  1162. Result :=
  1163. {$ifdef i8086}
  1164. (current_settings.cputype >= cpu_386) and
  1165. {$endif i8086}
  1166. (
  1167. { Always accept if optimising for size }
  1168. (cs_opt_size in current_settings.optimizerswitches) or
  1169. { From the Pentium II onwards, MOVZX only takes 1 cycle. [Kit] }
  1170. (current_settings.optimizecputype >= cpu_Pentium2)
  1171. );
  1172. {$endif x86_64}
  1173. end;
  1174. { Attempts to allocate a volatile integer register for use between p and hp,
  1175. using AUsedRegs for the current register usage information. Returns NR_NO
  1176. if no free register could be found }
  1177. function TX86AsmOptimizer.GetIntRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  1178. var
  1179. RegSet: TCPURegisterSet;
  1180. CurrentSuperReg: Integer;
  1181. CurrentReg: TRegister;
  1182. Currentp: tai;
  1183. Breakout: Boolean;
  1184. begin
  1185. Result := NR_NO;
  1186. RegSet :=
  1187. paramanager.get_volatile_registers_int(current_procinfo.procdef.proccalloption) +
  1188. current_procinfo.saved_regs_int;
  1189. for CurrentSuperReg in RegSet do
  1190. begin
  1191. CurrentReg := newreg(R_INTREGISTER, TSuperRegister(CurrentSuperReg), RegSize);
  1192. if not AUsedRegs[R_INTREGISTER].IsUsed(CurrentReg)
  1193. {$if defined(i386) or defined(i8086)}
  1194. { If the target size is 8-bit, make sure we can actually encode it }
  1195. and (
  1196. (RegSize >= R_SUBW) or { Not R_SUBL or R_SUBH }
  1197. (GetSupReg(CurrentReg) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX])
  1198. )
  1199. {$endif i386 or i8086}
  1200. then
  1201. begin
  1202. Currentp := p;
  1203. Breakout := False;
  1204. while not Breakout and GetNextInstruction(Currentp, Currentp) and (Currentp <> hp) do
  1205. begin
  1206. case Currentp.typ of
  1207. ait_instruction:
  1208. begin
  1209. if RegInInstruction(CurrentReg, Currentp) then
  1210. begin
  1211. Breakout := True;
  1212. Break;
  1213. end;
  1214. { Cannot allocate across an unconditional jump }
  1215. if is_calljmpuncondret(taicpu(Currentp).opcode) then
  1216. Exit;
  1217. end;
  1218. ait_marker:
  1219. { Don't try anything more if a marker is hit }
  1220. Exit;
  1221. ait_regalloc:
  1222. if (tai_regalloc(Currentp).ratype <> ra_dealloc) and SuperRegistersEqual(CurrentReg, tai_regalloc(Currentp).reg) then
  1223. begin
  1224. Breakout := True;
  1225. Break;
  1226. end;
  1227. else
  1228. ;
  1229. end;
  1230. end;
  1231. if Breakout then
  1232. { Try the next register }
  1233. Continue;
  1234. { We have a free register available }
  1235. Result := CurrentReg;
  1236. if not DontAlloc then
  1237. AllocRegBetween(CurrentReg, p, hp, AUsedRegs);
  1238. Exit;
  1239. end;
  1240. end;
  1241. end;
  1242. { Attempts to allocate a volatile MM register for use between p and hp,
  1243. using AUsedRegs for the current register usage information. Returns NR_NO
  1244. if no free register could be found }
  1245. function TX86AsmOptimizer.GetMMRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  1246. var
  1247. RegSet: TCPURegisterSet;
  1248. CurrentSuperReg: Integer;
  1249. CurrentReg: TRegister;
  1250. Currentp: tai;
  1251. Breakout: Boolean;
  1252. begin
  1253. Result := NR_NO;
  1254. RegSet :=
  1255. paramanager.get_volatile_registers_mm(current_procinfo.procdef.proccalloption) +
  1256. current_procinfo.saved_regs_mm;
  1257. for CurrentSuperReg in RegSet do
  1258. begin
  1259. CurrentReg := newreg(R_MMREGISTER, TSuperRegister(CurrentSuperReg), RegSize);
  1260. if not AUsedRegs[R_MMREGISTER].IsUsed(CurrentReg) then
  1261. begin
  1262. Currentp := p;
  1263. Breakout := False;
  1264. while not Breakout and GetNextInstruction(Currentp, Currentp) and (Currentp <> hp) do
  1265. begin
  1266. case Currentp.typ of
  1267. ait_instruction:
  1268. begin
  1269. if RegInInstruction(CurrentReg, Currentp) then
  1270. begin
  1271. Breakout := True;
  1272. Break;
  1273. end;
  1274. { Cannot allocate across an unconditional jump }
  1275. if is_calljmpuncondret(taicpu(Currentp).opcode) then
  1276. Exit;
  1277. end;
  1278. ait_marker:
  1279. { Don't try anything more if a marker is hit }
  1280. Exit;
  1281. ait_regalloc:
  1282. if (tai_regalloc(Currentp).ratype <> ra_dealloc) and SuperRegistersEqual(CurrentReg, tai_regalloc(Currentp).reg) then
  1283. begin
  1284. Breakout := True;
  1285. Break;
  1286. end;
  1287. else
  1288. ;
  1289. end;
  1290. end;
  1291. if Breakout then
  1292. { Try the next register }
  1293. Continue;
  1294. { We have a free register available }
  1295. Result := CurrentReg;
  1296. if not DontAlloc then
  1297. AllocRegBetween(CurrentReg, p, hp, AUsedRegs);
  1298. Exit;
  1299. end;
  1300. end;
  1301. end;
  1302. class function TX86AsmOptimizer.Reg1WriteOverwritesReg2Entirely(reg1, reg2: tregister): boolean;
  1303. begin
  1304. if not SuperRegistersEqual(reg1,reg2) then
  1305. exit(false);
  1306. if getregtype(reg1)<>R_INTREGISTER then
  1307. exit(true); {because SuperRegisterEqual is true}
  1308. case getsubreg(reg1) of
  1309. { A write to R_SUBL doesn't change R_SUBH and if reg2 is R_SUBW or
  1310. higher, it preserves the high bits, so the new value depends on
  1311. reg2's previous value. In other words, it is equivalent to doing:
  1312. reg2 := (reg2 and $ffffff00) or byte(reg1); }
  1313. R_SUBL:
  1314. exit(getsubreg(reg2)=R_SUBL);
  1315. { A write to R_SUBH doesn't change R_SUBL and if reg2 is R_SUBW or
  1316. higher, it actually does a:
  1317. reg2 := (reg2 and $ffff00ff) or (reg1 and $ff00); }
  1318. R_SUBH:
  1319. exit(getsubreg(reg2)=R_SUBH);
  1320. { If reg2 is R_SUBD or larger, a write to R_SUBW preserves the high 16
  1321. bits of reg2:
  1322. reg2 := (reg2 and $ffff0000) or word(reg1); }
  1323. R_SUBW:
  1324. exit(getsubreg(reg2) in [R_SUBL,R_SUBH,R_SUBW]);
  1325. { a write to R_SUBD always overwrites every other subregister,
  1326. because it clears the high 32 bits of R_SUBQ on x86_64 }
  1327. R_SUBD,
  1328. R_SUBQ:
  1329. exit(true);
  1330. else
  1331. internalerror(2017042801);
  1332. end;
  1333. end;
  1334. class function TX86AsmOptimizer.Reg1ReadDependsOnReg2(reg1, reg2: tregister): boolean;
  1335. begin
  1336. if not SuperRegistersEqual(reg1,reg2) then
  1337. exit(false);
  1338. if getregtype(reg1)<>R_INTREGISTER then
  1339. exit(true); {because SuperRegisterEqual is true}
  1340. case getsubreg(reg1) of
  1341. R_SUBL:
  1342. exit(getsubreg(reg2)<>R_SUBH);
  1343. R_SUBH:
  1344. exit(getsubreg(reg2)<>R_SUBL);
  1345. R_SUBW,
  1346. R_SUBD,
  1347. R_SUBQ:
  1348. exit(true);
  1349. else
  1350. internalerror(2017042802);
  1351. end;
  1352. end;
  1353. function TX86AsmOptimizer.PrePeepholeOptSxx(var p : tai) : boolean;
  1354. var
  1355. hp1 : tai;
  1356. l : TCGInt;
  1357. begin
  1358. result:=false;
  1359. { changes the code sequence
  1360. shr/sar const1, x
  1361. shl const2, x
  1362. to
  1363. either "sar/and", "shl/and" or just "and" depending on const1 and const2 }
  1364. if GetNextInstruction(p, hp1) and
  1365. MatchInstruction(hp1,A_SHL,[]) and
  1366. (taicpu(p).oper[0]^.typ = top_const) and
  1367. (taicpu(hp1).oper[0]^.typ = top_const) and
  1368. (taicpu(hp1).opsize = taicpu(p).opsize) and
  1369. (taicpu(hp1).oper[1]^.typ = taicpu(p).oper[1]^.typ) and
  1370. OpsEqual(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^) then
  1371. begin
  1372. if (taicpu(p).oper[0]^.val > taicpu(hp1).oper[0]^.val) and
  1373. not(cs_opt_size in current_settings.optimizerswitches) then
  1374. begin
  1375. { shr/sar const1, %reg
  1376. shl const2, %reg
  1377. with const1 > const2 }
  1378. taicpu(p).loadConst(0,taicpu(p).oper[0]^.val-taicpu(hp1).oper[0]^.val);
  1379. taicpu(hp1).opcode := A_AND;
  1380. l := (1 shl (taicpu(hp1).oper[0]^.val)) - 1;
  1381. case taicpu(p).opsize Of
  1382. S_B: taicpu(hp1).loadConst(0,l Xor $ff);
  1383. S_W: taicpu(hp1).loadConst(0,l Xor $ffff);
  1384. S_L: taicpu(hp1).loadConst(0,l Xor tcgint($ffffffff));
  1385. S_Q: taicpu(hp1).loadConst(0,l Xor tcgint($ffffffffffffffff));
  1386. else
  1387. Internalerror(2017050703)
  1388. end;
  1389. end
  1390. else if (taicpu(p).oper[0]^.val<taicpu(hp1).oper[0]^.val) and
  1391. not(cs_opt_size in current_settings.optimizerswitches) then
  1392. begin
  1393. { shr/sar const1, %reg
  1394. shl const2, %reg
  1395. with const1 < const2 }
  1396. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val-taicpu(p).oper[0]^.val);
  1397. taicpu(p).opcode := A_AND;
  1398. l := (1 shl (taicpu(p).oper[0]^.val))-1;
  1399. case taicpu(p).opsize Of
  1400. S_B: taicpu(p).loadConst(0,l Xor $ff);
  1401. S_W: taicpu(p).loadConst(0,l Xor $ffff);
  1402. S_L: taicpu(p).loadConst(0,l Xor tcgint($ffffffff));
  1403. S_Q: taicpu(p).loadConst(0,l Xor tcgint($ffffffffffffffff));
  1404. else
  1405. Internalerror(2017050702)
  1406. end;
  1407. end
  1408. else if (taicpu(p).oper[0]^.val = taicpu(hp1).oper[0]^.val) then
  1409. begin
  1410. { shr/sar const1, %reg
  1411. shl const2, %reg
  1412. with const1 = const2 }
  1413. taicpu(p).opcode := A_AND;
  1414. l := (1 shl (taicpu(p).oper[0]^.val))-1;
  1415. case taicpu(p).opsize Of
  1416. S_B: taicpu(p).loadConst(0,l Xor $ff);
  1417. S_W: taicpu(p).loadConst(0,l Xor $ffff);
  1418. S_L: taicpu(p).loadConst(0,l Xor tcgint($ffffffff));
  1419. S_Q: taicpu(p).loadConst(0,l Xor tcgint($ffffffffffffffff));
  1420. else
  1421. Internalerror(2017050701)
  1422. end;
  1423. RemoveInstruction(hp1);
  1424. end;
  1425. end;
  1426. end;
  1427. function TX86AsmOptimizer.PrePeepholeOptIMUL(var p : tai) : boolean;
  1428. var
  1429. opsize : topsize;
  1430. hp1, hp2 : tai;
  1431. tmpref : treference;
  1432. ShiftValue : Cardinal;
  1433. BaseValue : TCGInt;
  1434. begin
  1435. result:=false;
  1436. opsize:=taicpu(p).opsize;
  1437. { changes certain "imul const, %reg"'s to lea sequences }
  1438. if (MatchOpType(taicpu(p),top_const,top_reg) or
  1439. MatchOpType(taicpu(p),top_const,top_reg,top_reg)) and
  1440. (opsize in [S_L{$ifdef x86_64},S_Q{$endif x86_64}]) then
  1441. if (taicpu(p).oper[0]^.val = 1) then
  1442. if (taicpu(p).ops = 2) then
  1443. { remove "imul $1, reg" }
  1444. begin
  1445. DebugMsg(SPeepholeOptimization + 'Imul2Nop done',p);
  1446. Result := RemoveCurrentP(p);
  1447. end
  1448. else
  1449. { change "imul $1, reg1, reg2" to "mov reg1, reg2" }
  1450. begin
  1451. hp1 := taicpu.Op_Reg_Reg(A_MOV, opsize, taicpu(p).oper[1]^.reg,taicpu(p).oper[2]^.reg);
  1452. taicpu(hp1).fileinfo := taicpu(p).fileinfo;
  1453. asml.InsertAfter(hp1, p);
  1454. DebugMsg(SPeepholeOptimization + 'Imul2Mov done',p);
  1455. RemoveCurrentP(p, hp1);
  1456. Result := True;
  1457. end
  1458. else if ((taicpu(p).ops <= 2) or
  1459. (taicpu(p).oper[2]^.typ = Top_Reg)) and
  1460. not(cs_opt_size in current_settings.optimizerswitches) and
  1461. (not(GetNextInstruction(p, hp1)) or
  1462. not((tai(hp1).typ = ait_instruction) and
  1463. ((taicpu(hp1).opcode=A_Jcc) and
  1464. (taicpu(hp1).condition in [C_O,C_NO])))) then
  1465. begin
  1466. {
  1467. imul X, reg1, reg2 to
  1468. lea (reg1,reg1,Y), reg2
  1469. shl ZZ,reg2
  1470. imul XX, reg1 to
  1471. lea (reg1,reg1,YY), reg1
  1472. shl ZZ,reg2
  1473. This optimziation makes sense for pretty much every x86, except the VIA Nano3000: it has IMUL latency 2, lea/shl pair as well,
  1474. it does not exist as a separate optimization target in FPC though.
  1475. This optimziation can be applied as long as only two bits are set in the constant and those two bits are separated by
  1476. at most two zeros
  1477. }
  1478. reference_reset(tmpref,1,[]);
  1479. if (PopCnt(QWord(taicpu(p).oper[0]^.val))=2) and (BsrQWord(taicpu(p).oper[0]^.val)-BsfQWord(taicpu(p).oper[0]^.val)<=3) then
  1480. begin
  1481. ShiftValue:=BsfQWord(taicpu(p).oper[0]^.val);
  1482. BaseValue:=taicpu(p).oper[0]^.val shr ShiftValue;
  1483. TmpRef.base := taicpu(p).oper[1]^.reg;
  1484. TmpRef.index := taicpu(p).oper[1]^.reg;
  1485. if not(BaseValue in [3,5,9]) then
  1486. Internalerror(2018110101);
  1487. TmpRef.ScaleFactor := BaseValue-1;
  1488. if (taicpu(p).ops = 2) then
  1489. hp1 := taicpu.op_ref_reg(A_LEA, opsize, TmpRef, taicpu(p).oper[1]^.reg)
  1490. else
  1491. hp1 := taicpu.op_ref_reg(A_LEA, opsize, TmpRef, taicpu(p).oper[2]^.reg);
  1492. AsmL.InsertAfter(hp1,p);
  1493. DebugMsg(SPeepholeOptimization + 'Imul2LeaShl done',p);
  1494. taicpu(hp1).fileinfo:=taicpu(p).fileinfo;
  1495. RemoveCurrentP(p, hp1);
  1496. if ShiftValue>0 then
  1497. begin
  1498. hp2 := taicpu.op_const_reg(A_SHL, opsize, ShiftValue, taicpu(hp1).oper[1]^.reg);
  1499. AsmL.InsertAfter(hp2,hp1);
  1500. taicpu(hp2).fileinfo:=taicpu(hp1).fileinfo;
  1501. end;
  1502. Result := True;
  1503. end;
  1504. end;
  1505. end;
  1506. function TX86AsmOptimizer.PrePeepholeOptAND(var p : tai) : boolean;
  1507. begin
  1508. Result := False;
  1509. if MatchOperand(taicpu(p).oper[0]^, 0) and
  1510. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  1511. begin
  1512. DebugMsg(SPeepholeOptimization + 'AND 0 -> MOV 0', p);
  1513. taicpu(p).opcode := A_MOV;
  1514. Result := True;
  1515. end;
  1516. end;
  1517. function TX86AsmOptimizer.RegLoadedWithNewValue(reg: tregister; hp: tai): boolean;
  1518. var
  1519. p: taicpu absolute hp; { Implicit typecast }
  1520. i: Integer;
  1521. begin
  1522. Result := False;
  1523. if not assigned(hp) or
  1524. (hp.typ <> ait_instruction) then
  1525. Exit;
  1526. Prefetch(insprop[p.opcode]);
  1527. if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  1528. with insprop[p.opcode] do
  1529. begin
  1530. case getsubreg(reg) of
  1531. R_SUBW,R_SUBD,R_SUBQ:
  1532. Result:=
  1533. { ZF, CF, OF, SF, PF and AF must all be set in some way (ordered so the most
  1534. uncommon flags are checked first }
  1535. ([Ch_W0AuxiliaryFlag,Ch_W1AuxiliaryFlag,Ch_WAuxiliaryFlag,Ch_WUAuxiliaryFlag,Ch_WFlags] * Ch <> []) and
  1536. ([Ch_W0ParityFlag,Ch_W1ParityFlag,Ch_WParityFlag,Ch_WUParityFlag,Ch_WFlags]*Ch <> []) and
  1537. ([Ch_W0SignFlag,Ch_W1SignFlag,Ch_WSignFlag,Ch_WUSignFlag,Ch_WFlags]*Ch <> []) and
  1538. ([Ch_W0OverflowFlag,Ch_W1OverflowFlag,Ch_WOverflowFlag,Ch_WUOverflowFlag,Ch_WFlags]*Ch <> []) and
  1539. ([Ch_W0CarryFlag,Ch_W1CarryFlag,Ch_WCarryFlag,Ch_WUCarryFlag,Ch_WFlags]*Ch <> []) and
  1540. ([Ch_W0ZeroFlag,Ch_W1ZeroFlag,Ch_WZeroFlag,Ch_WUZeroFlag,Ch_WFlags]*Ch <> []);
  1541. R_SUBFLAGCARRY:
  1542. Result:=[Ch_W0CarryFlag,Ch_W1CarryFlag,Ch_WCarryFlag,Ch_WUCarryFlag,Ch_WFlags]*Ch<>[];
  1543. R_SUBFLAGPARITY:
  1544. Result:=[Ch_W0ParityFlag,Ch_W1ParityFlag,Ch_WParityFlag,Ch_WUParityFlag,Ch_WFlags]*Ch<>[];
  1545. R_SUBFLAGAUXILIARY:
  1546. Result:=[Ch_W0AuxiliaryFlag,Ch_W1AuxiliaryFlag,Ch_WAuxiliaryFlag,Ch_WUAuxiliaryFlag,Ch_WFlags]*Ch<>[];
  1547. R_SUBFLAGZERO:
  1548. Result:=[Ch_W0ZeroFlag,Ch_W1ZeroFlag,Ch_WZeroFlag,Ch_WUZeroFlag,Ch_WFlags]*Ch<>[];
  1549. R_SUBFLAGSIGN:
  1550. Result:=[Ch_W0SignFlag,Ch_W1SignFlag,Ch_WSignFlag,Ch_WUSignFlag,Ch_WFlags]*Ch<>[];
  1551. R_SUBFLAGOVERFLOW:
  1552. Result:=[Ch_W0OverflowFlag,Ch_W1OverflowFlag,Ch_WOverflowFlag,Ch_WUOverflowFlag,Ch_WFlags]*Ch<>[];
  1553. R_SUBFLAGINTERRUPT:
  1554. Result:=[Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags]*Ch<>[];
  1555. R_SUBFLAGDIRECTION:
  1556. Result:=[Ch_W0DirFlag,Ch_W1DirFlag,Ch_WFlags]*Ch<>[];
  1557. else
  1558. internalerror(2017050501);
  1559. end;
  1560. exit;
  1561. end;
  1562. { Handle special cases first }
  1563. case p.opcode of
  1564. A_MOV, A_MOVZX, A_MOVSX, A_LEA, A_VMOVSS, A_VMOVSD, A_VMOVAPD,
  1565. A_VMOVAPS, A_VMOVQ, A_MOVSS, A_MOVSD, A_MOVQ, A_MOVAPD, A_MOVAPS:
  1566. begin
  1567. Result :=
  1568. (p.ops=2) and { A_MOVSD can have zero operands, so this check is needed }
  1569. (p.oper[1]^.typ = top_reg) and
  1570. (Reg1WriteOverwritesReg2Entirely(p.oper[1]^.reg,reg)) and
  1571. (
  1572. (p.oper[0]^.typ = top_const) or
  1573. (
  1574. (p.oper[0]^.typ = top_reg) and
  1575. not(Reg1ReadDependsOnReg2(p.oper[0]^.reg,reg))
  1576. ) or (
  1577. (p.oper[0]^.typ = top_ref) and
  1578. not RegInRef(reg,p.oper[0]^.ref^)
  1579. )
  1580. );
  1581. end;
  1582. A_MUL, A_IMUL:
  1583. Result :=
  1584. (
  1585. (p.ops=3) and { IMUL only }
  1586. (Reg1WriteOverwritesReg2Entirely(p.oper[2]^.reg,reg)) and
  1587. (
  1588. (
  1589. (p.oper[1]^.typ=top_reg) and
  1590. not Reg1ReadDependsOnReg2(p.oper[1]^.reg,reg)
  1591. ) or (
  1592. (p.oper[1]^.typ=top_ref) and
  1593. not RegInRef(reg,p.oper[1]^.ref^)
  1594. )
  1595. )
  1596. ) or (
  1597. (
  1598. (p.ops=1) and
  1599. (
  1600. (
  1601. (
  1602. (p.oper[0]^.typ=top_reg) and
  1603. not Reg1ReadDependsOnReg2(p.oper[0]^.reg,reg)
  1604. )
  1605. ) or (
  1606. (p.oper[0]^.typ=top_ref) and
  1607. not RegInRef(reg,p.oper[0]^.ref^)
  1608. )
  1609. ) and (
  1610. (
  1611. (p.opsize=S_B) and
  1612. Reg1WriteOverwritesReg2Entirely(NR_AX,reg) and
  1613. not Reg1ReadDependsOnReg2(NR_AL,reg)
  1614. ) or (
  1615. (p.opsize=S_W) and
  1616. Reg1WriteOverwritesReg2Entirely(NR_DX,reg)
  1617. ) or (
  1618. (p.opsize=S_L) and
  1619. Reg1WriteOverwritesReg2Entirely(NR_EDX,reg)
  1620. {$ifdef x86_64}
  1621. ) or (
  1622. (p.opsize=S_Q) and
  1623. Reg1WriteOverwritesReg2Entirely(NR_RDX,reg)
  1624. {$endif x86_64}
  1625. )
  1626. )
  1627. )
  1628. );
  1629. A_CBW:
  1630. Result := Reg1WriteOverwritesReg2Entirely(NR_AX,reg) and not(Reg1ReadDependsOnReg2(NR_AL,reg));
  1631. {$ifndef x86_64}
  1632. A_LDS:
  1633. Result := (reg=NR_DS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1634. A_LES:
  1635. Result := (reg=NR_ES) and not(RegInRef(reg,p.oper[0]^.ref^));
  1636. {$endif not x86_64}
  1637. A_LFS:
  1638. Result := (reg=NR_FS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1639. A_LGS:
  1640. Result := (reg=NR_GS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1641. A_LSS:
  1642. Result := (reg=NR_SS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1643. A_LAHF{$ifndef x86_64}, A_AAM{$endif not x86_64}:
  1644. Result := Reg1WriteOverwritesReg2Entirely(NR_AH,reg);
  1645. A_LODSB:
  1646. Result := Reg1WriteOverwritesReg2Entirely(NR_AL,reg);
  1647. A_LODSW:
  1648. Result := Reg1WriteOverwritesReg2Entirely(NR_AX,reg);
  1649. {$ifdef x86_64}
  1650. A_LODSQ:
  1651. Result := Reg1WriteOverwritesReg2Entirely(NR_RAX,reg);
  1652. {$endif x86_64}
  1653. A_LODSD:
  1654. Result := Reg1WriteOverwritesReg2Entirely(NR_EAX,reg);
  1655. A_FSTSW, A_FNSTSW:
  1656. Result := (p.oper[0]^.typ=top_reg) and Reg1WriteOverwritesReg2Entirely(p.oper[0]^.reg,reg);
  1657. else
  1658. begin
  1659. with insprop[p.opcode] do
  1660. begin
  1661. if (
  1662. { xor %reg,%reg etc. is classed as a new value }
  1663. (([Ch_NoReadIfEqualRegs]*Ch)<>[]) and
  1664. MatchOpType(p, top_reg, top_reg) and
  1665. (p.oper[0]^.reg = p.oper[1]^.reg) and
  1666. Reg1WriteOverwritesReg2Entirely(p.oper[1]^.reg,reg)
  1667. ) then
  1668. begin
  1669. Result := True;
  1670. Exit;
  1671. end;
  1672. { Make sure the entire register is overwritten }
  1673. if (getregtype(reg) = R_INTREGISTER) then
  1674. begin
  1675. if (p.ops > 0) then
  1676. begin
  1677. if RegInOp(reg, p.oper[0]^) then
  1678. begin
  1679. if (p.oper[0]^.typ = top_ref) then
  1680. begin
  1681. if RegInRef(reg, p.oper[0]^.ref^) then
  1682. begin
  1683. Result := False;
  1684. Exit;
  1685. end;
  1686. end
  1687. else if (p.oper[0]^.typ = top_reg) then
  1688. begin
  1689. if ([Ch_ROp1, Ch_RWOp1, Ch_MOp1]*Ch<>[]) then
  1690. begin
  1691. Result := False;
  1692. Exit;
  1693. end
  1694. else if ([Ch_WOp1]*Ch<>[]) then
  1695. begin
  1696. if Reg1WriteOverwritesReg2Entirely(p.oper[0]^.reg, reg) then
  1697. Result := True
  1698. else
  1699. begin
  1700. Result := False;
  1701. Exit;
  1702. end;
  1703. end;
  1704. end;
  1705. end;
  1706. if (p.ops > 1) then
  1707. begin
  1708. if RegInOp(reg, p.oper[1]^) then
  1709. begin
  1710. if (p.oper[1]^.typ = top_ref) then
  1711. begin
  1712. if RegInRef(reg, p.oper[1]^.ref^) then
  1713. begin
  1714. Result := False;
  1715. Exit;
  1716. end;
  1717. end
  1718. else if (p.oper[1]^.typ = top_reg) then
  1719. begin
  1720. if ([Ch_ROp2, Ch_RWOp2, Ch_MOp2]*Ch<>[]) then
  1721. begin
  1722. Result := False;
  1723. Exit;
  1724. end
  1725. else if ([Ch_WOp2]*Ch<>[]) then
  1726. begin
  1727. if Reg1WriteOverwritesReg2Entirely(p.oper[1]^.reg, reg) then
  1728. Result := True
  1729. else
  1730. begin
  1731. Result := False;
  1732. Exit;
  1733. end;
  1734. end;
  1735. end;
  1736. end;
  1737. if (p.ops > 2) then
  1738. begin
  1739. if RegInOp(reg, p.oper[2]^) then
  1740. begin
  1741. if (p.oper[2]^.typ = top_ref) then
  1742. begin
  1743. if RegInRef(reg, p.oper[2]^.ref^) then
  1744. begin
  1745. Result := False;
  1746. Exit;
  1747. end;
  1748. end
  1749. else if (p.oper[2]^.typ = top_reg) then
  1750. begin
  1751. if ([Ch_ROp3, Ch_RWOp3, Ch_MOp3]*Ch<>[]) then
  1752. begin
  1753. Result := False;
  1754. Exit;
  1755. end
  1756. else if ([Ch_WOp3]*Ch<>[]) then
  1757. begin
  1758. if Reg1WriteOverwritesReg2Entirely(p.oper[2]^.reg, reg) then
  1759. Result := True
  1760. else
  1761. begin
  1762. Result := False;
  1763. Exit;
  1764. end;
  1765. end;
  1766. end;
  1767. end;
  1768. if (p.ops > 3) and RegInOp(reg, p.oper[3]^) then
  1769. begin
  1770. if (p.oper[3]^.typ = top_ref) then
  1771. begin
  1772. if RegInRef(reg, p.oper[3]^.ref^) then
  1773. begin
  1774. Result := False;
  1775. Exit;
  1776. end;
  1777. end
  1778. else if (p.oper[3]^.typ = top_reg) then
  1779. begin
  1780. if ([Ch_ROp4, Ch_RWOp4, Ch_MOp4]*Ch<>[]) then
  1781. begin
  1782. Result := False;
  1783. Exit;
  1784. end
  1785. else if ([Ch_WOp4]*Ch<>[]) then
  1786. begin
  1787. if Reg1WriteOverwritesReg2Entirely(p.oper[3]^.reg, reg) then
  1788. Result := True
  1789. else
  1790. begin
  1791. Result := False;
  1792. Exit;
  1793. end;
  1794. end;
  1795. end;
  1796. end;
  1797. end;
  1798. end;
  1799. end;
  1800. { Don't do these ones first in case an input operand is equal to an explicit output register }
  1801. case getsupreg(reg) of
  1802. RS_EAX:
  1803. if ([Ch_WEAX{$ifdef x86_64},Ch_WRAX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EAX, reg) then
  1804. begin
  1805. Result := True;
  1806. Exit;
  1807. end;
  1808. RS_ECX:
  1809. if ([Ch_WECX{$ifdef x86_64},Ch_WRCX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_ECX, reg) then
  1810. begin
  1811. Result := True;
  1812. Exit;
  1813. end;
  1814. RS_EDX:
  1815. if ([Ch_REDX{$ifdef x86_64},Ch_WRDX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EDX, reg) then
  1816. begin
  1817. Result := True;
  1818. Exit;
  1819. end;
  1820. RS_EBX:
  1821. if ([Ch_WEBX{$ifdef x86_64},Ch_WRBX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EBX, reg) then
  1822. begin
  1823. Result := True;
  1824. Exit;
  1825. end;
  1826. RS_ESP:
  1827. if ([Ch_WESP{$ifdef x86_64},Ch_WRSP{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_ESP, reg) then
  1828. begin
  1829. Result := True;
  1830. Exit;
  1831. end;
  1832. RS_EBP:
  1833. if ([Ch_WEBP{$ifdef x86_64},Ch_WRBP{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EBP, reg) then
  1834. begin
  1835. Result := True;
  1836. Exit;
  1837. end;
  1838. RS_ESI:
  1839. if ([Ch_WESI{$ifdef x86_64},Ch_WRSI{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_ESI, reg) then
  1840. begin
  1841. Result := True;
  1842. Exit;
  1843. end;
  1844. RS_EDI:
  1845. if ([Ch_WEDI{$ifdef x86_64},Ch_WRDI{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EDI, reg) then
  1846. begin
  1847. Result := True;
  1848. Exit;
  1849. end;
  1850. else
  1851. ;
  1852. end;
  1853. end;
  1854. end;
  1855. end;
  1856. end;
  1857. end;
  1858. class function TX86AsmOptimizer.IsExitCode(p : tai) : boolean;
  1859. var
  1860. hp2,hp3 : tai;
  1861. begin
  1862. { some x86-64 issue a NOP before the real exit code }
  1863. if MatchInstruction(p,A_NOP,[]) then
  1864. GetNextInstruction(p,p);
  1865. result:=assigned(p) and (p.typ=ait_instruction) and
  1866. ((taicpu(p).opcode = A_RET) or
  1867. ((taicpu(p).opcode=A_LEAVE) and
  1868. GetNextInstruction(p,hp2) and
  1869. MatchInstruction(hp2,A_RET,[S_NO])
  1870. ) or
  1871. (((taicpu(p).opcode=A_LEA) and
  1872. MatchOpType(taicpu(p),top_ref,top_reg) and
  1873. (taicpu(p).oper[0]^.ref^.base=NR_STACK_POINTER_REG) and
  1874. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG)
  1875. ) and
  1876. GetNextInstruction(p,hp2) and
  1877. MatchInstruction(hp2,A_RET,[S_NO])
  1878. ) or
  1879. ((((taicpu(p).opcode=A_MOV) and
  1880. MatchOpType(taicpu(p),top_reg,top_reg) and
  1881. (taicpu(p).oper[0]^.reg=current_procinfo.framepointer) and
  1882. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG)) or
  1883. ((taicpu(p).opcode=A_LEA) and
  1884. MatchOpType(taicpu(p),top_ref,top_reg) and
  1885. (taicpu(p).oper[0]^.ref^.base=current_procinfo.framepointer) and
  1886. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG)
  1887. )
  1888. ) and
  1889. GetNextInstruction(p,hp2) and
  1890. MatchInstruction(hp2,A_POP,[reg2opsize(current_procinfo.framepointer)]) and
  1891. MatchOpType(taicpu(hp2),top_reg) and
  1892. (taicpu(hp2).oper[0]^.reg=current_procinfo.framepointer) and
  1893. GetNextInstruction(hp2,hp3) and
  1894. MatchInstruction(hp3,A_RET,[S_NO])
  1895. )
  1896. );
  1897. end;
  1898. class function TX86AsmOptimizer.isFoldableArithOp(hp1: taicpu; reg: tregister): boolean;
  1899. begin
  1900. isFoldableArithOp := False;
  1901. case hp1.opcode of
  1902. A_ADD,A_SUB,A_OR,A_XOR,A_AND,A_SHL,A_SHR,A_SAR:
  1903. isFoldableArithOp :=
  1904. ((taicpu(hp1).oper[0]^.typ = top_const) or
  1905. ((taicpu(hp1).oper[0]^.typ = top_reg) and
  1906. (taicpu(hp1).oper[0]^.reg <> reg))) and
  1907. (taicpu(hp1).oper[1]^.typ = top_reg) and
  1908. (taicpu(hp1).oper[1]^.reg = reg);
  1909. A_INC,A_DEC,A_NEG,A_NOT:
  1910. isFoldableArithOp :=
  1911. (taicpu(hp1).oper[0]^.typ = top_reg) and
  1912. (taicpu(hp1).oper[0]^.reg = reg);
  1913. else
  1914. ;
  1915. end;
  1916. end;
  1917. procedure TX86AsmOptimizer.RemoveLastDeallocForFuncRes(p: tai);
  1918. procedure DoRemoveLastDeallocForFuncRes( supreg: tsuperregister);
  1919. var
  1920. hp2: tai;
  1921. begin
  1922. hp2 := p;
  1923. repeat
  1924. hp2 := tai(hp2.previous);
  1925. if assigned(hp2) and
  1926. (hp2.typ = ait_regalloc) and
  1927. (tai_regalloc(hp2).ratype=ra_dealloc) and
  1928. (getregtype(tai_regalloc(hp2).reg) = R_INTREGISTER) and
  1929. (getsupreg(tai_regalloc(hp2).reg) = supreg) then
  1930. begin
  1931. RemoveInstruction(hp2);
  1932. break;
  1933. end;
  1934. until not(assigned(hp2)) or regInInstruction(newreg(R_INTREGISTER,supreg,R_SUBWHOLE),hp2);
  1935. end;
  1936. begin
  1937. case current_procinfo.procdef.returndef.typ of
  1938. arraydef,recorddef,pointerdef,
  1939. stringdef,enumdef,procdef,objectdef,errordef,
  1940. filedef,setdef,procvardef,
  1941. classrefdef,forwarddef:
  1942. DoRemoveLastDeallocForFuncRes(RS_EAX);
  1943. orddef:
  1944. if current_procinfo.procdef.returndef.size <> 0 then
  1945. begin
  1946. DoRemoveLastDeallocForFuncRes(RS_EAX);
  1947. { for int64/qword }
  1948. if current_procinfo.procdef.returndef.size = 8 then
  1949. DoRemoveLastDeallocForFuncRes(RS_EDX);
  1950. end;
  1951. else
  1952. ;
  1953. end;
  1954. end;
  1955. function TX86AsmOptimizer.OptPass1_V_MOVAP(var p : tai) : boolean;
  1956. var
  1957. hp1,hp2 : tai;
  1958. begin
  1959. result:=false;
  1960. if MatchOpType(taicpu(p),top_reg,top_reg) then
  1961. begin
  1962. { vmova* reg1,reg1
  1963. =>
  1964. <nop> }
  1965. if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^) then
  1966. begin
  1967. RemoveCurrentP(p);
  1968. result:=true;
  1969. exit;
  1970. end
  1971. else if GetNextInstruction(p,hp1) then
  1972. begin
  1973. if MatchInstruction(hp1,[taicpu(p).opcode],[S_NO]) and
  1974. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) then
  1975. begin
  1976. { vmova* reg1,reg2
  1977. vmova* reg2,reg3
  1978. dealloc reg2
  1979. =>
  1980. vmova* reg1,reg3 }
  1981. TransferUsedRegs(TmpUsedRegs);
  1982. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  1983. if MatchOpType(taicpu(hp1),top_reg,top_reg) and
  1984. not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  1985. begin
  1986. DebugMsg(SPeepholeOptimization + '(V)MOVA*(V)MOVA*2(V)MOVA* 1',p);
  1987. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  1988. RemoveInstruction(hp1);
  1989. result:=true;
  1990. exit;
  1991. end
  1992. { special case:
  1993. vmova* reg1,<op>
  1994. vmova* <op>,reg1
  1995. =>
  1996. vmova* reg1,<op> }
  1997. else if MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[1]^) and
  1998. ((taicpu(p).oper[0]^.typ<>top_ref) or
  1999. (not(vol_read in taicpu(p).oper[0]^.ref^.volatility))
  2000. ) then
  2001. begin
  2002. DebugMsg(SPeepholeOptimization + '(V)MOVA*(V)MOVA*2(V)MOVA* 2',p);
  2003. RemoveInstruction(hp1);
  2004. result:=true;
  2005. exit;
  2006. end
  2007. end
  2008. else if ((MatchInstruction(p,[A_MOVAPS,A_VMOVAPS],[S_NO]) and
  2009. MatchInstruction(hp1,[A_MOVSS,A_VMOVSS],[S_NO])) or
  2010. ((MatchInstruction(p,[A_MOVAPD,A_VMOVAPD],[S_NO]) and
  2011. MatchInstruction(hp1,[A_MOVSD,A_VMOVSD],[S_NO])))
  2012. ) and
  2013. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) then
  2014. begin
  2015. { vmova* reg1,reg2
  2016. vmovs* reg2,<op>
  2017. dealloc reg2
  2018. =>
  2019. vmovs* reg1,reg3 }
  2020. TransferUsedRegs(TmpUsedRegs);
  2021. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2022. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  2023. begin
  2024. DebugMsg(SPeepholeOptimization + '(V)MOVA*(V)MOVS*2(V)MOVS* 1',p);
  2025. taicpu(p).opcode:=taicpu(hp1).opcode;
  2026. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  2027. RemoveInstruction(hp1);
  2028. result:=true;
  2029. exit;
  2030. end
  2031. end;
  2032. end;
  2033. if GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg) then
  2034. begin
  2035. if MatchInstruction(hp1,[A_VFMADDPD,
  2036. A_VFMADD132PD,
  2037. A_VFMADD132PS,
  2038. A_VFMADD132SD,
  2039. A_VFMADD132SS,
  2040. A_VFMADD213PD,
  2041. A_VFMADD213PS,
  2042. A_VFMADD213SD,
  2043. A_VFMADD213SS,
  2044. A_VFMADD231PD,
  2045. A_VFMADD231PS,
  2046. A_VFMADD231SD,
  2047. A_VFMADD231SS,
  2048. A_VFMADDSUB132PD,
  2049. A_VFMADDSUB132PS,
  2050. A_VFMADDSUB213PD,
  2051. A_VFMADDSUB213PS,
  2052. A_VFMADDSUB231PD,
  2053. A_VFMADDSUB231PS,
  2054. A_VFMSUB132PD,
  2055. A_VFMSUB132PS,
  2056. A_VFMSUB132SD,
  2057. A_VFMSUB132SS,
  2058. A_VFMSUB213PD,
  2059. A_VFMSUB213PS,
  2060. A_VFMSUB213SD,
  2061. A_VFMSUB213SS,
  2062. A_VFMSUB231PD,
  2063. A_VFMSUB231PS,
  2064. A_VFMSUB231SD,
  2065. A_VFMSUB231SS,
  2066. A_VFMSUBADD132PD,
  2067. A_VFMSUBADD132PS,
  2068. A_VFMSUBADD213PD,
  2069. A_VFMSUBADD213PS,
  2070. A_VFMSUBADD231PD,
  2071. A_VFMSUBADD231PS,
  2072. A_VFNMADD132PD,
  2073. A_VFNMADD132PS,
  2074. A_VFNMADD132SD,
  2075. A_VFNMADD132SS,
  2076. A_VFNMADD213PD,
  2077. A_VFNMADD213PS,
  2078. A_VFNMADD213SD,
  2079. A_VFNMADD213SS,
  2080. A_VFNMADD231PD,
  2081. A_VFNMADD231PS,
  2082. A_VFNMADD231SD,
  2083. A_VFNMADD231SS,
  2084. A_VFNMSUB132PD,
  2085. A_VFNMSUB132PS,
  2086. A_VFNMSUB132SD,
  2087. A_VFNMSUB132SS,
  2088. A_VFNMSUB213PD,
  2089. A_VFNMSUB213PS,
  2090. A_VFNMSUB213SD,
  2091. A_VFNMSUB213SS,
  2092. A_VFNMSUB231PD,
  2093. A_VFNMSUB231PS,
  2094. A_VFNMSUB231SD,
  2095. A_VFNMSUB231SS],[S_NO]) and
  2096. { we mix single and double opperations here because we assume that the compiler
  2097. generates vmovapd only after double operations and vmovaps only after single operations }
  2098. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[2]^) and
  2099. GetNextInstruction(hp1,hp2) and
  2100. MatchInstruction(hp2,[A_VMOVAPD,A_VMOVAPS,A_MOVAPD,A_MOVAPS],[S_NO]) and
  2101. MatchOperand(taicpu(p).oper[0]^,taicpu(hp2).oper[1]^) then
  2102. begin
  2103. TransferUsedRegs(TmpUsedRegs);
  2104. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2105. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  2106. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  2107. begin
  2108. taicpu(hp1).loadoper(2,taicpu(p).oper[0]^);
  2109. RemoveCurrentP(p);
  2110. RemoveInstruction(hp2);
  2111. end;
  2112. end
  2113. else if (hp1.typ = ait_instruction) and
  2114. GetNextInstruction(hp1, hp2) and
  2115. MatchInstruction(hp2,taicpu(p).opcode,[]) and
  2116. OpsEqual(taicpu(hp2).oper[1]^, taicpu(p).oper[0]^) and
  2117. MatchOpType(taicpu(hp2),top_reg,top_reg) and
  2118. MatchOperand(taicpu(hp2).oper[0]^,taicpu(p).oper[1]^) and
  2119. (((taicpu(p).opcode=A_MOVAPS) and
  2120. ((taicpu(hp1).opcode=A_ADDSS) or (taicpu(hp1).opcode=A_SUBSS) or
  2121. (taicpu(hp1).opcode=A_MULSS) or (taicpu(hp1).opcode=A_DIVSS))) or
  2122. ((taicpu(p).opcode=A_MOVAPD) and
  2123. ((taicpu(hp1).opcode=A_ADDSD) or (taicpu(hp1).opcode=A_SUBSD) or
  2124. (taicpu(hp1).opcode=A_MULSD) or (taicpu(hp1).opcode=A_DIVSD)))
  2125. ) then
  2126. { change
  2127. movapX reg,reg2
  2128. addsX/subsX/... reg3, reg2
  2129. movapX reg2,reg
  2130. to
  2131. addsX/subsX/... reg3,reg
  2132. }
  2133. begin
  2134. TransferUsedRegs(TmpUsedRegs);
  2135. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2136. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  2137. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  2138. begin
  2139. DebugMsg(SPeepholeOptimization + 'MovapXOpMovapX2Op ('+
  2140. debug_op2str(taicpu(p).opcode)+' '+
  2141. debug_op2str(taicpu(hp1).opcode)+' '+
  2142. debug_op2str(taicpu(hp2).opcode)+') done',p);
  2143. { we cannot eliminate the first move if
  2144. the operations uses the same register for source and dest }
  2145. if not(OpsEqual(taicpu(hp1).oper[1]^,taicpu(hp1).oper[0]^)) then
  2146. { Remember that hp1 is not necessarily the immediate
  2147. next instruction }
  2148. RemoveCurrentP(p);
  2149. taicpu(hp1).loadoper(1, taicpu(hp2).oper[1]^);
  2150. RemoveInstruction(hp2);
  2151. result:=true;
  2152. end;
  2153. end
  2154. else if (hp1.typ = ait_instruction) and
  2155. (((taicpu(p).opcode=A_VMOVAPD) and
  2156. (taicpu(hp1).opcode=A_VCOMISD)) or
  2157. ((taicpu(p).opcode=A_VMOVAPS) and
  2158. ((taicpu(hp1).opcode=A_VCOMISS))
  2159. )
  2160. ) and not(OpsEqual(taicpu(hp1).oper[1]^,taicpu(hp1).oper[0]^)) then
  2161. { change
  2162. movapX reg,reg1
  2163. vcomisX reg1,reg1
  2164. to
  2165. vcomisX reg,reg
  2166. }
  2167. begin
  2168. TransferUsedRegs(TmpUsedRegs);
  2169. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2170. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  2171. begin
  2172. DebugMsg(SPeepholeOptimization + 'MovapXComisX2ComisX2 ('+
  2173. debug_op2str(taicpu(p).opcode)+' '+
  2174. debug_op2str(taicpu(hp1).opcode)+') done',p);
  2175. if OpsEqual(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) then
  2176. taicpu(hp1).loadoper(0, taicpu(p).oper[0]^);
  2177. if OpsEqual(taicpu(p).oper[1]^,taicpu(hp1).oper[1]^) then
  2178. taicpu(hp1).loadoper(1, taicpu(p).oper[0]^);
  2179. RemoveCurrentP(p);
  2180. result:=true;
  2181. exit;
  2182. end;
  2183. end
  2184. end;
  2185. end;
  2186. end;
  2187. function TX86AsmOptimizer.OptPass1VOP(var p : tai) : boolean;
  2188. var
  2189. hp1 : tai;
  2190. begin
  2191. result:=false;
  2192. { replace
  2193. V<Op>X %mreg1,%mreg2,%mreg3
  2194. VMovX %mreg3,%mreg4
  2195. dealloc %mreg3
  2196. by
  2197. V<Op>X %mreg1,%mreg2,%mreg4
  2198. ?
  2199. }
  2200. if GetNextInstruction(p,hp1) and
  2201. { we mix single and double operations here because we assume that the compiler
  2202. generates vmovapd only after double operations and vmovaps only after single operations }
  2203. MatchInstruction(hp1,A_VMOVAPD,A_VMOVAPS,[S_NO]) and
  2204. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  2205. (taicpu(hp1).oper[1]^.typ=top_reg) then
  2206. begin
  2207. TransferUsedRegs(TmpUsedRegs);
  2208. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2209. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  2210. begin
  2211. taicpu(p).loadoper(2,taicpu(hp1).oper[1]^);
  2212. DebugMsg(SPeepholeOptimization + 'VOpVmov2VOp done',p);
  2213. RemoveInstruction(hp1);
  2214. result:=true;
  2215. end;
  2216. end;
  2217. end;
  2218. { Replaces all references to AOldReg in a memory reference to ANewReg }
  2219. class function TX86AsmOptimizer.ReplaceRegisterInRef(var ref: TReference; const AOldReg, ANewReg: TRegister): Boolean;
  2220. begin
  2221. Result := False;
  2222. { For safety reasons, only check for exact register matches }
  2223. { Check base register }
  2224. if (ref.base = AOldReg) then
  2225. begin
  2226. ref.base := ANewReg;
  2227. Result := True;
  2228. end;
  2229. { Check index register }
  2230. if (ref.index = AOldReg) then
  2231. begin
  2232. ref.index := ANewReg;
  2233. Result := True;
  2234. end;
  2235. end;
  2236. { Replaces all references to AOldReg in an operand to ANewReg }
  2237. class function TX86AsmOptimizer.ReplaceRegisterInOper(const p: taicpu; const OperIdx: Integer; const AOldReg, ANewReg: TRegister): Boolean;
  2238. var
  2239. OldSupReg, NewSupReg: TSuperRegister;
  2240. OldSubReg, NewSubReg: TSubRegister;
  2241. OldRegType: TRegisterType;
  2242. ThisOper: POper;
  2243. begin
  2244. ThisOper := p.oper[OperIdx]; { Faster to access overall }
  2245. Result := False;
  2246. if (AOldReg = NR_NO) or (ANewReg = NR_NO) then
  2247. InternalError(2020011801);
  2248. OldSupReg := getsupreg(AOldReg);
  2249. OldSubReg := getsubreg(AOldReg);
  2250. OldRegType := getregtype(AOldReg);
  2251. NewSupReg := getsupreg(ANewReg);
  2252. NewSubReg := getsubreg(ANewReg);
  2253. if OldRegType <> getregtype(ANewReg) then
  2254. InternalError(2020011802);
  2255. if OldSubReg <> NewSubReg then
  2256. InternalError(2020011803);
  2257. case ThisOper^.typ of
  2258. top_reg:
  2259. if (
  2260. (ThisOper^.reg = AOldReg) or
  2261. (
  2262. (OldRegType = R_INTREGISTER) and
  2263. (getsupreg(ThisOper^.reg) = OldSupReg) and
  2264. (getregtype(ThisOper^.reg) = R_INTREGISTER) and
  2265. (
  2266. (getsubreg(ThisOper^.reg) <= OldSubReg)
  2267. {$ifndef x86_64}
  2268. and (
  2269. { Under i386 and i8086, ESI, EDI, EBP and ESP
  2270. don't have an 8-bit representation }
  2271. (getsubreg(ThisOper^.reg) >= R_SUBW) or
  2272. not (NewSupReg in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  2273. )
  2274. {$endif x86_64}
  2275. )
  2276. )
  2277. ) then
  2278. begin
  2279. ThisOper^.reg := newreg(getregtype(ANewReg), NewSupReg, getsubreg(p.oper[OperIdx]^.reg));
  2280. Result := True;
  2281. end;
  2282. top_ref:
  2283. if ReplaceRegisterInRef(ThisOper^.ref^, AOldReg, ANewReg) then
  2284. Result := True;
  2285. else
  2286. ;
  2287. end;
  2288. end;
  2289. { Replaces all references to AOldReg in an instruction to ANewReg }
  2290. class function TX86AsmOptimizer.ReplaceRegisterInInstruction(const p: taicpu; const AOldReg, ANewReg: TRegister): Boolean;
  2291. const
  2292. ReadFlag: array[0..3] of TInsChange = (Ch_Rop1, Ch_Rop2, Ch_Rop3, Ch_Rop4);
  2293. var
  2294. OperIdx: Integer;
  2295. begin
  2296. Result := False;
  2297. for OperIdx := 0 to p.ops - 1 do
  2298. if (ReadFlag[OperIdx] in InsProp[p.Opcode].Ch) then
  2299. begin
  2300. { The shift and rotate instructions can only use CL }
  2301. if not (
  2302. (OperIdx = 0) and
  2303. { This second condition just helps to avoid unnecessarily
  2304. calling MatchInstruction for 10 different opcodes }
  2305. (p.oper[0]^.reg = NR_CL) and
  2306. MatchInstruction(p, [A_RCL, A_RCR, A_ROL, A_ROR, A_SAL, A_SAR, A_SHL, A_SHLD, A_SHR, A_SHRD], [])
  2307. ) then
  2308. Result := ReplaceRegisterInOper(p, OperIdx, AOldReg, ANewReg) or Result;
  2309. end
  2310. else if p.oper[OperIdx]^.typ = top_ref then
  2311. { It's okay to replace registers in references that get written to }
  2312. Result := ReplaceRegisterInOper(p, OperIdx, AOldReg, ANewReg) or Result;
  2313. end;
  2314. class function TX86AsmOptimizer.IsRefSafe(const ref: PReference): Boolean;
  2315. begin
  2316. Result :=
  2317. (ref^.index = NR_NO) and
  2318. (
  2319. {$ifdef x86_64}
  2320. (
  2321. (ref^.base = NR_RIP) and
  2322. (ref^.refaddr in [addr_pic, addr_pic_no_got])
  2323. ) or
  2324. {$endif x86_64}
  2325. (ref^.refaddr = addr_full) or
  2326. (ref^.base = NR_STACK_POINTER_REG) or
  2327. (ref^.base = current_procinfo.framepointer)
  2328. );
  2329. end;
  2330. function TX86AsmOptimizer.ConvertLEA(const p: taicpu): Boolean;
  2331. var
  2332. l: asizeint;
  2333. begin
  2334. Result := False;
  2335. { Should have been checked previously }
  2336. if p.opcode <> A_LEA then
  2337. InternalError(2020072501);
  2338. { do not mess with the stack point as adjusting it by lea is recommend, except if we optimize for size }
  2339. if (p.oper[1]^.reg=NR_STACK_POINTER_REG) and
  2340. not(cs_opt_size in current_settings.optimizerswitches) then
  2341. exit;
  2342. with p.oper[0]^.ref^ do
  2343. begin
  2344. if (base <> p.oper[1]^.reg) or
  2345. (index <> NR_NO) or
  2346. assigned(symbol) then
  2347. exit;
  2348. l:=offset;
  2349. if (l=1) and UseIncDec then
  2350. begin
  2351. p.opcode:=A_INC;
  2352. p.loadreg(0,p.oper[1]^.reg);
  2353. p.ops:=1;
  2354. DebugMsg(SPeepholeOptimization + 'Lea2Inc done',p);
  2355. end
  2356. else if (l=-1) and UseIncDec then
  2357. begin
  2358. p.opcode:=A_DEC;
  2359. p.loadreg(0,p.oper[1]^.reg);
  2360. p.ops:=1;
  2361. DebugMsg(SPeepholeOptimization + 'Lea2Dec done',p);
  2362. end
  2363. else
  2364. begin
  2365. if (l<0) and (l<>-2147483648) then
  2366. begin
  2367. p.opcode:=A_SUB;
  2368. p.loadConst(0,-l);
  2369. DebugMsg(SPeepholeOptimization + 'Lea2Sub done',p);
  2370. end
  2371. else
  2372. begin
  2373. p.opcode:=A_ADD;
  2374. p.loadConst(0,l);
  2375. DebugMsg(SPeepholeOptimization + 'Lea2Add done',p);
  2376. end;
  2377. end;
  2378. end;
  2379. Result := True;
  2380. end;
  2381. function TX86AsmOptimizer.DeepMOVOpt(const p_mov: taicpu; const hp: taicpu): Boolean;
  2382. var
  2383. CurrentReg, ReplaceReg: TRegister;
  2384. begin
  2385. Result := False;
  2386. ReplaceReg := taicpu(p_mov).oper[0]^.reg;
  2387. CurrentReg := taicpu(p_mov).oper[1]^.reg;
  2388. case hp.opcode of
  2389. A_FSTSW, A_FNSTSW,
  2390. A_IN, A_INS, A_OUT, A_OUTS,
  2391. A_CMPS, A_LODS, A_MOVS, A_SCAS, A_STOS:
  2392. { These routines have explicit operands, but they are restricted in
  2393. what they can be (e.g. IN and OUT can only read from AL, AX or
  2394. EAX. }
  2395. Exit;
  2396. A_IMUL:
  2397. begin
  2398. { The 1-operand version writes to implicit registers
  2399. The 2-operand version reads from the first operator, and reads
  2400. from and writes to the second (equivalent to Ch_ROp1, ChRWOp2).
  2401. the 3-operand version reads from a register that it doesn't write to
  2402. }
  2403. case hp.ops of
  2404. 1:
  2405. if (
  2406. (
  2407. (hp.opsize = S_B) and (getsupreg(CurrentReg) <> RS_EAX)
  2408. ) or
  2409. not (getsupreg(CurrentReg) in [RS_EAX, RS_EDX])
  2410. ) and ReplaceRegisterInOper(hp, 0, CurrentReg, ReplaceReg) then
  2411. begin
  2412. Result := True;
  2413. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovIMul2MovIMul 1)', hp);
  2414. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2415. end;
  2416. 2:
  2417. { Only modify the first parameter }
  2418. if ReplaceRegisterInOper(hp, 0, CurrentReg, ReplaceReg) then
  2419. begin
  2420. Result := True;
  2421. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovIMul2MovIMul 2)', hp);
  2422. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2423. end;
  2424. 3:
  2425. { Only modify the second parameter }
  2426. if ReplaceRegisterInOper(hp, 1, CurrentReg, ReplaceReg) then
  2427. begin
  2428. Result := True;
  2429. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovIMul2MovIMul 3)', hp);
  2430. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2431. end;
  2432. else
  2433. InternalError(2020012901);
  2434. end;
  2435. end;
  2436. else
  2437. if (hp.ops > 0) and
  2438. ReplaceRegisterInInstruction(hp, CurrentReg, ReplaceReg) then
  2439. begin
  2440. Result := True;
  2441. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovXXX2MovXXX)', hp);
  2442. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2443. end;
  2444. end;
  2445. end;
  2446. function TX86AsmOptimizer.FuncMov2Func(var p: tai; const hp1: tai): Boolean;
  2447. var
  2448. hp2: tai;
  2449. p_SourceReg, p_TargetReg: TRegister;
  2450. begin
  2451. Result := False;
  2452. { Backward optimisation. If we have:
  2453. func. %reg1,%reg2
  2454. mov %reg2,%reg3
  2455. (dealloc %reg2)
  2456. Change to:
  2457. func. %reg1,%reg3 (see comment below for what a valid func. is)
  2458. Perform similar optimisations with 1, 3 and 4-operand instructions
  2459. that only have one output.
  2460. }
  2461. if MatchOpType(taicpu(p), top_reg, top_reg) then
  2462. begin
  2463. p_SourceReg := taicpu(p).oper[0]^.reg;
  2464. p_TargetReg := taicpu(p).oper[1]^.reg;
  2465. TransferUsedRegs(TmpUsedRegs);
  2466. if not RegUsedAfterInstruction(p_SourceReg, p, TmpUsedRegs) and
  2467. GetLastInstruction(p, hp2) and
  2468. (hp2.typ = ait_instruction) and
  2469. { Have to make sure it's an instruction that only reads from
  2470. the first operands and only writes (not reads or modifies) to
  2471. the last one; in essence, a pure function such as BSR, POPCNT
  2472. or ANDN }
  2473. (
  2474. (
  2475. (taicpu(hp2).ops = 1) and
  2476. (insprop[taicpu(hp2).opcode].Ch * [Ch_Wop1] = [Ch_Wop1])
  2477. ) or
  2478. (
  2479. (taicpu(hp2).ops = 2) and
  2480. (insprop[taicpu(hp2).opcode].Ch * [Ch_Rop1, Ch_Wop2] = [Ch_Rop1, Ch_Wop2])
  2481. ) or
  2482. (
  2483. (taicpu(hp2).ops = 3) and
  2484. (insprop[taicpu(hp2).opcode].Ch * [Ch_Rop1, Ch_Rop2, Ch_Wop3] = [Ch_Rop1, Ch_Rop2, Ch_Wop3])
  2485. ) or
  2486. (
  2487. (taicpu(hp2).ops = 4) and
  2488. (insprop[taicpu(hp2).opcode].Ch * [Ch_Rop1, Ch_Rop2, Ch_Rop3, Ch_Wop4] = [Ch_Rop1, Ch_Rop2, Ch_Rop3, Ch_Wop4])
  2489. )
  2490. ) and
  2491. (taicpu(hp2).oper[taicpu(hp2).ops-1]^.typ = top_reg) and
  2492. (taicpu(hp2).oper[taicpu(hp2).ops-1]^.reg = p_SourceReg) then
  2493. begin
  2494. case taicpu(hp2).opcode of
  2495. A_FSTSW, A_FNSTSW,
  2496. A_IN, A_INS, A_OUT, A_OUTS,
  2497. A_CMPS, A_LODS, A_MOVS, A_SCAS, A_STOS:
  2498. { These routines have explicit operands, but they are restricted in
  2499. what they can be (e.g. IN and OUT can only read from AL, AX or
  2500. EAX. }
  2501. ;
  2502. else
  2503. begin
  2504. DebugMsg(SPeepholeOptimization + 'Removed MOV and changed destination on previous instruction to optimise register usage (FuncMov2Func)', p);
  2505. taicpu(hp2).oper[taicpu(hp2).ops-1]^.reg := p_TargetReg;
  2506. if not RegInInstruction(p_TargetReg, hp2) then
  2507. begin
  2508. { Since we're allocating from an earlier point, we
  2509. need to remove the register from the tracking }
  2510. ExcludeRegFromUsedRegs(p_TargetReg, TmpUsedRegs);
  2511. AllocRegBetween(p_TargetReg, hp2, p, TmpUsedRegs);
  2512. end;
  2513. RemoveCurrentp(p, hp1);
  2514. { If the Func was another MOV instruction, we might get
  2515. "mov %reg,%reg" that doesn't get removed in Pass 2
  2516. otherwise, so deal with it here (also do something
  2517. similar with lea (%reg),%reg}
  2518. if (taicpu(hp2).opcode = A_MOV) and MatchOperand(taicpu(hp2).oper[0]^, taicpu(hp2).oper[1]^.reg) then
  2519. begin
  2520. DebugMsg(SPeepholeOptimization + 'Mov2Nop 1a done', hp2);
  2521. if p = hp2 then
  2522. RemoveCurrentp(p)
  2523. else
  2524. RemoveInstruction(hp2);
  2525. end;
  2526. Result := True;
  2527. Exit;
  2528. end;
  2529. end;
  2530. end;
  2531. end;
  2532. end;
  2533. function TX86AsmOptimizer.OptPass1MOV(var p : tai) : boolean;
  2534. var
  2535. hp1, hp2, hp3: tai;
  2536. DoOptimisation, TempBool: Boolean;
  2537. {$ifdef x86_64}
  2538. NewConst: TCGInt;
  2539. {$endif x86_64}
  2540. procedure convert_mov_value(signed_movop: tasmop; max_value: tcgint); inline;
  2541. begin
  2542. if taicpu(hp1).opcode = signed_movop then
  2543. begin
  2544. if taicpu(p).oper[0]^.val > max_value shr 1 then
  2545. taicpu(p).oper[0]^.val:=taicpu(p).oper[0]^.val - max_value - 1 { Convert to signed }
  2546. end
  2547. else
  2548. taicpu(p).oper[0]^.val:=taicpu(p).oper[0]^.val and max_value; { Trim to unsigned }
  2549. end;
  2550. function TryConstMerge(var p1, p2: tai): Boolean;
  2551. var
  2552. ThisRef: TReference;
  2553. begin
  2554. Result := False;
  2555. ThisRef := taicpu(p2).oper[1]^.ref^;
  2556. { Only permit writes to the stack, since we can guarantee alignment with that }
  2557. if (ThisRef.index = NR_NO) and
  2558. (
  2559. (ThisRef.base = NR_STACK_POINTER_REG) or
  2560. (ThisRef.base = current_procinfo.framepointer)
  2561. ) then
  2562. begin
  2563. case taicpu(p).opsize of
  2564. S_B:
  2565. begin
  2566. { Word writes must be on a 2-byte boundary }
  2567. if (taicpu(p1).oper[1]^.ref^.offset mod 2) = 0 then
  2568. begin
  2569. { Reduce offset of second reference to see if it is sequential with the first }
  2570. Dec(ThisRef.offset, 1);
  2571. if RefsEqual(taicpu(p1).oper[1]^.ref^, ThisRef) then
  2572. begin
  2573. { Make sure the constants aren't represented as a
  2574. negative number, as these won't merge properly }
  2575. taicpu(p1).opsize := S_W;
  2576. taicpu(p1).oper[0]^.val := (taicpu(p1).oper[0]^.val and $FF) or ((taicpu(p2).oper[0]^.val and $FF) shl 8);
  2577. DebugMsg(SPeepholeOptimization + 'Merged two byte-sized constant writes to stack (MovMov2Mov 2a)', p1);
  2578. RemoveInstruction(p2);
  2579. Result := True;
  2580. end;
  2581. end;
  2582. end;
  2583. S_W:
  2584. begin
  2585. { Longword writes must be on a 4-byte boundary }
  2586. if (taicpu(p1).oper[1]^.ref^.offset mod 4) = 0 then
  2587. begin
  2588. { Reduce offset of second reference to see if it is sequential with the first }
  2589. Dec(ThisRef.offset, 2);
  2590. if RefsEqual(taicpu(p1).oper[1]^.ref^, ThisRef) then
  2591. begin
  2592. { Make sure the constants aren't represented as a
  2593. negative number, as these won't merge properly }
  2594. taicpu(p1).opsize := S_L;
  2595. taicpu(p1).oper[0]^.val := (taicpu(p1).oper[0]^.val and $FFFF) or ((taicpu(p2).oper[0]^.val and $FFFF) shl 16);
  2596. DebugMsg(SPeepholeOptimization + 'Merged two word-sized constant writes to stack (MovMov2Mov 2b)', p1);
  2597. RemoveInstruction(p2);
  2598. Result := True;
  2599. end;
  2600. end;
  2601. end;
  2602. {$ifdef x86_64}
  2603. S_L:
  2604. begin
  2605. { Only sign-extended 32-bit constants can be written to 64-bit memory directly, so check to
  2606. see if the constants can be encoded this way. }
  2607. NewConst := (taicpu(p1).oper[0]^.val and $FFFFFFFF) or (taicpu(p2).oper[0]^.val shl 32);
  2608. if (NewConst >= -2147483648) and (NewConst <= 2147483647) and
  2609. { Quadword writes must be on an 8-byte boundary }
  2610. ((taicpu(p1).oper[1]^.ref^.offset mod 8) = 0) then
  2611. begin
  2612. { Reduce offset of second reference to see if it is sequential with the first }
  2613. Dec(ThisRef.offset, 4);
  2614. if RefsEqual(taicpu(p1).oper[1]^.ref^, ThisRef) then
  2615. begin
  2616. { Make sure the constants aren't represented as a
  2617. negative number, as these won't merge properly }
  2618. taicpu(p1).opsize := S_Q;
  2619. { Force a typecast into a 32-bit signed integer (that will then be sign-extended to 64-bit) }
  2620. taicpu(p1).oper[0]^.val := NewConst;
  2621. DebugMsg(SPeepholeOptimization + 'Merged two longword-sized constant writes to stack (MovMov2Mov 2c)', p1);
  2622. RemoveInstruction(p2);
  2623. Result := True;
  2624. end;
  2625. end;
  2626. end;
  2627. {$endif x86_64}
  2628. else
  2629. ;
  2630. end;
  2631. end;
  2632. end;
  2633. var
  2634. GetNextInstruction_p, TempRegUsed, CrossJump: Boolean;
  2635. PreMessage, RegName1, RegName2, InputVal, MaskNum: string;
  2636. NewSize: topsize; NewOffset: asizeint;
  2637. p_SourceReg, p_TargetReg, NewMMReg: TRegister;
  2638. SourceRef, TargetRef: TReference;
  2639. MovAligned, MovUnaligned: TAsmOp;
  2640. ThisRef: TReference;
  2641. JumpTracking: TLinkedList;
  2642. begin
  2643. Result:=false;
  2644. GetNextInstruction_p:=GetNextInstruction(p, hp1);
  2645. { remove mov reg1,reg1? }
  2646. if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^)
  2647. then
  2648. begin
  2649. DebugMsg(SPeepholeOptimization + 'Mov2Nop 1 done',p);
  2650. { take care of the register (de)allocs following p }
  2651. RemoveCurrentP(p, hp1);
  2652. Result:=true;
  2653. exit;
  2654. end;
  2655. { All the next optimisations require a next instruction }
  2656. if not GetNextInstruction_p or (hp1.typ <> ait_instruction) then
  2657. Exit;
  2658. { Prevent compiler warnings }
  2659. p_TargetReg := NR_NO;
  2660. if taicpu(p).oper[1]^.typ = top_reg then
  2661. begin
  2662. { Saves on a large number of dereferences }
  2663. p_TargetReg := taicpu(p).oper[1]^.reg;
  2664. { Look for:
  2665. mov %reg1,%reg2
  2666. ??? %reg2,r/m
  2667. Change to:
  2668. mov %reg1,%reg2
  2669. ??? %reg1,r/m
  2670. }
  2671. if taicpu(p).oper[0]^.typ = top_reg then
  2672. begin
  2673. if RegReadByInstruction(p_TargetReg, hp1) and
  2674. DeepMOVOpt(taicpu(p), taicpu(hp1)) then
  2675. begin
  2676. { A change has occurred, just not in p }
  2677. Result := True;
  2678. TransferUsedRegs(TmpUsedRegs);
  2679. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  2680. if not RegUsedAfterInstruction(p_TargetReg, hp1, TmpUsedRegs) and
  2681. { Just in case something didn't get modified (e.g. an
  2682. implicit register) }
  2683. not RegReadByInstruction(p_TargetReg, hp1) then
  2684. begin
  2685. { We can remove the original MOV }
  2686. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3 done',p);
  2687. RemoveCurrentp(p, hp1);
  2688. { UsedRegs got updated by RemoveCurrentp }
  2689. Result := True;
  2690. Exit;
  2691. end;
  2692. { If we know a MOV instruction has become a null operation, we might as well
  2693. get rid of it now to save time. }
  2694. if (taicpu(hp1).opcode = A_MOV) and
  2695. (taicpu(hp1).oper[1]^.typ = top_reg) and
  2696. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[0]^.reg) and
  2697. { Just being a register is enough to confirm it's a null operation }
  2698. (taicpu(hp1).oper[0]^.typ = top_reg) then
  2699. begin
  2700. Result := True;
  2701. { Speed-up to reduce a pipeline stall... if we had something like...
  2702. movl %eax,%edx
  2703. movw %dx,%ax
  2704. ... the second instruction would change to movw %ax,%ax, but
  2705. given that it is now %ax that's active rather than %eax,
  2706. penalties might occur due to a partial register write, so instead,
  2707. change it to a MOVZX instruction when optimising for speed.
  2708. }
  2709. if not (cs_opt_size in current_settings.optimizerswitches) and
  2710. IsMOVZXAcceptable and
  2711. (taicpu(hp1).opsize < taicpu(p).opsize)
  2712. {$ifdef x86_64}
  2713. { operations already implicitly set the upper 64 bits to zero }
  2714. and not ((taicpu(hp1).opsize = S_L) and (taicpu(p).opsize = S_Q))
  2715. {$endif x86_64}
  2716. then
  2717. begin
  2718. DebugMsg(SPeepholeOptimization + 'Zero-extension to minimise pipeline stall (Mov2Movz)',hp1);
  2719. case taicpu(p).opsize of
  2720. S_W:
  2721. if taicpu(hp1).opsize = S_B then
  2722. taicpu(hp1).opsize := S_BL
  2723. else
  2724. InternalError(2020012911);
  2725. S_L{$ifdef x86_64}, S_Q{$endif x86_64}:
  2726. case taicpu(hp1).opsize of
  2727. S_B:
  2728. taicpu(hp1).opsize := S_BL;
  2729. S_W:
  2730. taicpu(hp1).opsize := S_WL;
  2731. else
  2732. InternalError(2020012912);
  2733. end;
  2734. else
  2735. InternalError(2020012910);
  2736. end;
  2737. taicpu(hp1).opcode := A_MOVZX;
  2738. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  2739. end
  2740. else
  2741. begin
  2742. GetNextInstruction_p := GetNextInstruction(hp1, hp2);
  2743. DebugMsg(SPeepholeOptimization + 'Mov2Nop 4 done',hp1);
  2744. RemoveInstruction(hp1);
  2745. { The instruction after what was hp1 is now the immediate next instruction,
  2746. so we can continue to make optimisations if it's present }
  2747. if not GetNextInstruction_p or (hp2.typ <> ait_instruction) then
  2748. Exit;
  2749. hp1 := hp2;
  2750. end;
  2751. end;
  2752. end;
  2753. end;
  2754. end;
  2755. { Depending on the DeepMOVOpt above, it may turn out that hp1 completely
  2756. overwrites the original destination register. e.g.
  2757. movl ###,%reg2d
  2758. movslq ###,%reg2q (### doesn't have to be the same as the first one)
  2759. In this case, we can remove the MOV (Go to "Mov2Nop 5" below)
  2760. }
  2761. if (taicpu(p).oper[1]^.typ = top_reg) and
  2762. MatchInstruction(hp1, [A_LEA, A_MOV, A_MOVSX, A_MOVZX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}], []) and
  2763. (taicpu(hp1).oper[1]^.typ = top_reg) and
  2764. Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[1]^.reg) then
  2765. begin
  2766. if RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^) then
  2767. begin
  2768. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  2769. case taicpu(p).oper[0]^.typ of
  2770. top_const:
  2771. { We have something like:
  2772. movb $x, %regb
  2773. movzbl %regb,%regd
  2774. Change to:
  2775. movl $x, %regd
  2776. }
  2777. begin
  2778. case taicpu(hp1).opsize of
  2779. S_BW:
  2780. begin
  2781. convert_mov_value(A_MOVSX, $FF);
  2782. setsubreg(taicpu(p).oper[1]^.reg, R_SUBW);
  2783. taicpu(p).opsize := S_W;
  2784. end;
  2785. S_BL:
  2786. begin
  2787. convert_mov_value(A_MOVSX, $FF);
  2788. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  2789. taicpu(p).opsize := S_L;
  2790. end;
  2791. S_WL:
  2792. begin
  2793. convert_mov_value(A_MOVSX, $FFFF);
  2794. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  2795. taicpu(p).opsize := S_L;
  2796. end;
  2797. {$ifdef x86_64}
  2798. S_BQ:
  2799. begin
  2800. convert_mov_value(A_MOVSX, $FF);
  2801. setsubreg(taicpu(p).oper[1]^.reg, R_SUBQ);
  2802. taicpu(p).opsize := S_Q;
  2803. end;
  2804. S_WQ:
  2805. begin
  2806. convert_mov_value(A_MOVSX, $FFFF);
  2807. setsubreg(taicpu(p).oper[1]^.reg, R_SUBQ);
  2808. taicpu(p).opsize := S_Q;
  2809. end;
  2810. S_LQ:
  2811. begin
  2812. convert_mov_value(A_MOVSXD, $FFFFFFFF); { Note it's MOVSXD, not MOVSX }
  2813. setsubreg(taicpu(p).oper[1]^.reg, R_SUBQ);
  2814. taicpu(p).opsize := S_Q;
  2815. end;
  2816. {$endif x86_64}
  2817. else
  2818. { If hp1 was a MOV instruction, it should have been
  2819. optimised already }
  2820. InternalError(2020021001);
  2821. end;
  2822. DebugMsg(SPeepholeOptimization + 'MovMovXX2MovXX 2 done',p);
  2823. RemoveInstruction(hp1);
  2824. Result := True;
  2825. Exit;
  2826. end;
  2827. top_ref:
  2828. begin
  2829. { We have something like:
  2830. movb mem, %regb
  2831. movzbl %regb,%regd
  2832. Change to:
  2833. movzbl mem, %regd
  2834. }
  2835. ThisRef := taicpu(p).oper[0]^.ref^;
  2836. if (ThisRef.refaddr<>addr_full) and (IsMOVZXAcceptable or (taicpu(hp1).opcode<>A_MOVZX)) then
  2837. begin
  2838. DebugMsg(SPeepholeOptimization + 'MovMovXX2MovXX 1 done',p);
  2839. taicpu(hp1).loadref(0, ThisRef);
  2840. { Make sure any registers in the references are properly tracked }
  2841. if (ThisRef.base <> NR_NO){$ifdef x86_64} and (ThisRef.base <> NR_RIP){$endif x86_64} then
  2842. AllocRegBetween(ThisRef.base, p, hp1, UsedRegs);
  2843. if (ThisRef.index <> NR_NO) then
  2844. AllocRegBetween(ThisRef.index, p, hp1, UsedRegs);
  2845. RemoveCurrentP(p, hp1);
  2846. Result := True;
  2847. Exit;
  2848. end;
  2849. end;
  2850. else
  2851. if (taicpu(hp1).opcode <> A_MOV) and (taicpu(hp1).opcode <> A_LEA) then
  2852. { Just to make a saving, since there are no more optimisations with MOVZX and MOVSX/D }
  2853. Exit;
  2854. end;
  2855. end
  2856. { The RegInOp check makes sure that movl r/m,%reg1l; movzbl (%reg1l),%reg1l"
  2857. and "movl r/m,%reg1; leal $1(%reg1,%reg2),%reg1" etc. are not incorrectly
  2858. optimised }
  2859. else
  2860. begin
  2861. DebugMsg(SPeepholeOptimization + 'Mov2Nop 5 done',p);
  2862. RemoveCurrentP(p, hp1);
  2863. Result := True;
  2864. Exit;
  2865. end;
  2866. end;
  2867. if (taicpu(hp1).opcode = A_AND) and
  2868. (taicpu(p).oper[1]^.typ = top_reg) and
  2869. MatchOpType(taicpu(hp1),top_const,top_reg) then
  2870. begin
  2871. if MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[1]^) then
  2872. begin
  2873. case taicpu(p).opsize of
  2874. S_L:
  2875. if (taicpu(hp1).oper[0]^.val = $ffffffff) then
  2876. begin
  2877. { Optimize out:
  2878. mov x, %reg
  2879. and ffffffffh, %reg
  2880. }
  2881. DebugMsg(SPeepholeOptimization + 'MovAnd2Mov 1 done',p);
  2882. RemoveInstruction(hp1);
  2883. Result:=true;
  2884. exit;
  2885. end;
  2886. S_Q: { TODO: Confirm if this is even possible }
  2887. if (taicpu(hp1).oper[0]^.val = $ffffffffffffffff) then
  2888. begin
  2889. { Optimize out:
  2890. mov x, %reg
  2891. and ffffffffffffffffh, %reg
  2892. }
  2893. DebugMsg(SPeepholeOptimization + 'MovAnd2Mov 2 done',p);
  2894. RemoveInstruction(hp1);
  2895. Result:=true;
  2896. exit;
  2897. end;
  2898. else
  2899. ;
  2900. end;
  2901. if (
  2902. (taicpu(p).oper[0]^.typ=top_reg) or
  2903. (
  2904. (taicpu(p).oper[0]^.typ=top_ref) and
  2905. (taicpu(p).oper[0]^.ref^.refaddr<>addr_full)
  2906. )
  2907. ) and
  2908. GetNextInstruction(hp1,hp2) and
  2909. MatchInstruction(hp2,A_TEST,[]) and
  2910. (
  2911. MatchOperand(taicpu(hp1).oper[1]^,taicpu(hp2).oper[1]^) or
  2912. (
  2913. { If the register being tested is smaller than the one
  2914. that received a bitwise AND, permit it if the constant
  2915. fits into the smaller size }
  2916. (taicpu(hp1).oper[1]^.typ = top_reg) and (taicpu(hp2).oper[1]^.typ = top_reg) and
  2917. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg,taicpu(hp2).oper[1]^.reg) and
  2918. (taicpu(hp1).oper[0]^.typ = top_const) and (taicpu(hp1).oper[0]^.val >= 0) and
  2919. (GetSubReg(taicpu(hp2).oper[1]^.reg) < GetSubReg(taicpu(hp1).oper[1]^.reg)) and
  2920. (
  2921. (
  2922. (GetSubReg(taicpu(hp2).oper[1]^.reg) = R_SUBL) and
  2923. (taicpu(hp1).oper[0]^.val <= $FF)
  2924. ) or
  2925. (
  2926. (GetSubReg(taicpu(hp2).oper[1]^.reg) = R_SUBW) and
  2927. (taicpu(hp1).oper[0]^.val <= $FFFF)
  2928. {$ifdef x86_64}
  2929. ) or
  2930. (
  2931. (GetSubReg(taicpu(hp2).oper[1]^.reg) = R_SUBD) and
  2932. (taicpu(hp1).oper[0]^.val <= $FFFFFFFF)
  2933. {$endif x86_64}
  2934. )
  2935. )
  2936. )
  2937. ) and
  2938. (
  2939. MatchOperand(taicpu(hp2).oper[0]^,taicpu(hp2).oper[1]^) or
  2940. MatchOperand(taicpu(hp2).oper[0]^,-1)
  2941. ) and
  2942. GetNextInstruction(hp2,hp3) and
  2943. MatchInstruction(hp3,A_Jcc,A_Setcc,[]) and
  2944. (taicpu(hp3).condition in [C_E,C_NE]) then
  2945. begin
  2946. TransferUsedRegs(TmpUsedRegs);
  2947. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  2948. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  2949. if not(RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp2, TmpUsedRegs)) then
  2950. begin
  2951. DebugMsg(SPeepholeOptimization + 'MovAndTest2Test done',p);
  2952. taicpu(hp1).loadoper(1,taicpu(p).oper[0]^);
  2953. taicpu(hp1).opcode:=A_TEST;
  2954. { Shrink the TEST instruction down to the smallest possible size }
  2955. case taicpu(hp1).oper[0]^.val of
  2956. 0..255:
  2957. if (taicpu(hp1).opsize <> S_B)
  2958. {$ifndef x86_64}
  2959. and (
  2960. (taicpu(hp1).oper[1]^.typ <> top_reg) or
  2961. { Cannot encode byte-sized ESI, EDI, EBP or ESP under i386 }
  2962. (GetSupReg(taicpu(hp1).oper[1]^.reg) in [RS_EAX, RS_EBX, RS_ECX, RS_EDX])
  2963. )
  2964. {$endif x86_64}
  2965. then
  2966. begin
  2967. if taicpu(hp1).opsize <> taicpu(hp2).opsize then
  2968. { Only print debug message if the TEST instruction
  2969. is a different size before and after }
  2970. DebugMsg(SPeepholeOptimization + 'test' + debug_opsize2str(taicpu(hp1).opsize) + ' -> testb to reduce instruction size (Test2Test 1a)' , p);
  2971. taicpu(hp1).opsize := S_B;
  2972. if (taicpu(hp1).oper[1]^.typ = top_reg) then
  2973. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBL);
  2974. end;
  2975. 256..65535:
  2976. if (taicpu(hp1).opsize <> S_W) then
  2977. begin
  2978. if taicpu(hp1).opsize <> taicpu(hp2).opsize then
  2979. { Only print debug message if the TEST instruction
  2980. is a different size before and after }
  2981. DebugMsg(SPeepholeOptimization + 'test' + debug_opsize2str(taicpu(hp1).opsize) + ' -> testw to reduce instruction size (Test2Test 1b)' , p);
  2982. taicpu(hp1).opsize := S_W;
  2983. if (taicpu(hp1).oper[1]^.typ = top_reg) then
  2984. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBW);
  2985. end;
  2986. {$ifdef x86_64}
  2987. 65536..$7FFFFFFF:
  2988. if (taicpu(hp1).opsize <> S_L) then
  2989. begin
  2990. if taicpu(hp1).opsize <> taicpu(hp2).opsize then
  2991. { Only print debug message if the TEST instruction
  2992. is a different size before and after }
  2993. DebugMsg(SPeepholeOptimization + 'test' + debug_opsize2str(taicpu(hp1).opsize) + ' -> testl to reduce instruction size (Test2Test 1c)' , p);
  2994. taicpu(hp1).opsize := S_L;
  2995. if (taicpu(hp1).oper[1]^.typ = top_reg) then
  2996. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  2997. end;
  2998. {$endif x86_64}
  2999. else
  3000. ;
  3001. end;
  3002. RemoveInstruction(hp2);
  3003. RemoveCurrentP(p, hp1);
  3004. Result:=true;
  3005. exit;
  3006. end;
  3007. end;
  3008. end
  3009. else if IsMOVZXAcceptable and
  3010. (taicpu(p).oper[1]^.typ = top_reg) and (taicpu(hp1).oper[1]^.typ = top_reg) and
  3011. (taicpu(p).oper[0]^.typ <> top_const) and { MOVZX only supports registers and memory, not immediates (use MOV for that!) }
  3012. (getsupreg(taicpu(p).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg))
  3013. then
  3014. begin
  3015. InputVal := debug_operstr(taicpu(p).oper[0]^);
  3016. MaskNum := debug_tostr(taicpu(hp1).oper[0]^.val);
  3017. case taicpu(p).opsize of
  3018. S_B:
  3019. if (taicpu(hp1).oper[0]^.val = $ff) then
  3020. begin
  3021. { Convert:
  3022. movb x, %regl movb x, %regl
  3023. andw ffh, %regw andl ffh, %regd
  3024. To:
  3025. movzbw x, %regd movzbl x, %regd
  3026. (Identical registers, just different sizes)
  3027. }
  3028. RegName1 := debug_regname(taicpu(p).oper[1]^.reg); { 8-bit register name }
  3029. RegName2 := debug_regname(taicpu(hp1).oper[1]^.reg); { 16/32-bit register name }
  3030. case taicpu(hp1).opsize of
  3031. S_W: NewSize := S_BW;
  3032. S_L: NewSize := S_BL;
  3033. {$ifdef x86_64}
  3034. S_Q: NewSize := S_BQ;
  3035. {$endif x86_64}
  3036. else
  3037. InternalError(2018011510);
  3038. end;
  3039. end
  3040. else
  3041. NewSize := S_NO;
  3042. S_W:
  3043. if (taicpu(hp1).oper[0]^.val = $ffff) then
  3044. begin
  3045. { Convert:
  3046. movw x, %regw
  3047. andl ffffh, %regd
  3048. To:
  3049. movzwl x, %regd
  3050. (Identical registers, just different sizes)
  3051. }
  3052. RegName1 := debug_regname(taicpu(p).oper[1]^.reg); { 16-bit register name }
  3053. RegName2 := debug_regname(taicpu(hp1).oper[1]^.reg); { 32-bit register name }
  3054. case taicpu(hp1).opsize of
  3055. S_L: NewSize := S_WL;
  3056. {$ifdef x86_64}
  3057. S_Q: NewSize := S_WQ;
  3058. {$endif x86_64}
  3059. else
  3060. InternalError(2018011511);
  3061. end;
  3062. end
  3063. else
  3064. NewSize := S_NO;
  3065. else
  3066. NewSize := S_NO;
  3067. end;
  3068. if NewSize <> S_NO then
  3069. begin
  3070. PreMessage := 'mov' + debug_opsize2str(taicpu(p).opsize) + ' ' + InputVal + ',' + RegName1;
  3071. { The actual optimization }
  3072. taicpu(p).opcode := A_MOVZX;
  3073. taicpu(p).changeopsize(NewSize);
  3074. taicpu(p).oper[1]^ := taicpu(hp1).oper[1]^;
  3075. { Safeguard if "and" is followed by a conditional command }
  3076. TransferUsedRegs(TmpUsedRegs);
  3077. UpdateUsedRegs(TmpUsedRegs,tai(p.next));
  3078. if (RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)) then
  3079. begin
  3080. { At this point, the "and" command is effectively equivalent to
  3081. "test %reg,%reg". This will be handled separately by the
  3082. Peephole Optimizer. [Kit] }
  3083. DebugMsg(SPeepholeOptimization + PreMessage +
  3084. ' -> movz' + debug_opsize2str(NewSize) + ' ' + InputVal + ',' + RegName2, p);
  3085. end
  3086. else
  3087. begin
  3088. DebugMsg(SPeepholeOptimization + PreMessage + '; and' + debug_opsize2str(taicpu(hp1).opsize) + ' $' + MaskNum + ',' + RegName2 +
  3089. ' -> movz' + debug_opsize2str(NewSize) + ' ' + InputVal + ',' + RegName2, p);
  3090. RemoveInstruction(hp1);
  3091. end;
  3092. Result := True;
  3093. Exit;
  3094. end;
  3095. end;
  3096. end;
  3097. if (taicpu(hp1).opcode = A_OR) and
  3098. (taicpu(p).oper[1]^.typ = top_reg) and
  3099. MatchOperand(taicpu(p).oper[0]^, 0) and
  3100. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) then
  3101. begin
  3102. { mov 0, %reg
  3103. or ###,%reg
  3104. Change to (only if the flags are not used):
  3105. mov ###,%reg
  3106. }
  3107. TransferUsedRegs(TmpUsedRegs);
  3108. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  3109. DoOptimisation := True;
  3110. { Even if the flags are used, we might be able to do the optimisation
  3111. if the conditions are predictable }
  3112. if RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  3113. begin
  3114. { Only perform if ### = %reg (the same register) or equal to 0,
  3115. so %reg is guaranteed to still have a value of zero }
  3116. if MatchOperand(taicpu(hp1).oper[0]^, 0) or
  3117. MatchOperand(taicpu(hp1).oper[0]^, taicpu(hp1).oper[1]^.reg) then
  3118. begin
  3119. hp2 := hp1;
  3120. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  3121. while RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  3122. GetNextInstruction(hp2, hp3) do
  3123. begin
  3124. { Don't continue modifying if the flags state is getting changed }
  3125. if RegModifiedByInstruction(NR_DEFAULTFLAGS, hp3) then
  3126. Break;
  3127. UpdateUsedRegs(TmpUsedRegs, tai(hp3.Next));
  3128. if MatchInstruction(hp3, A_Jcc, A_SETcc, A_CMOVcc, []) then
  3129. begin
  3130. if condition_in(C_E, taicpu(hp3).condition) or (taicpu(hp3).condition in [C_NC, C_NS, C_NO]) then
  3131. begin
  3132. { Condition is always true }
  3133. case taicpu(hp3).opcode of
  3134. A_Jcc:
  3135. begin
  3136. DebugMsg(SPeepholeOptimization + 'Condition is always true (jump made unconditional)', hp3);
  3137. { Check for jump shortcuts before we destroy the condition }
  3138. DoJumpOptimizations(hp3, TempBool);
  3139. MakeUnconditional(taicpu(hp3));
  3140. Result := True;
  3141. end;
  3142. A_CMOVcc:
  3143. begin
  3144. DebugMsg(SPeepholeOptimization + 'Condition is always true (CMOVcc -> MOV)', hp3);
  3145. taicpu(hp3).opcode := A_MOV;
  3146. taicpu(hp3).condition := C_None;
  3147. Result := True;
  3148. end;
  3149. A_SETcc:
  3150. begin
  3151. DebugMsg(SPeepholeOptimization + 'Condition is always true (changed to MOV 1)', hp3);
  3152. { Convert "set(c) %reg" instruction to "movb 1,%reg" }
  3153. taicpu(hp3).opcode := A_MOV;
  3154. taicpu(hp3).ops := 2;
  3155. taicpu(hp3).condition := C_None;
  3156. taicpu(hp3).opsize := S_B;
  3157. taicpu(hp3).loadreg(1,taicpu(hp3).oper[0]^.reg);
  3158. taicpu(hp3).loadconst(0, 1);
  3159. Result := True;
  3160. end;
  3161. else
  3162. InternalError(2021090701);
  3163. end;
  3164. end
  3165. else if (taicpu(hp3).condition in [C_A, C_B, C_C, C_G, C_L, C_NE, C_NZ, C_O, C_S]) then
  3166. begin
  3167. { Condition is always false }
  3168. case taicpu(hp3).opcode of
  3169. A_Jcc:
  3170. begin
  3171. DebugMsg(SPeepholeOptimization + 'Condition is always false (jump removed)', hp3);
  3172. TAsmLabel(taicpu(hp3).oper[0]^.ref^.symbol).decrefs;
  3173. RemoveInstruction(hp3);
  3174. Result := True;
  3175. { Since hp3 was deleted, hp2 must not be updated }
  3176. Continue;
  3177. end;
  3178. A_CMOVcc:
  3179. begin
  3180. DebugMsg(SPeepholeOptimization + 'Condition is always false (conditional load removed)', hp3);
  3181. RemoveInstruction(hp3);
  3182. Result := True;
  3183. { Since hp3 was deleted, hp2 must not be updated }
  3184. Continue;
  3185. end;
  3186. A_SETcc:
  3187. begin
  3188. DebugMsg(SPeepholeOptimization + 'Condition is always false (changed to MOV 0)', hp3);
  3189. { Convert "set(c) %reg" instruction to "movb 0,%reg" }
  3190. taicpu(hp3).opcode := A_MOV;
  3191. taicpu(hp3).ops := 2;
  3192. taicpu(hp3).condition := C_None;
  3193. taicpu(hp3).opsize := S_B;
  3194. taicpu(hp3).loadreg(1,taicpu(hp3).oper[0]^.reg);
  3195. taicpu(hp3).loadconst(0, 0);
  3196. Result := True;
  3197. end;
  3198. else
  3199. InternalError(2021090702);
  3200. end;
  3201. end
  3202. else
  3203. { Uncertain what to do - don't optimise (although optimise other conditional statements if present) }
  3204. DoOptimisation := False;
  3205. end;
  3206. hp2 := hp3;
  3207. end;
  3208. { Flags are still in use - don't optimise }
  3209. if DoOptimisation and RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  3210. DoOptimisation := False;
  3211. end
  3212. else
  3213. DoOptimisation := False;
  3214. end;
  3215. if DoOptimisation then
  3216. begin
  3217. {$ifdef x86_64}
  3218. { OR only supports 32-bit sign-extended constants for 64-bit
  3219. instructions, so compensate for this if the constant is
  3220. encoded as a value greater than or equal to 2^31 }
  3221. if (taicpu(hp1).opsize = S_Q) and
  3222. (taicpu(hp1).oper[0]^.typ = top_const) and
  3223. (taicpu(hp1).oper[0]^.val >= $80000000) then
  3224. taicpu(hp1).oper[0]^.val := taicpu(hp1).oper[0]^.val or $FFFFFFFF00000000;
  3225. {$endif x86_64}
  3226. DebugMsg(SPeepholeOptimization + 'MOV 0 / OR -> MOV', p);
  3227. taicpu(hp1).opcode := A_MOV;
  3228. RemoveCurrentP(p, hp1);
  3229. Result := True;
  3230. Exit;
  3231. end;
  3232. end;
  3233. { Next instruction is also a MOV ? }
  3234. if MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) then
  3235. begin
  3236. if MatchOpType(taicpu(p), top_const, top_ref) and
  3237. MatchOpType(taicpu(hp1), top_const, top_ref) and
  3238. TryConstMerge(p, hp1) then
  3239. begin
  3240. Result := True;
  3241. { In case we have four byte writes in a row, check for 2 more
  3242. right now so we don't have to wait for another iteration of
  3243. pass 1
  3244. }
  3245. { If two byte-writes were merged, the opsize is now S_W, not S_B }
  3246. case taicpu(p).opsize of
  3247. S_W:
  3248. begin
  3249. if GetNextInstruction(p, hp1) and
  3250. MatchInstruction(hp1, A_MOV, [S_B]) and
  3251. MatchOpType(taicpu(hp1), top_const, top_ref) and
  3252. GetNextInstruction(hp1, hp2) and
  3253. MatchInstruction(hp2, A_MOV, [S_B]) and
  3254. MatchOpType(taicpu(hp2), top_const, top_ref) and
  3255. { Try to merge the two bytes }
  3256. TryConstMerge(hp1, hp2) then
  3257. { Now try to merge the two words (hp2 will get deleted) }
  3258. TryConstMerge(p, hp1);
  3259. end;
  3260. S_L:
  3261. begin
  3262. { Though this only really benefits x86_64 and not i386, it
  3263. gets a potential optimisation done faster and hence
  3264. reduces the number of times OptPass1MOV is entered }
  3265. if GetNextInstruction(p, hp1) and
  3266. MatchInstruction(hp1, A_MOV, [S_W]) and
  3267. MatchOpType(taicpu(hp1), top_const, top_ref) and
  3268. GetNextInstruction(hp1, hp2) and
  3269. MatchInstruction(hp2, A_MOV, [S_W]) and
  3270. MatchOpType(taicpu(hp2), top_const, top_ref) and
  3271. { Try to merge the two words }
  3272. TryConstMerge(hp1, hp2) then
  3273. { This will always fail on i386, so don't bother
  3274. calling it unless we're doing x86_64 }
  3275. {$ifdef x86_64}
  3276. { Now try to merge the two longwords (hp2 will get deleted) }
  3277. TryConstMerge(p, hp1)
  3278. {$endif x86_64}
  3279. ;
  3280. end;
  3281. else
  3282. ;
  3283. end;
  3284. Exit;
  3285. end;
  3286. if (taicpu(p).oper[1]^.typ = top_reg) and
  3287. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) then
  3288. begin
  3289. { Remember that p_TargetReg contains taicpu(p).oper[1]^.reg }
  3290. TransferUsedRegs(TmpUsedRegs);
  3291. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  3292. { we have
  3293. mov x, %treg
  3294. mov %treg, y
  3295. }
  3296. if not(RegInOp(p_TargetReg, taicpu(hp1).oper[1]^)) then
  3297. if not(RegUsedAfterInstruction(p_TargetReg, hp1, TmpUsedRegs)) then
  3298. { we've got
  3299. mov x, %treg
  3300. mov %treg, y
  3301. with %treg is not used after }
  3302. case taicpu(p).oper[0]^.typ Of
  3303. { top_reg is covered by DeepMOVOpt }
  3304. top_const:
  3305. begin
  3306. { change
  3307. mov const, %treg
  3308. mov %treg, y
  3309. to
  3310. mov const, y
  3311. }
  3312. if (taicpu(hp1).oper[1]^.typ=top_reg) or
  3313. ((taicpu(p).oper[0]^.val>=low(longint)) and (taicpu(p).oper[0]^.val<=high(longint))) then
  3314. begin
  3315. if taicpu(hp1).oper[1]^.typ=top_reg then
  3316. AllocRegBetween(taicpu(hp1).oper[1]^.reg,p,hp1,usedregs);
  3317. taicpu(p).loadOper(1,taicpu(hp1).oper[1]^);
  3318. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 5 done',p);
  3319. RemoveInstruction(hp1);
  3320. Result:=true;
  3321. Exit;
  3322. end;
  3323. end;
  3324. top_ref:
  3325. case taicpu(hp1).oper[1]^.typ of
  3326. top_reg:
  3327. begin
  3328. { change
  3329. mov mem, %treg
  3330. mov %treg, %reg
  3331. to
  3332. mov mem, %reg"
  3333. }
  3334. AllocRegBetween(taicpu(hp1).oper[1]^.reg,p,hp1,usedregs);
  3335. taicpu(p).loadreg(1, taicpu(hp1).oper[1]^.reg);
  3336. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 3 done',p);
  3337. RemoveInstruction(hp1);
  3338. Result:=true;
  3339. Exit;
  3340. end;
  3341. top_ref:
  3342. begin
  3343. {$ifdef x86_64}
  3344. { Look for the following to simplify:
  3345. mov x(mem1), %reg
  3346. mov %reg, y(mem2)
  3347. mov x+8(mem1), %reg
  3348. mov %reg, y+8(mem2)
  3349. Change to:
  3350. movdqu x(mem1), %xmmreg
  3351. movdqu %xmmreg, y(mem2)
  3352. ...but only as long as the memory blocks don't overlap
  3353. }
  3354. SourceRef := taicpu(p).oper[0]^.ref^;
  3355. TargetRef := taicpu(hp1).oper[1]^.ref^;
  3356. if (taicpu(p).opsize = S_Q) and
  3357. GetNextInstruction(hp1, hp2) and
  3358. MatchInstruction(hp2, A_MOV, [taicpu(p).opsize]) and
  3359. MatchOpType(taicpu(hp2), top_ref, top_reg) then
  3360. begin
  3361. { Delay calling GetNextInstruction(hp2, hp3) for as long as possible }
  3362. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  3363. Inc(SourceRef.offset, 8);
  3364. if UseAVX then
  3365. begin
  3366. MovAligned := A_VMOVDQA;
  3367. MovUnaligned := A_VMOVDQU;
  3368. end
  3369. else
  3370. begin
  3371. MovAligned := A_MOVDQA;
  3372. MovUnaligned := A_MOVDQU;
  3373. end;
  3374. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) and
  3375. not RefsMightOverlap(taicpu(p).oper[0]^.ref^, TargetRef, 16) then
  3376. begin
  3377. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  3378. Inc(TargetRef.offset, 8);
  3379. if GetNextInstruction(hp2, hp3) and
  3380. MatchInstruction(hp3, A_MOV, [taicpu(p).opsize]) and
  3381. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  3382. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  3383. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  3384. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  3385. begin
  3386. NewMMReg := GetMMRegisterBetween(R_SUBMMX, UsedRegs, p, hp3);
  3387. if NewMMReg <> NR_NO then
  3388. begin
  3389. { Remember that the offsets are 8 ahead }
  3390. if ((SourceRef.offset mod 16) = 8) and
  3391. (
  3392. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3393. (SourceRef.base = current_procinfo.framepointer) or
  3394. ((SourceRef.alignment >= 16) and ((SourceRef.alignment mod 16) = 0))
  3395. ) then
  3396. taicpu(p).opcode := MovAligned
  3397. else
  3398. taicpu(p).opcode := MovUnaligned;
  3399. taicpu(p).opsize := S_XMM;
  3400. taicpu(p).oper[1]^.reg := NewMMReg;
  3401. if ((TargetRef.offset mod 16) = 8) and
  3402. (
  3403. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3404. (TargetRef.base = current_procinfo.framepointer) or
  3405. ((TargetRef.alignment >= 16) and ((TargetRef.alignment mod 16) = 0))
  3406. ) then
  3407. taicpu(hp1).opcode := MovAligned
  3408. else
  3409. taicpu(hp1).opcode := MovUnaligned;
  3410. taicpu(hp1).opsize := S_XMM;
  3411. taicpu(hp1).oper[0]^.reg := NewMMReg;
  3412. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(NewMMReg) + ' to merge a pair of memory moves (MovMovMovMov2MovdqMovdq 1)', p);
  3413. RemoveInstruction(hp2);
  3414. RemoveInstruction(hp3);
  3415. Result := True;
  3416. Exit;
  3417. end;
  3418. end;
  3419. end
  3420. else
  3421. begin
  3422. { See if the next references are 8 less rather than 8 greater }
  3423. Dec(SourceRef.offset, 16); { -8 the other way }
  3424. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) then
  3425. begin
  3426. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  3427. Dec(TargetRef.offset, 8); { Only 8, not 16, as it wasn't incremented unlike SourceRef }
  3428. if not RefsMightOverlap(SourceRef, TargetRef, 16) and
  3429. GetNextInstruction(hp2, hp3) and
  3430. MatchInstruction(hp3, A_MOV, [taicpu(p).opsize]) and
  3431. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  3432. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  3433. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  3434. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  3435. begin
  3436. NewMMReg := GetMMRegisterBetween(R_SUBMMX, UsedRegs, p, hp3);
  3437. if NewMMReg <> NR_NO then
  3438. begin
  3439. { hp2 and hp3 are the starting offsets, so mod = 0 this time }
  3440. if ((SourceRef.offset mod 16) = 0) and
  3441. (
  3442. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3443. (SourceRef.base = current_procinfo.framepointer) or
  3444. ((SourceRef.alignment >= 16) and ((SourceRef.alignment mod 16) = 0))
  3445. ) then
  3446. taicpu(hp2).opcode := MovAligned
  3447. else
  3448. taicpu(hp2).opcode := MovUnaligned;
  3449. taicpu(hp2).opsize := S_XMM;
  3450. taicpu(hp2).oper[1]^.reg := NewMMReg;
  3451. if ((TargetRef.offset mod 16) = 0) and
  3452. (
  3453. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3454. (TargetRef.base = current_procinfo.framepointer) or
  3455. ((TargetRef.alignment >= 16) and ((TargetRef.alignment mod 16) = 0))
  3456. ) then
  3457. taicpu(hp3).opcode := MovAligned
  3458. else
  3459. taicpu(hp3).opcode := MovUnaligned;
  3460. taicpu(hp3).opsize := S_XMM;
  3461. taicpu(hp3).oper[0]^.reg := NewMMReg;
  3462. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(NewMMReg) + ' to merge a pair of memory moves (MovMovMovMov2MovdqMovdq 2)', p);
  3463. RemoveInstruction(hp1);
  3464. RemoveCurrentP(p, hp2);
  3465. Result := True;
  3466. Exit;
  3467. end;
  3468. end;
  3469. end;
  3470. end;
  3471. end;
  3472. {$endif x86_64}
  3473. end;
  3474. else
  3475. { The write target should be a reg or a ref }
  3476. InternalError(2021091601);
  3477. end;
  3478. else
  3479. ;
  3480. end
  3481. else
  3482. { %treg is used afterwards, but all eventualities
  3483. other than the first MOV instruction being a constant
  3484. are covered by DeepMOVOpt, so only check for that }
  3485. if (taicpu(p).oper[0]^.typ = top_const) and
  3486. (
  3487. { For MOV operations, a size saving is only made if the register/const is byte-sized }
  3488. not (cs_opt_size in current_settings.optimizerswitches) or
  3489. (taicpu(hp1).opsize = S_B)
  3490. ) and
  3491. (
  3492. (taicpu(hp1).oper[1]^.typ = top_reg) or
  3493. ((taicpu(p).oper[0]^.val >= low(longint)) and (taicpu(p).oper[0]^.val <= high(longint)))
  3494. ) then
  3495. begin
  3496. DebugMsg(SPeepholeOptimization + debug_operstr(taicpu(hp1).oper[0]^) + ' = $' + debug_tostr(taicpu(p).oper[0]^.val) + '; changed to minimise pipeline stall (MovMov2Mov 6b)',hp1);
  3497. taicpu(hp1).loadconst(0, taicpu(p).oper[0]^.val);
  3498. end;
  3499. end;
  3500. if (taicpu(hp1).oper[0]^.typ = taicpu(p).oper[1]^.typ) and
  3501. (taicpu(hp1).oper[1]^.typ = taicpu(p).oper[0]^.typ) then
  3502. { mov reg1, mem1 or mov mem1, reg1
  3503. mov mem2, reg2 mov reg2, mem2}
  3504. begin
  3505. if OpsEqual(taicpu(hp1).oper[1]^,taicpu(p).oper[0]^) then
  3506. { mov reg1, mem1 or mov mem1, reg1
  3507. mov mem2, reg1 mov reg2, mem1}
  3508. begin
  3509. if OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  3510. { Removes the second statement from
  3511. mov reg1, mem1/reg2
  3512. mov mem1/reg2, reg1 }
  3513. begin
  3514. if taicpu(p).oper[0]^.typ=top_reg then
  3515. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,usedregs);
  3516. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 1',p);
  3517. RemoveInstruction(hp1);
  3518. Result:=true;
  3519. exit;
  3520. end
  3521. else
  3522. begin
  3523. TransferUsedRegs(TmpUsedRegs);
  3524. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  3525. if (taicpu(p).oper[1]^.typ = top_ref) and
  3526. { mov reg1, mem1
  3527. mov mem2, reg1 }
  3528. (taicpu(hp1).oper[0]^.ref^.refaddr = addr_no) and
  3529. GetNextInstruction(hp1, hp2) and
  3530. MatchInstruction(hp2,A_CMP,[taicpu(p).opsize]) and
  3531. OpsEqual(taicpu(p).oper[1]^,taicpu(hp2).oper[0]^) and
  3532. OpsEqual(taicpu(p).oper[0]^,taicpu(hp2).oper[1]^) and
  3533. not(RegUsedAfterInstruction(taicpu(p).oper[0]^.reg, hp2, TmpUsedRegs)) then
  3534. { change to
  3535. mov reg1, mem1 mov reg1, mem1
  3536. mov mem2, reg1 cmp reg1, mem2
  3537. cmp mem1, reg1
  3538. }
  3539. begin
  3540. RemoveInstruction(hp2);
  3541. taicpu(hp1).opcode := A_CMP;
  3542. taicpu(hp1).loadref(1,taicpu(hp1).oper[0]^.ref^);
  3543. taicpu(hp1).loadreg(0,taicpu(p).oper[0]^.reg);
  3544. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,UsedRegs);
  3545. DebugMsg(SPeepholeOptimization + 'MovMovCmp2MovCmp done',hp1);
  3546. end;
  3547. end;
  3548. end
  3549. else if (taicpu(p).oper[1]^.typ=top_ref) and
  3550. OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  3551. begin
  3552. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,UsedRegs);
  3553. taicpu(hp1).loadreg(0,taicpu(p).oper[0]^.reg);
  3554. DebugMsg(SPeepholeOptimization + 'MovMov2MovMov1 done',p);
  3555. end
  3556. else
  3557. begin
  3558. TransferUsedRegs(TmpUsedRegs);
  3559. if GetNextInstruction(hp1, hp2) and
  3560. MatchOpType(taicpu(p),top_ref,top_reg) and
  3561. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  3562. (taicpu(hp1).oper[1]^.typ = top_ref) and
  3563. MatchInstruction(hp2,A_MOV,[taicpu(p).opsize]) and
  3564. MatchOpType(taicpu(hp2),top_ref,top_reg) and
  3565. RefsEqual(taicpu(hp2).oper[0]^.ref^, taicpu(hp1).oper[1]^.ref^) then
  3566. if not RegInRef(taicpu(hp2).oper[1]^.reg,taicpu(hp2).oper[0]^.ref^) and
  3567. not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,tmpUsedRegs)) then
  3568. { mov mem1, %reg1
  3569. mov %reg1, mem2
  3570. mov mem2, reg2
  3571. to:
  3572. mov mem1, reg2
  3573. mov reg2, mem2}
  3574. begin
  3575. AllocRegBetween(taicpu(hp2).oper[1]^.reg,p,hp2,usedregs);
  3576. DebugMsg(SPeepholeOptimization + 'MovMovMov2MovMov 1 done',p);
  3577. taicpu(p).loadoper(1,taicpu(hp2).oper[1]^);
  3578. taicpu(hp1).loadoper(0,taicpu(hp2).oper[1]^);
  3579. RemoveInstruction(hp2);
  3580. Result := True;
  3581. end
  3582. {$ifdef i386}
  3583. { this is enabled for i386 only, as the rules to create the reg sets below
  3584. are too complicated for x86-64, so this makes this code too error prone
  3585. on x86-64
  3586. }
  3587. else if (taicpu(p).oper[1]^.reg <> taicpu(hp2).oper[1]^.reg) and
  3588. not(RegInRef(taicpu(p).oper[1]^.reg,taicpu(p).oper[0]^.ref^)) and
  3589. not(RegInRef(taicpu(hp2).oper[1]^.reg,taicpu(hp2).oper[0]^.ref^)) then
  3590. { mov mem1, reg1 mov mem1, reg1
  3591. mov reg1, mem2 mov reg1, mem2
  3592. mov mem2, reg2 mov mem2, reg1
  3593. to: to:
  3594. mov mem1, reg1 mov mem1, reg1
  3595. mov mem1, reg2 mov reg1, mem2
  3596. mov reg1, mem2
  3597. or (if mem1 depends on reg1
  3598. and/or if mem2 depends on reg2)
  3599. to:
  3600. mov mem1, reg1
  3601. mov reg1, mem2
  3602. mov reg1, reg2
  3603. }
  3604. begin
  3605. taicpu(hp1).loadRef(0,taicpu(p).oper[0]^.ref^);
  3606. taicpu(hp1).loadReg(1,taicpu(hp2).oper[1]^.reg);
  3607. taicpu(hp2).loadRef(1,taicpu(hp2).oper[0]^.ref^);
  3608. taicpu(hp2).loadReg(0,taicpu(p).oper[1]^.reg);
  3609. AllocRegBetween(taicpu(p).oper[1]^.reg,p,hp2,usedregs);
  3610. if (taicpu(p).oper[0]^.ref^.base <> NR_NO) and
  3611. (getsupreg(taicpu(p).oper[0]^.ref^.base) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX,RS_ESI,RS_EDI]) then
  3612. AllocRegBetween(taicpu(p).oper[0]^.ref^.base,p,hp2,usedregs);
  3613. if (taicpu(p).oper[0]^.ref^.index <> NR_NO) and
  3614. (getsupreg(taicpu(p).oper[0]^.ref^.index) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX,RS_ESI,RS_EDI]) then
  3615. AllocRegBetween(taicpu(p).oper[0]^.ref^.index,p,hp2,usedregs);
  3616. end
  3617. else if (taicpu(hp1).Oper[0]^.reg <> taicpu(hp2).Oper[1]^.reg) then
  3618. begin
  3619. taicpu(hp2).loadReg(0,taicpu(hp1).Oper[0]^.reg);
  3620. AllocRegBetween(taicpu(p).oper[1]^.reg,p,hp2,usedregs);
  3621. end
  3622. else
  3623. begin
  3624. RemoveInstruction(hp2);
  3625. end
  3626. {$endif i386}
  3627. ;
  3628. end;
  3629. end
  3630. { movl [mem1],reg1
  3631. movl [mem1],reg2
  3632. to
  3633. movl [mem1],reg1
  3634. movl reg1,reg2
  3635. }
  3636. else if MatchOpType(taicpu(p),top_ref,top_reg) and
  3637. MatchOpType(taicpu(hp1),top_ref,top_reg) and
  3638. (taicpu(p).opsize = taicpu(hp1).opsize) and
  3639. RefsEqual(taicpu(p).oper[0]^.ref^,taicpu(hp1).oper[0]^.ref^) and
  3640. (taicpu(p).oper[0]^.ref^.volatility=[]) and
  3641. (taicpu(hp1).oper[0]^.ref^.volatility=[]) and
  3642. not(SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.ref^.base)) and
  3643. not(SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.ref^.index)) then
  3644. begin
  3645. DebugMsg(SPeepholeOptimization + 'MovMov2MovMov 2',p);
  3646. taicpu(hp1).loadReg(0,taicpu(p).oper[1]^.reg);
  3647. end;
  3648. { movl const1,[mem1]
  3649. movl [mem1],reg1
  3650. to
  3651. movl const1,reg1
  3652. movl reg1,[mem1]
  3653. }
  3654. if MatchOpType(Taicpu(p),top_const,top_ref) and
  3655. MatchOpType(Taicpu(hp1),top_ref,top_reg) and
  3656. (taicpu(p).opsize = taicpu(hp1).opsize) and
  3657. RefsEqual(taicpu(hp1).oper[0]^.ref^,taicpu(p).oper[1]^.ref^) and
  3658. not(RegInRef(taicpu(hp1).oper[1]^.reg,taicpu(hp1).oper[0]^.ref^)) then
  3659. begin
  3660. AllocRegBetween(taicpu(hp1).oper[1]^.reg,p,hp1,usedregs);
  3661. taicpu(hp1).loadReg(0,taicpu(hp1).oper[1]^.reg);
  3662. taicpu(hp1).loadRef(1,taicpu(p).oper[1]^.ref^);
  3663. taicpu(p).loadReg(1,taicpu(hp1).oper[0]^.reg);
  3664. taicpu(hp1).fileinfo := taicpu(p).fileinfo;
  3665. DebugMsg(SPeepholeOptimization + 'MovMov2MovMov 1',p);
  3666. Result:=true;
  3667. exit;
  3668. end;
  3669. { mov x,reg1; mov y,reg1 -> mov y,reg1 is handled by the Mov2Nop 5 optimisation }
  3670. { Change:
  3671. movl %reg1,%reg2
  3672. movl x(%reg1),%reg1 (If something other than %reg1 is written to, DeepMOVOpt would have caught it)
  3673. movl x(%reg2),%regX (%regX can be %reg2 or something else)
  3674. To:
  3675. movl %reg1,%reg2 (if %regX = %reg2, then remove this instruction)
  3676. movl x(%reg1),%reg1
  3677. movl %reg1,%regX
  3678. }
  3679. if MatchOpType(taicpu(p), top_reg, top_reg) then
  3680. begin
  3681. p_SourceReg := taicpu(p).oper[0]^.reg;
  3682. { Remember that p_TargetReg contains taicpu(p).oper[1]^.reg }
  3683. if (taicpu(hp1).oper[0]^.typ = top_ref) { The other operand will be a register } and
  3684. (taicpu(hp1).oper[1]^.reg = p_SourceReg) and
  3685. RegInRef(p_SourceReg, taicpu(hp1).oper[0]^.ref^) and
  3686. GetNextInstruction(hp1, hp2) and
  3687. MatchInstruction(hp2, A_MOV, [taicpu(p).opsize]) and
  3688. (taicpu(hp2).oper[0]^.typ = top_ref) { The other operand will be a register } then
  3689. begin
  3690. SourceRef := taicpu(hp2).oper[0]^.ref^;
  3691. if RegInRef(p_TargetReg, SourceRef) and
  3692. { If %reg1 also appears in the second reference, then it will
  3693. not refer to the same memory block as the first reference }
  3694. not RegInRef(p_SourceReg, SourceRef) then
  3695. begin
  3696. { Check to see if the references match if %reg2 is changed to %reg1 }
  3697. if SourceRef.base = p_TargetReg then
  3698. SourceRef.base := p_SourceReg;
  3699. if SourceRef.index = p_TargetReg then
  3700. SourceRef.index := p_SourceReg;
  3701. { RefsEqual also checks to ensure both references are non-volatile }
  3702. if RefsEqual(taicpu(hp1).oper[0]^.ref^, SourceRef) then
  3703. begin
  3704. taicpu(hp2).loadreg(0, p_SourceReg);
  3705. DebugMsg(SPeepholeOptimization + 'Optimised register duplication and memory read (MovMovMov2MovMovMov)', p);
  3706. Result := True;
  3707. if taicpu(hp2).oper[1]^.reg = p_TargetReg then
  3708. begin
  3709. DebugMsg(SPeepholeOptimization + 'Mov2Nop 5a done', p);
  3710. RemoveCurrentP(p, hp1);
  3711. Exit;
  3712. end
  3713. else
  3714. begin
  3715. { Check to see if %reg2 is no longer in use }
  3716. TransferUsedRegs(TmpUsedRegs);
  3717. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  3718. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  3719. if not RegUsedAfterInstruction(p_TargetReg, hp2, TmpUsedRegs) then
  3720. begin
  3721. DebugMsg(SPeepholeOptimization + 'Mov2Nop 5b done', p);
  3722. RemoveCurrentP(p, hp1);
  3723. Exit;
  3724. end;
  3725. end;
  3726. { If we reach this point, p and hp1 weren't actually modified,
  3727. so we can do a bit more work on this pass }
  3728. end;
  3729. end;
  3730. end;
  3731. end;
  3732. end;
  3733. {$ifdef x86_64}
  3734. { Change:
  3735. movl %reg1l,%reg2l
  3736. movq %reg2q,%reg3q (%reg1 <> %reg3)
  3737. To:
  3738. movl %reg1l,%reg2l
  3739. movl %reg1l,%reg3l (Upper 32 bits of %reg3q will be zero)
  3740. If %reg1 = %reg3, convert to:
  3741. movl %reg1l,%reg2l
  3742. andl %reg1l,%reg1l
  3743. }
  3744. if (taicpu(p).opsize = S_L) and MatchInstruction(hp1,A_MOV,[S_Q]) and
  3745. MatchOpType(taicpu(p), top_reg, top_reg) and
  3746. MatchOpType(taicpu(hp1), top_reg, top_reg) and
  3747. SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^.reg) then
  3748. begin
  3749. TransferUsedRegs(TmpUsedRegs);
  3750. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  3751. taicpu(hp1).opsize := S_L;
  3752. taicpu(hp1).loadreg(0, taicpu(p).oper[0]^.reg);
  3753. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  3754. AllocRegBetween(taicpu(p).oper[0]^.reg, p, hp1, UsedRegs);
  3755. if (taicpu(p).oper[0]^.reg = taicpu(hp1).oper[1]^.reg) then
  3756. begin
  3757. { %reg1 = %reg3 }
  3758. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlAndl 1)', hp1);
  3759. taicpu(hp1).opcode := A_AND;
  3760. end
  3761. else
  3762. begin
  3763. { %reg1 <> %reg3 }
  3764. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlMovl 1)', hp1);
  3765. end;
  3766. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) then
  3767. begin
  3768. DebugMsg(SPeepholeOptimization + 'Mov2Nop 8 done', p);
  3769. RemoveCurrentP(p, hp1);
  3770. Result := True;
  3771. Exit;
  3772. end
  3773. else
  3774. begin
  3775. { Initial instruction wasn't actually changed }
  3776. Include(OptsToCheck, aoc_ForceNewIteration);
  3777. { if %reg1 = %reg3, don't do the long-distance lookahead that
  3778. appears below since %reg1 has technically changed }
  3779. if taicpu(hp1).opcode = A_AND then
  3780. Exit;
  3781. end;
  3782. end;
  3783. {$endif x86_64}
  3784. { search further than the next instruction for a mov (as long as it's not a jump) }
  3785. if not is_calljmpuncondret(taicpu(hp1).opcode) and
  3786. { check as much as possible before the expensive GetNextInstructionUsingRegCond call }
  3787. (taicpu(p).oper[1]^.typ = top_reg) and
  3788. (taicpu(p).oper[0]^.typ in [top_reg,top_const]) and
  3789. not RegModifiedByInstruction(taicpu(p).oper[1]^.reg, hp1) then
  3790. begin
  3791. { we work with hp2 here, so hp1 can be still used later on when
  3792. checking for GetNextInstruction_p }
  3793. hp3 := hp1;
  3794. { Initialise CrossJump (if it becomes True at any point, it will remain True) }
  3795. CrossJump := (taicpu(hp1).opcode = A_Jcc);
  3796. { Remember that p_TargetReg contains taicpu(p).oper[1]^.reg }
  3797. TransferUsedRegs(TmpUsedRegs);
  3798. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  3799. if NotFirstIteration then
  3800. JumpTracking := TLinkedList.Create
  3801. else
  3802. JumpTracking := nil;
  3803. while GetNextInstructionUsingRegCond(hp3,hp2,p_TargetReg,JumpTracking,CrossJump) and
  3804. { GetNextInstructionUsingRegCond only searches one instruction ahead unless -O3 is specified }
  3805. (hp2.typ=ait_instruction) do
  3806. begin
  3807. case taicpu(hp2).opcode of
  3808. A_POP:
  3809. if MatchOperand(taicpu(hp2).oper[0]^,p_TargetReg) then
  3810. begin
  3811. if not CrossJump and
  3812. not RegUsedBetween(p_TargetReg, p, hp2) then
  3813. begin
  3814. { We can remove the original MOV since the register
  3815. wasn't used between it and its popping from the stack }
  3816. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3c done',p);
  3817. RemoveCurrentp(p, hp1);
  3818. Result := True;
  3819. JumpTracking.Free;
  3820. Exit;
  3821. end;
  3822. { Can't go any further }
  3823. Break;
  3824. end;
  3825. A_MOV:
  3826. if MatchOperand(taicpu(hp2).oper[0]^,p_TargetReg) and
  3827. ((taicpu(p).oper[0]^.typ=top_const) or
  3828. ((taicpu(p).oper[0]^.typ=top_reg) and
  3829. not(RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp2))
  3830. )
  3831. ) then
  3832. begin
  3833. { we have
  3834. mov x, %treg
  3835. mov %treg, y
  3836. }
  3837. { We don't need to call UpdateUsedRegs for every instruction between
  3838. p and hp2 because the register we're concerned about will not
  3839. become deallocated (otherwise GetNextInstructionUsingReg would
  3840. have stopped at an earlier instruction). [Kit] }
  3841. TempRegUsed :=
  3842. CrossJump { Assume the register is in use if it crossed a conditional jump } or
  3843. RegReadByInstruction(p_TargetReg, hp3) or
  3844. RegUsedAfterInstruction(p_TargetReg, hp2, TmpUsedRegs);
  3845. case taicpu(p).oper[0]^.typ Of
  3846. top_reg:
  3847. begin
  3848. { change
  3849. mov %reg, %treg
  3850. mov %treg, y
  3851. to
  3852. mov %reg, y
  3853. }
  3854. p_SourceReg := taicpu(p).oper[0]^.reg; { Saves on a handful of pointer dereferences }
  3855. RegName1 := debug_regname(taicpu(hp2).oper[0]^.reg);
  3856. if MatchOperand(taicpu(hp2).oper[1]^, p_SourceReg) then
  3857. begin
  3858. { %reg = y - remove hp2 completely (doing it here instead of relying on
  3859. the "mov %reg,%reg" optimisation might cut down on a pass iteration) }
  3860. if TempRegUsed then
  3861. begin
  3862. DebugMsg(SPeepholeOptimization + debug_regname(p_SourceReg) + ' = ' + RegName1 + '; removed unnecessary instruction (MovMov2MovNop 6b}',hp2);
  3863. AllocRegBetween(p_SourceReg, p, hp2, UsedRegs);
  3864. { Set the start of the next GetNextInstructionUsingRegCond search
  3865. to start at the entry right before hp2 (which is about to be removed) }
  3866. hp3 := tai(hp2.Previous);
  3867. RemoveInstruction(hp2);
  3868. { See if there's more we can optimise }
  3869. Continue;
  3870. end
  3871. else
  3872. begin
  3873. RemoveInstruction(hp2);
  3874. { We can remove the original MOV too }
  3875. DebugMsg(SPeepholeOptimization + 'MovMov2NopNop 6b done',p);
  3876. RemoveCurrentP(p, hp1);
  3877. Result:=true;
  3878. JumpTracking.Free;
  3879. Exit;
  3880. end;
  3881. end
  3882. else
  3883. begin
  3884. AllocRegBetween(p_SourceReg, p, hp2, UsedRegs);
  3885. taicpu(hp2).loadReg(0, p_SourceReg);
  3886. DebugMsg(SPeepholeOptimization + RegName1 + ' = ' + debug_regname(p_SourceReg) + '; changed to minimise pipeline stall (MovMov2Mov 6a}',hp2);
  3887. { Check to see if the register also appears in the reference }
  3888. if (taicpu(hp2).oper[1]^.typ = top_ref) then
  3889. ReplaceRegisterInRef(taicpu(hp2).oper[1]^.ref^, p_TargetReg, p_SourceReg);
  3890. { Don't remove the first instruction if the temporary register is in use }
  3891. if not TempRegUsed and
  3892. { ReplaceRegisterInRef won't actually replace the register if it's a different size }
  3893. not RegInOp(p_TargetReg, taicpu(hp2).oper[1]^) then
  3894. begin
  3895. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 6 done',p);
  3896. RemoveCurrentP(p, hp1);
  3897. Result:=true;
  3898. JumpTracking.Free;
  3899. Exit;
  3900. end;
  3901. { No need to set Result to True here. If there's another instruction later
  3902. on that can be optimised, it will be detected when the main Pass 1 loop
  3903. reaches what is now hp2 and passes it through OptPass1MOV. [Kit] }
  3904. end;
  3905. end;
  3906. top_const:
  3907. if not (cs_opt_size in current_settings.optimizerswitches) or (taicpu(hp2).opsize = S_B) then
  3908. begin
  3909. { change
  3910. mov const, %treg
  3911. mov %treg, y
  3912. to
  3913. mov const, y
  3914. }
  3915. if (taicpu(hp2).oper[1]^.typ=top_reg) or
  3916. ((taicpu(p).oper[0]^.val>=low(longint)) and (taicpu(p).oper[0]^.val<=high(longint))) then
  3917. begin
  3918. RegName1 := debug_regname(taicpu(hp2).oper[0]^.reg);
  3919. taicpu(hp2).loadOper(0,taicpu(p).oper[0]^);
  3920. if TempRegUsed then
  3921. begin
  3922. { Don't remove the first instruction if the temporary register is in use }
  3923. DebugMsg(SPeepholeOptimization + RegName1 + ' = ' + debug_tostr(taicpu(p).oper[0]^.val) + '; changed to minimise pipeline stall (MovMov2Mov 7a)',hp2);
  3924. { No need to set Result to True. If there's another instruction later on
  3925. that can be optimised, it will be detected when the main Pass 1 loop
  3926. reaches what is now hp2 and passes it through OptPass1MOV. [Kit] };
  3927. end
  3928. else
  3929. begin
  3930. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 7 done',p);
  3931. RemoveCurrentP(p, hp1);
  3932. Result:=true;
  3933. Exit;
  3934. end;
  3935. end;
  3936. end;
  3937. else
  3938. Internalerror(2019103001);
  3939. end;
  3940. end
  3941. else if MatchOperand(taicpu(hp2).oper[1]^, p_TargetReg) then
  3942. begin
  3943. if not CrossJump and
  3944. not RegUsedBetween(p_TargetReg, p, hp2) and
  3945. not RegReadByInstruction(p_TargetReg, hp2) then
  3946. begin
  3947. { Register is not used before it is overwritten }
  3948. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3a done',p);
  3949. RemoveCurrentp(p, hp1);
  3950. Result := True;
  3951. Exit;
  3952. end;
  3953. if (taicpu(p).oper[0]^.typ = top_const) and
  3954. (taicpu(hp2).oper[0]^.typ = top_const) then
  3955. begin
  3956. if taicpu(p).oper[0]^.val = taicpu(hp2).oper[0]^.val then
  3957. begin
  3958. { Same value - register hasn't changed }
  3959. DebugMsg(SPeepholeOptimization + 'Mov2Nop 2 done', hp2);
  3960. RemoveInstruction(hp2);
  3961. Result := True;
  3962. { See if there's more we can optimise }
  3963. Continue;
  3964. end;
  3965. end;
  3966. {$ifdef x86_64}
  3967. end
  3968. { Change:
  3969. movl %reg1l,%reg2l
  3970. ...
  3971. movq %reg2q,%reg3q (%reg1 <> %reg3)
  3972. To:
  3973. movl %reg1l,%reg2l
  3974. ...
  3975. movl %reg1l,%reg3l (Upper 32 bits of %reg3q will be zero)
  3976. If %reg1 = %reg3, convert to:
  3977. movl %reg1l,%reg2l
  3978. ...
  3979. andl %reg1l,%reg1l
  3980. }
  3981. else if (taicpu(p).opsize = S_L) and MatchInstruction(hp2,A_MOV,[S_Q]) and
  3982. (taicpu(p).oper[0]^.typ = top_reg) and
  3983. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  3984. SuperRegistersEqual(p_TargetReg, taicpu(hp2).oper[0]^.reg) and
  3985. not RegModifiedBetween(p_TargetReg, p, hp2) then
  3986. begin
  3987. TempRegUsed :=
  3988. CrossJump { Assume the register is in use if it crossed a conditional jump } or
  3989. RegReadByInstruction(p_TargetReg, hp3) or
  3990. RegUsedAfterInstruction(p_TargetReg, hp2, TmpUsedRegs);
  3991. taicpu(hp2).opsize := S_L;
  3992. taicpu(hp2).loadreg(0, taicpu(p).oper[0]^.reg);
  3993. setsubreg(taicpu(hp2).oper[1]^.reg, R_SUBD);
  3994. AllocRegBetween(taicpu(p).oper[0]^.reg, p, hp2, UsedRegs);
  3995. if (taicpu(p).oper[0]^.reg = taicpu(hp2).oper[1]^.reg) then
  3996. begin
  3997. { %reg1 = %reg3 }
  3998. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlAndl 2)', hp2);
  3999. taicpu(hp2).opcode := A_AND;
  4000. end
  4001. else
  4002. begin
  4003. { %reg1 <> %reg3 }
  4004. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlMovl 2)', hp2);
  4005. end;
  4006. if not TempRegUsed then
  4007. begin
  4008. DebugMsg(SPeepholeOptimization + 'Mov2Nop 8a done', p);
  4009. RemoveCurrentP(p, hp1);
  4010. Result := True;
  4011. Exit;
  4012. end
  4013. else
  4014. begin
  4015. { Initial instruction wasn't actually changed }
  4016. Include(OptsToCheck, aoc_ForceNewIteration);
  4017. { if %reg1 = %reg3, don't do the long-distance lookahead that
  4018. appears below since %reg1 has technically changed }
  4019. if taicpu(hp2).opcode = A_AND then
  4020. Break;
  4021. end;
  4022. {$endif x86_64}
  4023. end;
  4024. A_MOVZX, A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  4025. if MatchOpType(taicpu(hp2), top_reg, top_reg) and
  4026. MatchOperand(taicpu(hp2).oper[0]^, p_TargetReg) and
  4027. SuperRegistersEqual(taicpu(hp2).oper[1]^.reg, p_TargetReg) then
  4028. begin
  4029. {
  4030. Change from:
  4031. mov ###, %reg
  4032. ...
  4033. movs/z %reg,%reg (Same register, just different sizes)
  4034. To:
  4035. movs/z ###, %reg (Longer version)
  4036. ...
  4037. (remove)
  4038. }
  4039. DebugMsg(SPeepholeOptimization + 'MovMovs/z2Mov/s/z done', p);
  4040. taicpu(p).oper[1]^.reg := taicpu(hp2).oper[1]^.reg;
  4041. { Keep the first instruction as mov if ### is a constant }
  4042. if taicpu(p).oper[0]^.typ = top_const then
  4043. taicpu(p).opsize := reg2opsize(taicpu(hp2).oper[1]^.reg)
  4044. else
  4045. begin
  4046. taicpu(p).opcode := taicpu(hp2).opcode;
  4047. taicpu(p).opsize := taicpu(hp2).opsize;
  4048. end;
  4049. DebugMsg(SPeepholeOptimization + 'Removed movs/z instruction and extended earlier write (MovMovs/z2Mov/s/z)', hp2);
  4050. AllocRegBetween(taicpu(hp2).oper[1]^.reg, p, hp2, UsedRegs);
  4051. RemoveInstruction(hp2);
  4052. Result := True;
  4053. JumpTracking.Free;
  4054. Exit;
  4055. end;
  4056. else
  4057. { Move down to the if-block below };
  4058. end;
  4059. { Also catches MOV/S/Z instructions that aren't modified }
  4060. if taicpu(p).oper[0]^.typ = top_reg then
  4061. begin
  4062. p_SourceReg := taicpu(p).oper[0]^.reg;
  4063. if
  4064. not RegModifiedByInstruction(p_SourceReg, hp3) and
  4065. not RegModifiedBetween(p_SourceReg, hp3, hp2) and
  4066. DeepMOVOpt(taicpu(p), taicpu(hp2)) then
  4067. begin
  4068. Result := True;
  4069. { Just in case something didn't get modified (e.g. an
  4070. implicit register). Also, if it does read from this
  4071. register, then there's no longer an advantage to
  4072. changing the register on subsequent instructions.}
  4073. if not RegReadByInstruction(p_TargetReg, hp2) then
  4074. begin
  4075. { If a conditional jump was crossed, do not delete
  4076. the original MOV no matter what }
  4077. if not CrossJump and
  4078. { RegEndOfLife returns True if the register is
  4079. deallocated before the next instruction or has
  4080. been loaded with a new value }
  4081. RegEndOfLife(p_TargetReg, taicpu(hp2)) then
  4082. begin
  4083. { We can remove the original MOV }
  4084. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3b done',p);
  4085. RemoveCurrentp(p, hp1);
  4086. JumpTracking.Free;
  4087. Result := True;
  4088. Exit;
  4089. end;
  4090. if not RegModifiedByInstruction(p_TargetReg, hp2) then
  4091. begin
  4092. { See if there's more we can optimise }
  4093. hp3 := hp2;
  4094. Continue;
  4095. end;
  4096. end;
  4097. end;
  4098. end;
  4099. { Break out of the while loop under normal circumstances }
  4100. Break;
  4101. end;
  4102. JumpTracking.Free;
  4103. end;
  4104. if (aoc_MovAnd2Mov_3 in OptsToCheck) and
  4105. (taicpu(p).oper[1]^.typ = top_reg) and
  4106. (taicpu(p).opsize = S_L) and
  4107. GetNextInstructionUsingRegTrackingUse(p,hp2,taicpu(p).oper[1]^.reg) and
  4108. (hp2.typ = ait_instruction) and
  4109. (taicpu(hp2).opcode = A_AND) and
  4110. (MatchOpType(taicpu(hp2),top_const,top_reg) or
  4111. (MatchOpType(taicpu(hp2),top_reg,top_reg) and
  4112. MatchOperand(taicpu(hp2).oper[0]^,taicpu(hp2).oper[1]^))
  4113. ) then
  4114. begin
  4115. if SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp2).oper[1]^.reg) then
  4116. begin
  4117. if ((taicpu(hp2).oper[0]^.typ=top_const) and (taicpu(hp2).oper[0]^.val = $ffffffff)) or
  4118. ((taicpu(hp2).oper[0]^.typ=top_reg) and (taicpu(hp2).opsize=S_L)) then
  4119. begin
  4120. { Optimize out:
  4121. mov x, %reg
  4122. and ffffffffh, %reg
  4123. }
  4124. DebugMsg(SPeepholeOptimization + 'MovAnd2Mov 3 done',p);
  4125. RemoveInstruction(hp2);
  4126. Result:=true;
  4127. exit;
  4128. end;
  4129. end;
  4130. end;
  4131. { leave out the mov from "mov reg, x(%frame_pointer); leave/ret" (with
  4132. x >= RetOffset) as it doesn't do anything (it writes either to a
  4133. parameter or to the temporary storage room for the function
  4134. result)
  4135. }
  4136. if IsExitCode(hp1) and
  4137. (taicpu(p).oper[1]^.typ = top_ref) and
  4138. (taicpu(p).oper[1]^.ref^.index = NR_NO) and
  4139. (
  4140. (
  4141. (taicpu(p).oper[1]^.ref^.base = current_procinfo.FramePointer) and
  4142. not (
  4143. assigned(current_procinfo.procdef.funcretsym) and
  4144. (taicpu(p).oper[1]^.ref^.offset <= tabstractnormalvarsym(current_procinfo.procdef.funcretsym).localloc.reference.offset)
  4145. )
  4146. ) or
  4147. { Also discard writes to the stack that are below the base pointer,
  4148. as this is temporary storage rather than a function result on the
  4149. stack, say. }
  4150. (
  4151. (taicpu(p).oper[1]^.ref^.base = NR_STACK_POINTER_REG) and
  4152. (taicpu(p).oper[1]^.ref^.offset < current_procinfo.final_localsize)
  4153. )
  4154. ) then
  4155. begin
  4156. RemoveCurrentp(p, hp1);
  4157. DebugMsg(SPeepholeOptimization + 'removed deadstore before leave/ret',p);
  4158. RemoveLastDeallocForFuncRes(p);
  4159. Result:=true;
  4160. exit;
  4161. end;
  4162. if MatchInstruction(hp1,A_CMP,A_TEST,[taicpu(p).opsize]) then
  4163. begin
  4164. if MatchOpType(taicpu(p),top_reg,top_ref) and
  4165. (taicpu(hp1).oper[1]^.typ = top_ref) and
  4166. RefsEqual(taicpu(p).oper[1]^.ref^, taicpu(hp1).oper[1]^.ref^) then
  4167. begin
  4168. { change
  4169. mov reg1, mem1
  4170. test/cmp x, mem1
  4171. to
  4172. mov reg1, mem1
  4173. test/cmp x, reg1
  4174. }
  4175. taicpu(hp1).loadreg(1,taicpu(p).oper[0]^.reg);
  4176. DebugMsg(SPeepholeOptimization + 'MovTestCmp2MovTestCmp 1',hp1);
  4177. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,usedregs);
  4178. Result := True;
  4179. Exit;
  4180. end;
  4181. if DoMovCmpMemOpt(p, hp1, True) then
  4182. begin
  4183. Result := True;
  4184. Exit;
  4185. end;
  4186. end;
  4187. if MatchInstruction(hp1,A_LEA,[S_L{$ifdef x86_64},S_Q{$endif x86_64}]) and
  4188. { If the flags register is in use, don't change the instruction to an
  4189. ADD otherwise this will scramble the flags. [Kit] }
  4190. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  4191. begin
  4192. if MatchOpType(Taicpu(p),top_ref,top_reg) and
  4193. ((MatchReference(Taicpu(hp1).oper[0]^.ref^,Taicpu(hp1).oper[1]^.reg,Taicpu(p).oper[1]^.reg) and
  4194. (Taicpu(hp1).oper[0]^.ref^.base<>Taicpu(p).oper[1]^.reg)
  4195. ) or
  4196. (MatchReference(Taicpu(hp1).oper[0]^.ref^,Taicpu(p).oper[1]^.reg,Taicpu(hp1).oper[1]^.reg) and
  4197. (Taicpu(hp1).oper[0]^.ref^.index<>Taicpu(p).oper[1]^.reg)
  4198. )
  4199. ) then
  4200. { mov reg1,ref
  4201. lea reg2,[reg1,reg2]
  4202. to
  4203. add reg2,ref}
  4204. begin
  4205. TransferUsedRegs(TmpUsedRegs);
  4206. { reg1 may not be used afterwards }
  4207. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs)) then
  4208. begin
  4209. Taicpu(hp1).opcode:=A_ADD;
  4210. Taicpu(hp1).oper[0]^.ref^:=Taicpu(p).oper[0]^.ref^;
  4211. DebugMsg(SPeepholeOptimization + 'MovLea2Add done',hp1);
  4212. RemoveCurrentp(p, hp1);
  4213. result:=true;
  4214. exit;
  4215. end;
  4216. end;
  4217. { If the LEA instruction can be converted into an arithmetic instruction,
  4218. it may be possible to then fold it in the next optimisation, otherwise
  4219. there's nothing more that can be optimised here. }
  4220. if not ConvertLEA(taicpu(hp1)) then
  4221. Exit;
  4222. end;
  4223. if (taicpu(p).oper[1]^.typ = top_reg) and
  4224. (hp1.typ = ait_instruction) and
  4225. GetNextInstruction(hp1, hp2) and
  4226. MatchInstruction(hp2,A_MOV,[]) and
  4227. (SuperRegistersEqual(taicpu(hp2).oper[0]^.reg,taicpu(p).oper[1]^.reg)) and
  4228. (topsize2memsize[taicpu(hp1).opsize]>=topsize2memsize[taicpu(hp2).opsize]) and
  4229. (
  4230. IsFoldableArithOp(taicpu(hp1), taicpu(p).oper[1]^.reg)
  4231. {$ifdef x86_64}
  4232. or
  4233. (
  4234. (taicpu(p).opsize=S_L) and (taicpu(hp1).opsize=S_Q) and (taicpu(hp2).opsize=S_L) and
  4235. IsFoldableArithOp(taicpu(hp1), newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[1]^.reg),R_SUBQ))
  4236. )
  4237. {$endif x86_64}
  4238. ) then
  4239. begin
  4240. if OpsEqual(taicpu(hp2).oper[1]^, taicpu(p).oper[0]^) and
  4241. (taicpu(hp2).oper[0]^.typ=top_reg) then
  4242. { change movsX/movzX reg/ref, reg2
  4243. add/sub/or/... reg3/$const, reg2
  4244. mov reg2 reg/ref
  4245. dealloc reg2
  4246. to
  4247. add/sub/or/... reg3/$const, reg/ref }
  4248. begin
  4249. TransferUsedRegs(TmpUsedRegs);
  4250. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  4251. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  4252. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  4253. begin
  4254. { by example:
  4255. movswl %si,%eax movswl %si,%eax p
  4256. decl %eax addl %edx,%eax hp1
  4257. movw %ax,%si movw %ax,%si hp2
  4258. ->
  4259. movswl %si,%eax movswl %si,%eax p
  4260. decw %eax addw %edx,%eax hp1
  4261. movw %ax,%si movw %ax,%si hp2
  4262. }
  4263. DebugMsg(SPeepholeOptimization + 'MovOpMov2Op ('+
  4264. debug_op2str(taicpu(p).opcode)+debug_opsize2str(taicpu(p).opsize)+' '+
  4265. debug_op2str(taicpu(hp1).opcode)+debug_opsize2str(taicpu(hp1).opsize)+' '+
  4266. debug_op2str(taicpu(hp2).opcode)+debug_opsize2str(taicpu(hp2).opsize)+')',p);
  4267. taicpu(hp1).changeopsize(taicpu(hp2).opsize);
  4268. {
  4269. ->
  4270. movswl %si,%eax movswl %si,%eax p
  4271. decw %si addw %dx,%si hp1
  4272. movw %ax,%si movw %ax,%si hp2
  4273. }
  4274. case taicpu(hp1).ops of
  4275. 1:
  4276. begin
  4277. taicpu(hp1).loadoper(0, taicpu(hp2).oper[1]^);
  4278. if taicpu(hp1).oper[0]^.typ=top_reg then
  4279. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4280. end;
  4281. 2:
  4282. begin
  4283. taicpu(hp1).loadoper(1, taicpu(hp2).oper[1]^);
  4284. if (taicpu(hp1).oper[0]^.typ=top_reg) and
  4285. (taicpu(hp1).opcode<>A_SHL) and
  4286. (taicpu(hp1).opcode<>A_SHR) and
  4287. (taicpu(hp1).opcode<>A_SAR) then
  4288. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4289. end;
  4290. else
  4291. internalerror(2008042701);
  4292. end;
  4293. {
  4294. ->
  4295. decw %si addw %dx,%si p
  4296. }
  4297. RemoveInstruction(hp2);
  4298. RemoveCurrentP(p, hp1);
  4299. Result:=True;
  4300. Exit;
  4301. end;
  4302. end;
  4303. if MatchOpType(taicpu(hp2),top_reg,top_reg) and
  4304. not(SuperRegistersEqual(taicpu(hp1).oper[0]^.reg,taicpu(hp2).oper[1]^.reg)) and
  4305. ((topsize2memsize[taicpu(hp1).opsize]<= topsize2memsize[taicpu(hp2).opsize]) or
  4306. { opsize matters for these opcodes, we could probably work around this, but it is not worth the effort }
  4307. ((taicpu(hp1).opcode<>A_SHL) and (taicpu(hp1).opcode<>A_SHR) and (taicpu(hp1).opcode<>A_SAR))
  4308. )
  4309. {$ifdef i386}
  4310. { byte registers of esi, edi, ebp, esp are not available on i386 }
  4311. and ((taicpu(hp2).opsize<>S_B) or not(getsupreg(taicpu(hp1).oper[0]^.reg) in [RS_ESI,RS_EDI,RS_EBP,RS_ESP]))
  4312. and ((taicpu(hp2).opsize<>S_B) or not(getsupreg(taicpu(p).oper[0]^.reg) in [RS_ESI,RS_EDI,RS_EBP,RS_ESP]))
  4313. {$endif i386}
  4314. then
  4315. { change movsX/movzX reg/ref, reg2
  4316. add/sub/or/... regX/$const, reg2
  4317. mov reg2, reg3
  4318. dealloc reg2
  4319. to
  4320. movsX/movzX reg/ref, reg3
  4321. add/sub/or/... reg3/$const, reg3
  4322. }
  4323. begin
  4324. TransferUsedRegs(TmpUsedRegs);
  4325. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  4326. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  4327. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  4328. begin
  4329. { by example:
  4330. movswl %si,%eax movswl %si,%eax p
  4331. decl %eax addl %edx,%eax hp1
  4332. movw %ax,%si movw %ax,%si hp2
  4333. ->
  4334. movswl %si,%eax movswl %si,%eax p
  4335. decw %eax addw %edx,%eax hp1
  4336. movw %ax,%si movw %ax,%si hp2
  4337. }
  4338. DebugMsg(SPeepholeOptimization + 'MovOpMov2MovOp ('+
  4339. debug_op2str(taicpu(p).opcode)+debug_opsize2str(taicpu(p).opsize)+' '+
  4340. debug_op2str(taicpu(hp1).opcode)+debug_opsize2str(taicpu(hp1).opsize)+' '+
  4341. debug_op2str(taicpu(hp2).opcode)+debug_opsize2str(taicpu(hp2).opsize)+')',p);
  4342. { limit size of constants as well to avoid assembler errors, but
  4343. check opsize to avoid overflow when left shifting the 1 }
  4344. if (taicpu(p).oper[0]^.typ=top_const) and (topsize2memsize[taicpu(hp2).opsize]<=63) then
  4345. taicpu(p).oper[0]^.val:=taicpu(p).oper[0]^.val and ((qword(1) shl topsize2memsize[taicpu(hp2).opsize])-1);
  4346. {$ifdef x86_64}
  4347. { Be careful of, for example:
  4348. movl %reg1,%reg2
  4349. addl %reg3,%reg2
  4350. movq %reg2,%reg4
  4351. This will cause problems if the upper 32-bits of %reg3 or %reg4 are non-zero
  4352. }
  4353. if (taicpu(hp1).opsize = S_L) and (taicpu(hp2).opsize = S_Q) then
  4354. begin
  4355. taicpu(hp2).changeopsize(S_L);
  4356. setsubreg(taicpu(hp2).oper[0]^.reg, R_SUBD);
  4357. setsubreg(taicpu(hp2).oper[1]^.reg, R_SUBD);
  4358. end;
  4359. {$endif x86_64}
  4360. taicpu(hp1).changeopsize(taicpu(hp2).opsize);
  4361. taicpu(p).changeopsize(taicpu(hp2).opsize);
  4362. if taicpu(p).oper[0]^.typ=top_reg then
  4363. setsubreg(taicpu(p).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4364. taicpu(p).loadoper(1, taicpu(hp2).oper[1]^);
  4365. AllocRegBetween(taicpu(p).oper[1]^.reg,p,hp1,usedregs);
  4366. {
  4367. ->
  4368. movswl %si,%eax movswl %si,%eax p
  4369. decw %si addw %dx,%si hp1
  4370. movw %ax,%si movw %ax,%si hp2
  4371. }
  4372. case taicpu(hp1).ops of
  4373. 1:
  4374. begin
  4375. taicpu(hp1).loadoper(0, taicpu(hp2).oper[1]^);
  4376. if taicpu(hp1).oper[0]^.typ=top_reg then
  4377. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4378. end;
  4379. 2:
  4380. begin
  4381. taicpu(hp1).loadoper(1, taicpu(hp2).oper[1]^);
  4382. if (taicpu(hp1).oper[0]^.typ=top_reg) and
  4383. (taicpu(hp1).opcode<>A_SHL) and
  4384. (taicpu(hp1).opcode<>A_SHR) and
  4385. (taicpu(hp1).opcode<>A_SAR) then
  4386. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4387. end;
  4388. else
  4389. internalerror(2018111801);
  4390. end;
  4391. {
  4392. ->
  4393. decw %si addw %dx,%si p
  4394. }
  4395. RemoveInstruction(hp2);
  4396. end;
  4397. end;
  4398. end;
  4399. if MatchInstruction(hp1,A_BTS,A_BTR,[Taicpu(p).opsize]) and
  4400. GetNextInstruction(hp1, hp2) and
  4401. MatchInstruction(hp2,A_OR,[Taicpu(p).opsize]) and
  4402. MatchOperand(Taicpu(p).oper[0]^,0) and
  4403. (Taicpu(p).oper[1]^.typ = top_reg) and
  4404. MatchOperand(Taicpu(p).oper[1]^,Taicpu(hp1).oper[1]^) and
  4405. MatchOperand(Taicpu(p).oper[1]^,Taicpu(hp2).oper[1]^) then
  4406. { mov reg1,0
  4407. bts reg1,operand1 --> mov reg1,operand2
  4408. or reg1,operand2 bts reg1,operand1}
  4409. begin
  4410. Taicpu(hp2).opcode:=A_MOV;
  4411. DebugMsg(SPeepholeOptimization + 'MovBtsOr2MovBts done',hp1);
  4412. asml.remove(hp1);
  4413. insertllitem(hp2,hp2.next,hp1);
  4414. RemoveCurrentp(p, hp1);
  4415. Result:=true;
  4416. exit;
  4417. end;
  4418. if MatchInstruction(hp1,A_SUB,[Taicpu(p).opsize]) and
  4419. MatchOperand(Taicpu(p).oper[1]^,Taicpu(hp1).oper[1]^) and
  4420. GetNextInstruction(hp1, hp2) and
  4421. MatchInstruction(hp2,A_CMP,[Taicpu(p).opsize]) and
  4422. MatchOperand(Taicpu(p).oper[0]^,Taicpu(hp2).oper[1]^) and
  4423. MatchOperand(Taicpu(hp1).oper[0]^,Taicpu(hp2).oper[0]^) then
  4424. { change
  4425. mov reg1,reg2
  4426. sub reg3,reg2
  4427. cmp reg3,reg1
  4428. into
  4429. mov reg1,reg2
  4430. sub reg3,reg2
  4431. }
  4432. begin
  4433. DebugMsg(SPeepholeOptimization + 'MovSubCmp2MovSub done',p);
  4434. RemoveInstruction(hp2);
  4435. Result:=true;
  4436. exit;
  4437. end;
  4438. {
  4439. mov ref,reg0
  4440. <op> reg0,reg1
  4441. dealloc reg0
  4442. to
  4443. <op> ref,reg1
  4444. }
  4445. if MatchOpType(taicpu(p),top_ref,top_reg) and
  4446. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  4447. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  4448. MatchInstruction(hp1,[A_AND,A_OR,A_XOR,A_ADD,A_SUB,A_CMP],[Taicpu(p).opsize]) and
  4449. not(MatchOperand(taicpu(hp1).oper[0]^,taicpu(hp1).oper[1]^)) and
  4450. RegEndOfLife(taicpu(p).oper[1]^.reg,taicpu(hp1)) then
  4451. begin
  4452. taicpu(hp1).loadoper(0,taicpu(p).oper[0]^);
  4453. DebugMsg(SPeepholeOptimization + 'MovOp2Op done',hp1);
  4454. RemoveCurrentp(p, hp1);
  4455. Result:=true;
  4456. exit;
  4457. end;
  4458. if (taicpu(p).oper[0]^.typ = top_ref) and { Second operand will be a register }
  4459. MatchInstruction(hp1, A_SHR, A_SAR, [taicpu(p).opsize]) and
  4460. MatchOpType(taicpu(hp1), top_const, top_reg) and
  4461. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  4462. begin
  4463. RegName1 := debug_regname(taicpu(hp1).oper[1]^.reg);
  4464. {$ifdef x86_64}
  4465. { Convert:
  4466. movq x(ref),%reg64
  4467. shrq y,%reg64
  4468. To:
  4469. movl x+4(ref),%reg32
  4470. shrl y-32,%reg32 (Remove if y = 32)
  4471. }
  4472. if (taicpu(p).opsize = S_Q) and
  4473. (taicpu(hp1).opcode = A_SHR) and
  4474. (taicpu(hp1).oper[0]^.val >= 32) then
  4475. begin
  4476. PreMessage := 'movq ' + debug_operstr(taicpu(p).oper[0]^) + ',' + RegName1 + '; ' +
  4477. 'shrq $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + RegName1 + ' -> movl ';
  4478. { Convert to 32-bit }
  4479. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  4480. taicpu(p).opsize := S_L;
  4481. Inc(taicpu(p).oper[0]^.ref^.offset, 4);
  4482. PreMessage := PreMessage + debug_operstr(taicpu(p).oper[0]^) + ',' + debug_regname(taicpu(p).oper[1]^.reg);
  4483. if (taicpu(hp1).oper[0]^.val = 32) then
  4484. begin
  4485. DebugMsg(SPeepholeOptimization + PreMessage + ' (MovShr2Mov)', p);
  4486. RemoveInstruction(hp1);
  4487. end
  4488. else
  4489. begin
  4490. { This will potentially open up more arithmetic operations since
  4491. the peephole optimizer now has a big hint that only the lower
  4492. 32 bits are currently in use (and opcodes are smaller in size) }
  4493. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  4494. taicpu(hp1).opsize := S_L;
  4495. Dec(taicpu(hp1).oper[0]^.val, 32);
  4496. DebugMsg(SPeepholeOptimization + PreMessage +
  4497. '; shrl $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' (MovShr2MovShr)', p);
  4498. end;
  4499. Result := True;
  4500. Exit;
  4501. end;
  4502. {$endif x86_64}
  4503. { Convert:
  4504. movl x(ref),%reg
  4505. shrl $24,%reg
  4506. To:
  4507. movzbl x+3(ref),%reg
  4508. Do similar things for movl; shrl $16 -> movzwl and movw; shrw $8 -> movzbw
  4509. Also accept sar instead of shr, but convert to movsx instead of movzx
  4510. }
  4511. if taicpu(hp1).opcode = A_SHR then
  4512. MovUnaligned := A_MOVZX
  4513. else
  4514. MovUnaligned := A_MOVSX;
  4515. NewSize := S_NO;
  4516. NewOffset := 0;
  4517. case taicpu(p).opsize of
  4518. S_B:
  4519. { No valid combinations };
  4520. S_W:
  4521. if (taicpu(hp1).oper[0]^.val = 8) then
  4522. begin
  4523. NewSize := S_BW;
  4524. NewOffset := 1;
  4525. end;
  4526. S_L:
  4527. case taicpu(hp1).oper[0]^.val of
  4528. 16:
  4529. begin
  4530. NewSize := S_WL;
  4531. NewOffset := 2;
  4532. end;
  4533. 24:
  4534. begin
  4535. NewSize := S_BL;
  4536. NewOffset := 3;
  4537. end;
  4538. else
  4539. ;
  4540. end;
  4541. {$ifdef x86_64}
  4542. S_Q:
  4543. case taicpu(hp1).oper[0]^.val of
  4544. 32:
  4545. begin
  4546. if taicpu(hp1).opcode = A_SAR then
  4547. begin
  4548. { 32-bit to 64-bit is a distinct instruction }
  4549. MovUnaligned := A_MOVSXD;
  4550. NewSize := S_LQ;
  4551. NewOffset := 4;
  4552. end
  4553. else
  4554. { Should have been handled by MovShr2Mov above }
  4555. InternalError(2022081811);
  4556. end;
  4557. 48:
  4558. begin
  4559. NewSize := S_WQ;
  4560. NewOffset := 6;
  4561. end;
  4562. 56:
  4563. begin
  4564. NewSize := S_BQ;
  4565. NewOffset := 7;
  4566. end;
  4567. else
  4568. ;
  4569. end;
  4570. {$endif x86_64}
  4571. else
  4572. InternalError(2022081810);
  4573. end;
  4574. if (NewSize <> S_NO) and
  4575. (taicpu(p).oper[0]^.ref^.offset <= $7FFFFFFF - NewOffset) then
  4576. begin
  4577. PreMessage := 'mov' + debug_opsize2str(taicpu(p).opsize) + ' ' + debug_operstr(taicpu(p).oper[0]^) + ',' + RegName1 + '; ' +
  4578. 'shr' + debug_opsize2str(taicpu(p).opsize) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + RegName1 + ' -> ' +
  4579. debug_op2str(MovUnaligned);
  4580. {$ifdef x86_64}
  4581. if MovUnaligned <> A_MOVSXD then
  4582. { Don't add size suffix for MOVSXD }
  4583. {$endif x86_64}
  4584. PreMessage := PreMessage + debug_opsize2str(NewSize);
  4585. Inc(taicpu(p).oper[0]^.ref^.offset, NewOffset);
  4586. taicpu(p).opcode := MovUnaligned;
  4587. taicpu(p).opsize := NewSize;
  4588. DebugMsg(SPeepholeOptimization + PreMessage + ' ' +
  4589. debug_operstr(taicpu(p).oper[0]^) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' (MovShr/Sar2Movx)', p);
  4590. RemoveInstruction(hp1);
  4591. Result := True;
  4592. Exit;
  4593. end;
  4594. end;
  4595. { Backward optimisation shared with OptPass2MOV }
  4596. if FuncMov2Func(p, hp1) then
  4597. begin
  4598. Result := True;
  4599. Exit;
  4600. end;
  4601. end;
  4602. function TX86AsmOptimizer.OptPass1MOVXX(var p : tai) : boolean;
  4603. var
  4604. hp1 : tai;
  4605. begin
  4606. Result:=false;
  4607. if taicpu(p).ops <> 2 then
  4608. exit;
  4609. if (MatchOpType(taicpu(p),top_reg,top_reg) and GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg)) or
  4610. GetNextInstruction(p,hp1) then
  4611. begin
  4612. if MatchInstruction(hp1,taicpu(p).opcode,[taicpu(p).opsize]) and
  4613. (taicpu(hp1).ops = 2) then
  4614. begin
  4615. if (taicpu(hp1).oper[0]^.typ = taicpu(p).oper[1]^.typ) and
  4616. (taicpu(hp1).oper[1]^.typ = taicpu(p).oper[0]^.typ) then
  4617. { movXX reg1, mem1 or movXX mem1, reg1
  4618. movXX mem2, reg2 movXX reg2, mem2}
  4619. begin
  4620. if OpsEqual(taicpu(hp1).oper[1]^,taicpu(p).oper[0]^) then
  4621. { movXX reg1, mem1 or movXX mem1, reg1
  4622. movXX mem2, reg1 movXX reg2, mem1}
  4623. begin
  4624. if OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  4625. begin
  4626. { Removes the second statement from
  4627. movXX reg1, mem1/reg2
  4628. movXX mem1/reg2, reg1
  4629. }
  4630. if taicpu(p).oper[0]^.typ=top_reg then
  4631. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,usedregs);
  4632. { Removes the second statement from
  4633. movXX mem1/reg1, reg2
  4634. movXX reg2, mem1/reg1
  4635. }
  4636. if (taicpu(p).oper[1]^.typ=top_reg) and
  4637. not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,UsedRegs)) then
  4638. begin
  4639. DebugMsg(SPeepholeOptimization + 'MovXXMovXX2Nop 1 done',p);
  4640. RemoveInstruction(hp1);
  4641. RemoveCurrentp(p); { p will now be equal to the instruction that follows what was hp1 }
  4642. Result:=true;
  4643. exit;
  4644. end
  4645. else if (taicpu(hp1).oper[1]^.typ<>top_ref) or (not(vol_write in taicpu(hp1).oper[1]^.ref^.volatility)) and
  4646. (taicpu(hp1).oper[0]^.typ<>top_ref) or (not(vol_read in taicpu(hp1).oper[0]^.ref^.volatility)) then
  4647. begin
  4648. DebugMsg(SPeepholeOptimization + 'MovXXMovXX2MoVXX 1 done',p);
  4649. RemoveInstruction(hp1);
  4650. Result:=true;
  4651. exit;
  4652. end;
  4653. end
  4654. end;
  4655. end;
  4656. end;
  4657. end;
  4658. end;
  4659. function TX86AsmOptimizer.OptPass1OP(var p : tai) : boolean;
  4660. var
  4661. hp1 : tai;
  4662. begin
  4663. result:=false;
  4664. { replace
  4665. <Op>X %mreg1,%mreg2 // Op in [ADD,MUL]
  4666. MovX %mreg2,%mreg1
  4667. dealloc %mreg2
  4668. by
  4669. <Op>X %mreg2,%mreg1
  4670. ?
  4671. }
  4672. if GetNextInstruction(p,hp1) and
  4673. { we mix single and double opperations here because we assume that the compiler
  4674. generates vmovapd only after double operations and vmovaps only after single operations }
  4675. MatchInstruction(hp1,A_MOVAPD,A_MOVAPS,[S_NO]) and
  4676. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  4677. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[1]^) and
  4678. (taicpu(p).oper[0]^.typ=top_reg) then
  4679. begin
  4680. TransferUsedRegs(TmpUsedRegs);
  4681. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  4682. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  4683. begin
  4684. taicpu(p).loadoper(0,taicpu(hp1).oper[0]^);
  4685. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  4686. DebugMsg(SPeepholeOptimization + 'OpMov2Op done',p);
  4687. RemoveInstruction(hp1);
  4688. result:=true;
  4689. end;
  4690. end;
  4691. end;
  4692. function TX86AsmOptimizer.OptPass1Test(var p: tai) : boolean;
  4693. var
  4694. hp1, p_label, p_dist, hp1_dist: tai;
  4695. JumpLabel, JumpLabel_dist: TAsmLabel;
  4696. FirstValue, SecondValue: TCGInt;
  4697. TempBool: Boolean;
  4698. begin
  4699. Result := False;
  4700. if (taicpu(p).oper[0]^.typ = top_const) and
  4701. (taicpu(p).oper[0]^.val <> -1) then
  4702. begin
  4703. { Convert unsigned maximum constants to -1 to aid optimisation }
  4704. case taicpu(p).opsize of
  4705. S_B:
  4706. if (taicpu(p).oper[0]^.val and $FF) = $FF then
  4707. begin
  4708. taicpu(p).oper[0]^.val := -1;
  4709. Result := True;
  4710. Exit;
  4711. end;
  4712. S_W:
  4713. if (taicpu(p).oper[0]^.val and $FFFF) = $FFFF then
  4714. begin
  4715. taicpu(p).oper[0]^.val := -1;
  4716. Result := True;
  4717. Exit;
  4718. end;
  4719. S_L:
  4720. if (taicpu(p).oper[0]^.val and $FFFFFFFF) = $FFFFFFFF then
  4721. begin
  4722. taicpu(p).oper[0]^.val := -1;
  4723. Result := True;
  4724. Exit;
  4725. end;
  4726. {$ifdef x86_64}
  4727. S_Q:
  4728. { Storing anything greater than $7FFFFFFF is not possible so do
  4729. nothing };
  4730. {$endif x86_64}
  4731. else
  4732. InternalError(2021121001);
  4733. end;
  4734. end;
  4735. if GetNextInstruction(p, hp1) and
  4736. TrySwapMovCmp(p, hp1) then
  4737. begin
  4738. Result := True;
  4739. Exit;
  4740. end;
  4741. if MatchInstruction(hp1, A_Jcc, []) then
  4742. begin
  4743. TempBool := True;
  4744. if DoJumpOptimizations(hp1, TempBool) or
  4745. not TempBool then
  4746. begin
  4747. Result := True;
  4748. if Assigned(hp1) then
  4749. begin
  4750. if (hp1.typ in [ait_align]) then
  4751. SkipAligns(hp1, hp1);
  4752. { CollapseZeroDistJump will be set to the label after the
  4753. jump if it optimises, whether or not it's live or dead }
  4754. if (hp1.typ in [ait_label]) and
  4755. not (tai_label(hp1).labsym.is_used) then
  4756. GetNextInstruction(hp1, hp1);
  4757. end;
  4758. TransferUsedRegs(TmpUsedRegs);
  4759. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  4760. if not Assigned(hp1) or
  4761. (
  4762. not MatchInstruction(hp1, A_Jcc, A_SETcc, A_CMOVcc, []) and
  4763. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)
  4764. ) then
  4765. begin
  4766. { No more conditional jumps; conditional statement is no longer required }
  4767. DebugMsg(SPeepholeOptimization + 'Removed unnecessary condition (Test2Nop)', p);
  4768. RemoveCurrentP(p);
  4769. end;
  4770. Exit;
  4771. end;
  4772. end;
  4773. { Search for:
  4774. test $x,(reg/ref)
  4775. jne @lbl1
  4776. test $y,(reg/ref) (same register or reference)
  4777. jne @lbl1
  4778. Change to:
  4779. test $(x or y),(reg/ref)
  4780. jne @lbl1
  4781. (Note, this doesn't work with je instead of jne)
  4782. Also catch cases where "cmp $0,(reg/ref)" and "test %reg,%reg" are used.
  4783. Also search for:
  4784. test $x,(reg/ref)
  4785. je @lbl1
  4786. test $y,(reg/ref)
  4787. je/jne @lbl2
  4788. If (x or y) = x, then the second jump is deterministic
  4789. }
  4790. if (
  4791. (
  4792. (taicpu(p).oper[0]^.typ = top_const) or
  4793. (
  4794. { test %reg,%reg can be considered equivalent to test, -1,%reg }
  4795. (taicpu(p).oper[0]^.typ = top_reg) and
  4796. MatchOperand(taicpu(p).oper[1]^, taicpu(p).oper[0]^.reg)
  4797. )
  4798. ) and
  4799. MatchInstruction(hp1, A_JCC, [])
  4800. ) then
  4801. begin
  4802. if (taicpu(p).oper[0]^.typ = top_reg) and
  4803. MatchOperand(taicpu(p).oper[1]^, taicpu(p).oper[0]^.reg) then
  4804. FirstValue := -1
  4805. else
  4806. FirstValue := taicpu(p).oper[0]^.val;
  4807. { If we have several test/jne's in a row, it might be the case that
  4808. the second label doesn't go to the same location, but the one
  4809. after it might (e.g. test; jne @lbl1; test; jne @lbl2; test @lbl1),
  4810. so accommodate for this with a while loop.
  4811. }
  4812. hp1_dist := hp1;
  4813. if GetNextInstruction(hp1, p_dist) and
  4814. (p_dist.typ = ait_instruction) and
  4815. (
  4816. (
  4817. (taicpu(p_dist).opcode = A_TEST) and
  4818. (
  4819. (taicpu(p_dist).oper[0]^.typ = top_const) or
  4820. { test %reg,%reg can be considered equivalent to test, -1,%reg }
  4821. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p_dist).oper[0]^)
  4822. )
  4823. ) or
  4824. (
  4825. { cmp 0,%reg = test %reg,%reg }
  4826. (taicpu(p_dist).opcode = A_CMP) and
  4827. MatchOperand(taicpu(p_dist).oper[0]^, 0)
  4828. )
  4829. ) and
  4830. { Make sure the destination operands are actually the same }
  4831. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p).oper[1]^) and
  4832. GetNextInstruction(p_dist, hp1_dist) and
  4833. MatchInstruction(hp1_dist, A_JCC, []) then
  4834. begin
  4835. if
  4836. (taicpu(p_dist).opcode = A_CMP) { constant will be zero } or
  4837. (
  4838. (taicpu(p_dist).oper[0]^.typ = top_reg) and
  4839. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p_dist).oper[0]^.reg)
  4840. ) then
  4841. SecondValue := -1
  4842. else
  4843. SecondValue := taicpu(p_dist).oper[0]^.val;
  4844. { If both of the TEST constants are identical, delete the second
  4845. TEST that is unnecessary. }
  4846. if (FirstValue = SecondValue) then
  4847. begin
  4848. DebugMsg(SPeepholeOptimization + 'TEST/Jcc/TEST; removed superfluous TEST', p_dist);
  4849. RemoveInstruction(p_dist);
  4850. { Don't let the flags register become deallocated and reallocated between the jumps }
  4851. AllocRegBetween(NR_DEFAULTFLAGS, hp1, hp1_dist, UsedRegs);
  4852. Result := True;
  4853. if condition_in(taicpu(hp1_dist).condition, taicpu(hp1).condition) then
  4854. begin
  4855. { Since the second jump's condition is a subset of the first, we
  4856. know it will never branch because the first jump dominates it.
  4857. Get it out of the way now rather than wait for the jump
  4858. optimisations for a speed boost. }
  4859. if IsJumpToLabel(taicpu(hp1_dist)) then
  4860. TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol).DecRefs;
  4861. DebugMsg(SPeepholeOptimization + 'Removed dominated jump (via TEST/Jcc/TEST)', hp1_dist);
  4862. RemoveInstruction(hp1_dist);
  4863. end
  4864. else if condition_in(inverse_cond(taicpu(hp1).condition), taicpu(hp1_dist).condition) then
  4865. begin
  4866. { If the inverse of the first condition is a subset of the second,
  4867. the second one will definitely branch if the first one doesn't }
  4868. DebugMsg(SPeepholeOptimization + 'Conditional jump will always branch (via TEST/Jcc/TEST)', hp1_dist);
  4869. MakeUnconditional(taicpu(hp1_dist));
  4870. RemoveDeadCodeAfterJump(hp1_dist);
  4871. end;
  4872. Exit;
  4873. end;
  4874. if (taicpu(hp1).condition in [C_NE, C_NZ]) and
  4875. (taicpu(hp1_dist).condition in [C_NE, C_NZ]) and
  4876. { If the first instruction is test %reg,%reg or test $-1,%reg,
  4877. then the second jump will never branch, so it can also be
  4878. removed regardless of where it goes }
  4879. (
  4880. (FirstValue = -1) or
  4881. (SecondValue = -1) or
  4882. MatchOperand(taicpu(hp1_dist).oper[0]^, taicpu(hp1).oper[0]^)
  4883. ) then
  4884. begin
  4885. { Same jump location... can be a register since nothing's changed }
  4886. { If any of the entries are equivalent to test %reg,%reg, then the
  4887. merged $(x or y) is also test %reg,%reg / test $-1,%reg }
  4888. taicpu(p).loadconst(0, FirstValue or SecondValue);
  4889. if IsJumpToLabel(taicpu(hp1_dist)) then
  4890. TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol).DecRefs;
  4891. DebugMsg(SPeepholeOptimization + 'TEST/JNE/TEST/JNE merged', p);
  4892. RemoveInstruction(hp1_dist);
  4893. { Only remove the second test if no jumps or other conditional instructions follow }
  4894. TransferUsedRegs(TmpUsedRegs);
  4895. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  4896. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  4897. if not RegUsedAfterInstruction(NR_DEFAULTFLAGS, p_dist, TmpUsedRegs) then
  4898. RemoveInstruction(p_dist);
  4899. Result := True;
  4900. Exit;
  4901. end;
  4902. end;
  4903. end;
  4904. { Search for:
  4905. test %reg,%reg
  4906. j(c1) @lbl1
  4907. ...
  4908. @lbl:
  4909. test %reg,%reg (same register)
  4910. j(c2) @lbl2
  4911. If c2 is a subset of c1, change to:
  4912. test %reg,%reg
  4913. j(c1) @lbl2
  4914. (@lbl1 may become a dead label as a result)
  4915. }
  4916. if (taicpu(p).oper[1]^.typ = top_reg) and
  4917. (taicpu(p).oper[0]^.typ = top_reg) and
  4918. (taicpu(p).oper[0]^.reg = taicpu(p).oper[1]^.reg) and
  4919. MatchInstruction(hp1, A_JCC, []) and
  4920. IsJumpToLabel(taicpu(hp1)) then
  4921. begin
  4922. JumpLabel := TAsmLabel(taicpu(hp1).oper[0]^.ref^.symbol);
  4923. p_label := nil;
  4924. if Assigned(JumpLabel) then
  4925. p_label := getlabelwithsym(JumpLabel);
  4926. if Assigned(p_label) and
  4927. GetNextInstruction(p_label, p_dist) and
  4928. MatchInstruction(p_dist, A_TEST, []) and
  4929. { It's fine if the second test uses smaller sub-registers }
  4930. (taicpu(p_dist).opsize <= taicpu(p).opsize) and
  4931. MatchOpType(taicpu(p_dist), top_reg, top_reg) and
  4932. SuperRegistersEqual(taicpu(p_dist).oper[0]^.reg, taicpu(p).oper[0]^.reg) and
  4933. SuperRegistersEqual(taicpu(p_dist).oper[1]^.reg, taicpu(p).oper[1]^.reg) and
  4934. GetNextInstruction(p_dist, hp1_dist) and
  4935. MatchInstruction(hp1_dist, A_JCC, []) then { This doesn't have to be an explicit label }
  4936. begin
  4937. JumpLabel_dist := TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol);
  4938. if JumpLabel = JumpLabel_dist then
  4939. { This is an infinite loop }
  4940. Exit;
  4941. { Best optimisation when the first condition is a subset (or equal) of the second }
  4942. if condition_in(taicpu(hp1).condition, taicpu(hp1_dist).condition) then
  4943. begin
  4944. { Any registers used here will already be allocated }
  4945. if Assigned(JumpLabel) then
  4946. JumpLabel.DecRefs;
  4947. DebugMsg(SPeepholeOptimization + 'TEST/Jcc/@Lbl/TEST/Jcc -> TEST/Jcc, redirecting first jump', hp1);
  4948. taicpu(hp1).loadref(0, taicpu(hp1_dist).oper[0]^.ref^); { This also increases the reference count }
  4949. Result := True;
  4950. Exit;
  4951. end;
  4952. end;
  4953. end;
  4954. end;
  4955. function TX86AsmOptimizer.OptPass1Add(var p : tai) : boolean;
  4956. var
  4957. hp1, hp2: tai;
  4958. ActiveReg: TRegister;
  4959. OldOffset: asizeint;
  4960. ThisConst: TCGInt;
  4961. function RegDeallocated: Boolean;
  4962. begin
  4963. TransferUsedRegs(TmpUsedRegs);
  4964. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  4965. Result := not(RegUsedAfterInstruction(ActiveReg,hp1,TmpUsedRegs))
  4966. end;
  4967. begin
  4968. result:=false;
  4969. hp1 := nil;
  4970. { replace
  4971. addX const,%reg1
  4972. leaX (%reg1,%reg1,Y),%reg2 // Base or index might not be equal to reg1
  4973. dealloc %reg1
  4974. by
  4975. leaX const+const*Y(%reg1,%reg1,Y),%reg2
  4976. }
  4977. if MatchOpType(taicpu(p),top_const,top_reg) then
  4978. begin
  4979. ActiveReg := taicpu(p).oper[1]^.reg;
  4980. { Ensures the entire register was updated }
  4981. if (taicpu(p).opsize >= S_L) and
  4982. GetNextInstructionUsingReg(p,hp1, ActiveReg) and
  4983. MatchInstruction(hp1,A_LEA,[]) and
  4984. (SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.base) or
  4985. SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.index)) and
  4986. (
  4987. { Cover the case where the register in the reference is also the destination register }
  4988. Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, ActiveReg) or
  4989. (
  4990. { Try to avoid the expensive check of RegUsedAfterInstruction if we know it will return False }
  4991. not SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ActiveReg) and
  4992. RegDeallocated
  4993. )
  4994. ) then
  4995. begin
  4996. OldOffset := taicpu(hp1).oper[0]^.ref^.offset;
  4997. {$push}
  4998. {$R-}{$Q-}
  4999. { Explicitly disable overflow checking for these offset calculation
  5000. as those do not matter for the final result }
  5001. if ActiveReg=taicpu(hp1).oper[0]^.ref^.base then
  5002. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val);
  5003. if ActiveReg=taicpu(hp1).oper[0]^.ref^.index then
  5004. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val*max(taicpu(hp1).oper[0]^.ref^.scalefactor,1));
  5005. {$pop}
  5006. {$ifdef x86_64}
  5007. if (taicpu(hp1).oper[0]^.ref^.offset > $7FFFFFFF) or (taicpu(hp1).oper[0]^.ref^.offset < -2147483648) then
  5008. begin
  5009. { Overflow; abort }
  5010. taicpu(hp1).oper[0]^.ref^.offset := OldOffset;
  5011. end
  5012. else
  5013. {$endif x86_64}
  5014. begin
  5015. DebugMsg(SPeepholeOptimization + 'AddLea2Lea done',p);
  5016. if not (cs_opt_level3 in current_settings.optimizerswitches) then
  5017. { hp1 is the immediate next instruction for sure - good for a quick speed boost }
  5018. RemoveCurrentP(p, hp1)
  5019. else
  5020. RemoveCurrentP(p);
  5021. result:=true;
  5022. Exit;
  5023. end;
  5024. end;
  5025. if (
  5026. { Save calling GetNextInstructionUsingReg again }
  5027. Assigned(hp1) or
  5028. GetNextInstructionUsingReg(p,hp1, ActiveReg)
  5029. ) and
  5030. MatchInstruction(hp1,A_ADD,A_SUB,[taicpu(p).opsize]) and
  5031. (taicpu(hp1).oper[1]^.reg = ActiveReg) then
  5032. begin
  5033. if taicpu(hp1).oper[0]^.typ = top_const then
  5034. begin
  5035. { Merge add const1,%reg; add/sub const2,%reg to add const1+/-const2,%reg }
  5036. if taicpu(hp1).opcode = A_ADD then
  5037. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val
  5038. else
  5039. ThisConst := taicpu(p).oper[0]^.val - taicpu(hp1).oper[0]^.val;
  5040. Result := True;
  5041. { Handle any overflows }
  5042. case taicpu(p).opsize of
  5043. S_B:
  5044. taicpu(p).oper[0]^.val := ThisConst and $FF;
  5045. S_W:
  5046. taicpu(p).oper[0]^.val := ThisConst and $FFFF;
  5047. S_L:
  5048. taicpu(p).oper[0]^.val := ThisConst and $FFFFFFFF;
  5049. {$ifdef x86_64}
  5050. S_Q:
  5051. if (ThisConst > $7FFFFFFF) or (ThisConst < -2147483648) then
  5052. { Overflow; abort }
  5053. Result := False
  5054. else
  5055. taicpu(p).oper[0]^.val := ThisConst;
  5056. {$endif x86_64}
  5057. else
  5058. InternalError(2021102610);
  5059. end;
  5060. { Result may get set to False again if the combined immediate overflows for S_Q sizes }
  5061. if Result then
  5062. begin
  5063. if (taicpu(p).oper[0]^.val < 0) and
  5064. (
  5065. ((taicpu(p).opsize = S_B) and (taicpu(p).oper[0]^.val <> -128)) or
  5066. ((taicpu(p).opsize = S_W) and (taicpu(p).oper[0]^.val <> -32768)) or
  5067. ((taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) and (taicpu(p).oper[0]^.val <> -2147483648))
  5068. ) then
  5069. begin
  5070. DebugMsg(SPeepholeOptimization + 'ADD; ADD/SUB -> SUB',p);
  5071. taicpu(p).opcode := A_SUB;
  5072. taicpu(p).oper[0]^.val := -taicpu(p).oper[0]^.val;
  5073. end
  5074. else
  5075. DebugMsg(SPeepholeOptimization + 'ADD; ADD/SUB -> ADD',p);
  5076. RemoveInstruction(hp1);
  5077. end;
  5078. end
  5079. else
  5080. begin
  5081. { Make doubly sure the flags aren't in use because the order of additions may affect them }
  5082. TransferUsedRegs(TmpUsedRegs);
  5083. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5084. hp2 := p;
  5085. while not (cs_opt_level3 in current_settings.optimizerswitches) and
  5086. GetNextInstruction(hp2, hp2) and (hp2 <> hp1) do
  5087. UpdateUsedRegs(TmpUsedRegs, tai(hp2.next));
  5088. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  5089. begin
  5090. { Move the constant addition to after the reg/ref addition to improve optimisation }
  5091. DebugMsg(SPeepholeOptimization + 'Add/sub swap 1a done',p);
  5092. Asml.Remove(p);
  5093. Asml.InsertAfter(p, hp1);
  5094. p := hp1;
  5095. Result := True;
  5096. Exit;
  5097. end;
  5098. end;
  5099. end;
  5100. if DoArithCombineOpt(p) then
  5101. Result:=true;
  5102. end;
  5103. end;
  5104. function TX86AsmOptimizer.OptPass1LEA(var p : tai) : boolean;
  5105. var
  5106. hp1: tai;
  5107. ref: Integer;
  5108. saveref: treference;
  5109. Multiple: TCGInt;
  5110. Adjacent: Boolean;
  5111. begin
  5112. Result:=false;
  5113. { play save and throw an error if LEA uses a seg register prefix,
  5114. this is most likely an error somewhere else }
  5115. if taicpu(p).oper[0]^.ref^.Segment<>NR_NO then
  5116. internalerror(2022022001);
  5117. { changes "lea (%reg1), %reg2" into "mov %reg1, %reg2" }
  5118. if (taicpu(p).oper[0]^.ref^.base <> NR_NO) and
  5119. (taicpu(p).oper[0]^.ref^.index = NR_NO) and
  5120. (
  5121. { do not mess with leas accessing the stack pointer
  5122. unless it's a null operation }
  5123. (taicpu(p).oper[1]^.reg <> NR_STACK_POINTER_REG) or
  5124. (
  5125. (taicpu(p).oper[0]^.ref^.base = NR_STACK_POINTER_REG) and
  5126. (taicpu(p).oper[0]^.ref^.offset = 0)
  5127. )
  5128. ) and
  5129. (not(Assigned(taicpu(p).oper[0]^.ref^.Symbol))) then
  5130. begin
  5131. if (taicpu(p).oper[0]^.ref^.offset = 0) then
  5132. begin
  5133. if (taicpu(p).oper[0]^.ref^.base <> taicpu(p).oper[1]^.reg) then
  5134. begin
  5135. taicpu(p).opcode := A_MOV;
  5136. taicpu(p).loadreg(0, taicpu(p).oper[0]^.ref^.base);
  5137. DebugMsg(SPeepholeOptimization + 'Lea2Mov done',p);
  5138. end
  5139. else
  5140. begin
  5141. DebugMsg(SPeepholeOptimization + 'Lea2Nop done',p);
  5142. RemoveCurrentP(p);
  5143. end;
  5144. Result:=true;
  5145. exit;
  5146. end
  5147. else if (
  5148. { continue to use lea to adjust the stack pointer,
  5149. it is the recommended way, but only if not optimizing for size }
  5150. (taicpu(p).oper[1]^.reg<>NR_STACK_POINTER_REG) or
  5151. (cs_opt_size in current_settings.optimizerswitches)
  5152. ) and
  5153. { If the flags register is in use, don't change the instruction
  5154. to an ADD otherwise this will scramble the flags. [Kit] }
  5155. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  5156. ConvertLEA(taicpu(p)) then
  5157. begin
  5158. Result:=true;
  5159. exit;
  5160. end;
  5161. end;
  5162. { Don't optimise if the stack or frame pointer is the destination register }
  5163. if (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG) or (taicpu(p).oper[1]^.reg=current_procinfo.framepointer) then
  5164. Exit;
  5165. if GetNextInstruction(p,hp1) and
  5166. (hp1.typ=ait_instruction) then
  5167. begin
  5168. if MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) and
  5169. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  5170. MatchOpType(Taicpu(hp1),top_reg,top_reg) then
  5171. begin
  5172. TransferUsedRegs(TmpUsedRegs);
  5173. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5174. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  5175. begin
  5176. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  5177. DebugMsg(SPeepholeOptimization + 'LeaMov2Lea done',p);
  5178. RemoveInstruction(hp1);
  5179. result:=true;
  5180. exit;
  5181. end;
  5182. end;
  5183. { changes
  5184. lea <ref1>, reg1
  5185. <op> ...,<ref. with reg1>,...
  5186. to
  5187. <op> ...,<ref1>,... }
  5188. { find a reference which uses reg1 }
  5189. if (taicpu(hp1).ops>=1) and (taicpu(hp1).oper[0]^.typ=top_ref) and RegInOp(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^) then
  5190. ref:=0
  5191. else if (taicpu(hp1).ops>=2) and (taicpu(hp1).oper[1]^.typ=top_ref) and RegInOp(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^) then
  5192. ref:=1
  5193. else
  5194. ref:=-1;
  5195. if (ref<>-1) and
  5196. { reg1 must be either the base or the index }
  5197. ((taicpu(hp1).oper[ref]^.ref^.base=taicpu(p).oper[1]^.reg) xor (taicpu(hp1).oper[ref]^.ref^.index=taicpu(p).oper[1]^.reg)) then
  5198. begin
  5199. { reg1 can be removed from the reference }
  5200. saveref:=taicpu(hp1).oper[ref]^.ref^;
  5201. if taicpu(hp1).oper[ref]^.ref^.base=taicpu(p).oper[1]^.reg then
  5202. taicpu(hp1).oper[ref]^.ref^.base:=NR_NO
  5203. else if taicpu(hp1).oper[ref]^.ref^.index=taicpu(p).oper[1]^.reg then
  5204. taicpu(hp1).oper[ref]^.ref^.index:=NR_NO
  5205. else
  5206. Internalerror(2019111201);
  5207. { check if the can insert all data of the lea into the second instruction }
  5208. if ((taicpu(hp1).oper[ref]^.ref^.base=taicpu(p).oper[1]^.reg) or (taicpu(hp1).oper[ref]^.ref^.scalefactor <= 1)) and
  5209. ((taicpu(p).oper[0]^.ref^.base=NR_NO) or (taicpu(hp1).oper[ref]^.ref^.base=NR_NO)) and
  5210. ((taicpu(p).oper[0]^.ref^.index=NR_NO) or (taicpu(hp1).oper[ref]^.ref^.index=NR_NO)) and
  5211. ((taicpu(p).oper[0]^.ref^.symbol=nil) or (taicpu(hp1).oper[ref]^.ref^.symbol=nil)) and
  5212. ((taicpu(p).oper[0]^.ref^.relsymbol=nil) or (taicpu(hp1).oper[ref]^.ref^.relsymbol=nil)) and
  5213. ((taicpu(p).oper[0]^.ref^.scalefactor <= 1) or (taicpu(hp1).oper[ref]^.ref^.scalefactor <= 1)) and
  5214. (taicpu(p).oper[0]^.ref^.segment=NR_NO) and (taicpu(hp1).oper[ref]^.ref^.segment=NR_NO)
  5215. {$ifdef x86_64}
  5216. and (abs(taicpu(hp1).oper[ref]^.ref^.offset+taicpu(p).oper[0]^.ref^.offset)<=$7fffffff)
  5217. and (((taicpu(p).oper[0]^.ref^.base<>NR_RIP) and (taicpu(p).oper[0]^.ref^.index<>NR_RIP)) or
  5218. ((taicpu(hp1).oper[ref]^.ref^.base=NR_NO) and (taicpu(hp1).oper[ref]^.ref^.index=NR_NO))
  5219. )
  5220. {$endif x86_64}
  5221. then
  5222. begin
  5223. { reg1 might not used by the second instruction after it is remove from the reference }
  5224. if not(RegInInstruction(taicpu(p).oper[1]^.reg,taicpu(hp1))) then
  5225. begin
  5226. TransferUsedRegs(TmpUsedRegs);
  5227. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5228. { reg1 is not updated so it might not be used afterwards }
  5229. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  5230. begin
  5231. DebugMsg(SPeepholeOptimization + 'LeaOp2Op done',p);
  5232. if taicpu(p).oper[0]^.ref^.base<>NR_NO then
  5233. taicpu(hp1).oper[ref]^.ref^.base:=taicpu(p).oper[0]^.ref^.base;
  5234. if taicpu(p).oper[0]^.ref^.index<>NR_NO then
  5235. taicpu(hp1).oper[ref]^.ref^.index:=taicpu(p).oper[0]^.ref^.index;
  5236. if taicpu(p).oper[0]^.ref^.symbol<>nil then
  5237. taicpu(hp1).oper[ref]^.ref^.symbol:=taicpu(p).oper[0]^.ref^.symbol;
  5238. if taicpu(p).oper[0]^.ref^.relsymbol<>nil then
  5239. taicpu(hp1).oper[ref]^.ref^.relsymbol:=taicpu(p).oper[0]^.ref^.relsymbol;
  5240. if taicpu(p).oper[0]^.ref^.scalefactor > 1 then
  5241. taicpu(hp1).oper[ref]^.ref^.scalefactor:=taicpu(p).oper[0]^.ref^.scalefactor;
  5242. inc(taicpu(hp1).oper[ref]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset);
  5243. RemoveCurrentP(p, hp1);
  5244. result:=true;
  5245. exit;
  5246. end
  5247. end;
  5248. end;
  5249. { recover }
  5250. taicpu(hp1).oper[ref]^.ref^:=saveref;
  5251. end;
  5252. Adjacent := RegInInstruction(taicpu(p).oper[1]^.reg, hp1);
  5253. if Adjacent or
  5254. { Check further ahead (up to 2 instructions ahead for -O2) }
  5255. GetNextInstructionUsingReg(hp1,hp1,taicpu(p).oper[1]^.reg) then
  5256. begin
  5257. { Check common LEA/LEA conditions }
  5258. if MatchInstruction(hp1,A_LEA,[taicpu(p).opsize]) and
  5259. (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg) and
  5260. (taicpu(p).oper[0]^.ref^.relsymbol = nil) and
  5261. (taicpu(p).oper[0]^.ref^.segment = NR_NO) and
  5262. (taicpu(p).oper[0]^.ref^.symbol = nil) and
  5263. (taicpu(hp1).oper[0]^.ref^.relsymbol = nil) and
  5264. (taicpu(hp1).oper[0]^.ref^.segment = NR_NO) and
  5265. (taicpu(hp1).oper[0]^.ref^.symbol = nil) and
  5266. (
  5267. { If p and hp1 are adjacent, RegModifiedBetween always returns False, so avoid
  5268. calling it (since it calls GetNextInstruction) }
  5269. Adjacent or
  5270. (
  5271. (
  5272. (taicpu(p).oper[0]^.ref^.base = NR_NO) or { Don't call RegModifiedBetween unnecessarily }
  5273. not(RegModifiedBetween(taicpu(p).oper[0]^.ref^.base,p,hp1))
  5274. ) and (
  5275. (taicpu(p).oper[0]^.ref^.index = taicpu(p).oper[0]^.ref^.base) or { Don't call RegModifiedBetween unnecessarily }
  5276. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  5277. not(RegModifiedBetween(taicpu(p).oper[0]^.ref^.index,p,hp1))
  5278. )
  5279. )
  5280. ) then
  5281. begin
  5282. { changes
  5283. lea (regX,scale), reg1
  5284. lea offset(reg1,reg1), reg1
  5285. to
  5286. lea offset(regX,scale*2), reg1
  5287. and
  5288. lea (regX,scale1), reg1
  5289. lea offset(reg1,scale2), reg1
  5290. to
  5291. lea offset(regX,scale1*scale2), reg1
  5292. ... so long as the final scale does not exceed 8
  5293. (Similarly, allow the first instruction to be "lea (regX,regX),reg1")
  5294. }
  5295. if (taicpu(p).oper[0]^.ref^.base<>NR_STACK_POINTER_REG) and { lea (%rsp,scale),reg is not a valid encoding }
  5296. (taicpu(p).oper[0]^.ref^.offset = 0) and
  5297. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  5298. (
  5299. (
  5300. (taicpu(p).oper[0]^.ref^.base = NR_NO)
  5301. ) or (
  5302. (taicpu(p).oper[0]^.ref^.scalefactor <= 1) and
  5303. (
  5304. (taicpu(p).oper[0]^.ref^.base = taicpu(p).oper[0]^.ref^.index) and
  5305. (
  5306. { RegUsedBetween always returns False if p and hp1 are adjacent }
  5307. Adjacent or
  5308. not(RegUsedBetween(taicpu(p).oper[0]^.ref^.index, p, hp1))
  5309. )
  5310. )
  5311. )
  5312. ) and (
  5313. (
  5314. { lea (reg1,scale2), reg1 variant }
  5315. (taicpu(hp1).oper[0]^.ref^.base = NR_NO) and
  5316. (
  5317. (
  5318. (taicpu(p).oper[0]^.ref^.base = NR_NO) and
  5319. (taicpu(hp1).oper[0]^.ref^.scalefactor * taicpu(p).oper[0]^.ref^.scalefactor <= 8)
  5320. ) or (
  5321. { lea (regX,regX), reg1 variant }
  5322. (taicpu(p).oper[0]^.ref^.base <> NR_NO) and
  5323. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 4)
  5324. )
  5325. )
  5326. ) or (
  5327. { lea (reg1,reg1), reg1 variant }
  5328. (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) and
  5329. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 1)
  5330. )
  5331. ) then
  5332. begin
  5333. DebugMsg(SPeepholeOptimization + 'LeaLea2Lea 2 done',p);
  5334. { Make everything homogeneous to make calculations easier }
  5335. if (taicpu(p).oper[0]^.ref^.base <> NR_NO) then
  5336. begin
  5337. if taicpu(p).oper[0]^.ref^.index <> NR_NO then
  5338. { Convert lea (regX,regX),reg1 to lea (regX,2),reg1 }
  5339. taicpu(p).oper[0]^.ref^.scalefactor := 2
  5340. else
  5341. taicpu(p).oper[0]^.ref^.index := taicpu(p).oper[0]^.ref^.base;
  5342. taicpu(p).oper[0]^.ref^.base := NR_NO;
  5343. end;
  5344. if (taicpu(hp1).oper[0]^.ref^.base = NR_NO) then
  5345. begin
  5346. { Just to prevent miscalculations }
  5347. if (taicpu(hp1).oper[0]^.ref^.scalefactor = 0) then
  5348. taicpu(hp1).oper[0]^.ref^.scalefactor := taicpu(p).oper[0]^.ref^.scalefactor
  5349. else
  5350. taicpu(hp1).oper[0]^.ref^.scalefactor := taicpu(hp1).oper[0]^.ref^.scalefactor * taicpu(p).oper[0]^.ref^.scalefactor;
  5351. end
  5352. else
  5353. begin
  5354. taicpu(hp1).oper[0]^.ref^.base := NR_NO;
  5355. taicpu(hp1).oper[0]^.ref^.scalefactor := taicpu(p).oper[0]^.ref^.scalefactor * 2;
  5356. end;
  5357. taicpu(hp1).oper[0]^.ref^.index := taicpu(p).oper[0]^.ref^.index;
  5358. RemoveCurrentP(p);
  5359. result:=true;
  5360. exit;
  5361. end
  5362. { changes
  5363. lea offset1(regX), reg1
  5364. lea offset2(reg1), reg1
  5365. to
  5366. lea offset1+offset2(regX), reg1 }
  5367. else if
  5368. (
  5369. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  5370. (taicpu(p).oper[0]^.ref^.index = NR_NO)
  5371. ) or (
  5372. (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) and
  5373. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 1) and
  5374. (
  5375. (
  5376. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  5377. (taicpu(p).oper[0]^.ref^.base = NR_NO)
  5378. ) or (
  5379. (taicpu(p).oper[0]^.ref^.scalefactor <= 1) and
  5380. (
  5381. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  5382. (
  5383. (taicpu(p).oper[0]^.ref^.index = taicpu(p).oper[0]^.ref^.base) and
  5384. (
  5385. (taicpu(hp1).oper[0]^.ref^.index = NR_NO) or
  5386. (taicpu(hp1).oper[0]^.ref^.base = NR_NO)
  5387. )
  5388. )
  5389. )
  5390. )
  5391. )
  5392. ) then
  5393. begin
  5394. DebugMsg(SPeepholeOptimization + 'LeaLea2Lea 1 done',p);
  5395. if taicpu(hp1).oper[0]^.ref^.index=taicpu(p).oper[1]^.reg then
  5396. begin
  5397. taicpu(hp1).oper[0]^.ref^.index:=taicpu(p).oper[0]^.ref^.base;
  5398. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset*max(taicpu(hp1).oper[0]^.ref^.scalefactor,1));
  5399. { if the register is used as index and base, we have to increase for base as well
  5400. and adapt base }
  5401. if taicpu(hp1).oper[0]^.ref^.base=taicpu(p).oper[1]^.reg then
  5402. begin
  5403. taicpu(hp1).oper[0]^.ref^.base:=taicpu(p).oper[0]^.ref^.base;
  5404. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset);
  5405. end;
  5406. end
  5407. else
  5408. begin
  5409. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset);
  5410. taicpu(hp1).oper[0]^.ref^.base:=taicpu(p).oper[0]^.ref^.base;
  5411. end;
  5412. if taicpu(p).oper[0]^.ref^.index<>NR_NO then
  5413. begin
  5414. taicpu(hp1).oper[0]^.ref^.base:=taicpu(hp1).oper[0]^.ref^.index;
  5415. taicpu(hp1).oper[0]^.ref^.index:=taicpu(p).oper[0]^.ref^.index;
  5416. taicpu(hp1).oper[0]^.ref^.scalefactor:=taicpu(p).oper[0]^.ref^.scalefactor;
  5417. end;
  5418. RemoveCurrentP(p);
  5419. result:=true;
  5420. exit;
  5421. end;
  5422. end;
  5423. { Change:
  5424. leal/q $x(%reg1),%reg2
  5425. ...
  5426. shll/q $y,%reg2
  5427. To:
  5428. leal/q $(x+2^y)(%reg1,2^y),%reg2 (if y <= 3)
  5429. }
  5430. if (taicpu(p).oper[0]^.ref^.base<>NR_STACK_POINTER_REG) and { lea (%rsp,scale),reg is not a valid encoding }
  5431. MatchInstruction(hp1, A_SHL, [taicpu(p).opsize]) and
  5432. MatchOpType(taicpu(hp1), top_const, top_reg) and
  5433. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  5434. (taicpu(hp1).oper[0]^.val <= 3) then
  5435. begin
  5436. Multiple := 1 shl taicpu(hp1).oper[0]^.val;
  5437. TransferUsedRegs(TmpUsedRegs);
  5438. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  5439. if
  5440. { This allows the optimisation in some circumstances even if the lea instruction already has a scale factor
  5441. (this works even if scalefactor is zero) }
  5442. ((Multiple * taicpu(p).oper[0]^.ref^.scalefactor) <= 8) and
  5443. { Ensure offset doesn't go out of bounds }
  5444. (abs(taicpu(p).oper[0]^.ref^.offset * Multiple) <= $7FFFFFFF) and
  5445. not (RegInUsedRegs(NR_DEFAULTFLAGS,TmpUsedRegs)) and
  5446. (
  5447. (
  5448. not SuperRegistersEqual(taicpu(p).oper[0]^.ref^.base, taicpu(p).oper[1]^.reg) and
  5449. (
  5450. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  5451. (taicpu(p).oper[0]^.ref^.index = NR_INVALID) or
  5452. (
  5453. { Check for lea $x(%reg1,%reg1),%reg2 and treat as it it were lea $x(%reg1,2),%reg2 }
  5454. (taicpu(p).oper[0]^.ref^.index = taicpu(p).oper[0]^.ref^.base) and
  5455. (taicpu(p).oper[0]^.ref^.scalefactor <= 1)
  5456. )
  5457. )
  5458. ) or (
  5459. (
  5460. (taicpu(p).oper[0]^.ref^.base = NR_NO) or
  5461. (taicpu(p).oper[0]^.ref^.base = NR_INVALID)
  5462. ) and
  5463. not SuperRegistersEqual(taicpu(p).oper[0]^.ref^.index, taicpu(p).oper[1]^.reg)
  5464. )
  5465. ) then
  5466. begin
  5467. repeat
  5468. with taicpu(p).oper[0]^.ref^ do
  5469. begin
  5470. { Convert lea $x(%reg1,%reg1),%reg2 to lea $x(%reg1,2),%reg2 }
  5471. if index = base then
  5472. begin
  5473. if Multiple > 4 then
  5474. { Optimisation will no longer work because resultant
  5475. scale factor will exceed 8 }
  5476. Break;
  5477. base := NR_NO;
  5478. scalefactor := 2;
  5479. DebugMsg(SPeepholeOptimization + 'lea $x(%reg1,%reg1),%reg2 -> lea $x(%reg1,2),%reg2 for following optimisation', p);
  5480. end
  5481. else if (base <> NR_NO) and (base <> NR_INVALID) then
  5482. begin
  5483. { Scale factor only works on the index register }
  5484. index := base;
  5485. base := NR_NO;
  5486. end;
  5487. { For safety }
  5488. if scalefactor <= 1 then
  5489. begin
  5490. DebugMsg(SPeepholeOptimization + 'LeaShl2Lea 1', p);
  5491. scalefactor := Multiple;
  5492. end
  5493. else
  5494. begin
  5495. DebugMsg(SPeepholeOptimization + 'LeaShl2Lea 2', p);
  5496. scalefactor := scalefactor * Multiple;
  5497. end;
  5498. offset := offset * Multiple;
  5499. end;
  5500. RemoveInstruction(hp1);
  5501. Result := True;
  5502. Exit;
  5503. { This repeat..until loop exists for the benefit of Break }
  5504. until True;
  5505. end;
  5506. end;
  5507. end;
  5508. end;
  5509. end;
  5510. function TX86AsmOptimizer.DoArithCombineOpt(var p: tai): Boolean;
  5511. var
  5512. hp1 : tai;
  5513. SubInstr: Boolean;
  5514. ThisConst: TCGInt;
  5515. const
  5516. OverflowMin: array[S_B..S_Q] of TCGInt = (-128, -32768, -2147483648, -2147483648);
  5517. { Note: 64-bit-sized arithmetic instructions can only take signed 32-bit immediates }
  5518. OverflowMax: array[S_B..S_Q] of TCGInt = ( 255, 65535, $FFFFFFFF, 2147483647);
  5519. begin
  5520. Result := False;
  5521. if taicpu(p).oper[0]^.typ <> top_const then
  5522. { Should have been confirmed before calling }
  5523. InternalError(2021102601);
  5524. SubInstr := (taicpu(p).opcode = A_SUB);
  5525. if GetLastInstruction(p, hp1) and
  5526. (hp1.typ = ait_instruction) and
  5527. (taicpu(hp1).opsize = taicpu(p).opsize) then
  5528. begin
  5529. if not (taicpu(p).opsize in [S_B, S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  5530. { Bad size }
  5531. InternalError(2022042001);
  5532. case taicpu(hp1).opcode Of
  5533. A_INC:
  5534. if MatchOperand(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  5535. begin
  5536. if SubInstr then
  5537. ThisConst := taicpu(p).oper[0]^.val - 1
  5538. else
  5539. ThisConst := taicpu(p).oper[0]^.val + 1;
  5540. end
  5541. else
  5542. Exit;
  5543. A_DEC:
  5544. if MatchOperand(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  5545. begin
  5546. if SubInstr then
  5547. ThisConst := taicpu(p).oper[0]^.val + 1
  5548. else
  5549. ThisConst := taicpu(p).oper[0]^.val - 1;
  5550. end
  5551. else
  5552. Exit;
  5553. A_SUB:
  5554. if (taicpu(hp1).oper[0]^.typ = top_const) and
  5555. MatchOperand(taicpu(hp1).oper[1]^,taicpu(p).oper[1]^) then
  5556. begin
  5557. if SubInstr then
  5558. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val
  5559. else
  5560. ThisConst := taicpu(p).oper[0]^.val - taicpu(hp1).oper[0]^.val;
  5561. end
  5562. else
  5563. Exit;
  5564. A_ADD:
  5565. if (taicpu(hp1).oper[0]^.typ = top_const) and
  5566. MatchOperand(taicpu(hp1).oper[1]^,taicpu(p).oper[1]^) then
  5567. begin
  5568. if SubInstr then
  5569. ThisConst := taicpu(p).oper[0]^.val - taicpu(hp1).oper[0]^.val
  5570. else
  5571. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val;
  5572. end
  5573. else
  5574. Exit;
  5575. else
  5576. Exit;
  5577. end;
  5578. { Check that the values are in range }
  5579. if (ThisConst < OverflowMin[taicpu(p).opsize]) or (ThisConst > OverflowMax[taicpu(p).opsize]) then
  5580. { Overflow; abort }
  5581. Exit;
  5582. if (ThisConst = 0) then
  5583. begin
  5584. DebugMsg(SPeepholeOptimization + 'Arithmetic combine: ' +
  5585. debug_op2str(taicpu(hp1).opcode) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + debug_operstr(taicpu(hp1).oper[1]^) + '; ' +
  5586. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(taicpu(p).oper[0]^.val) + ',' + debug_operstr(taicpu(p).oper[1]^) + ' cancel out (NOP)', p);
  5587. RemoveInstruction(hp1);
  5588. hp1 := tai(p.next);
  5589. RemoveInstruction(p); { Note, the choice to not use RemoveCurrentp is deliberate }
  5590. if not GetLastInstruction(hp1, p) then
  5591. p := hp1;
  5592. end
  5593. else
  5594. begin
  5595. if taicpu(hp1).opercnt=1 then
  5596. DebugMsg(SPeepholeOptimization + 'Arithmetic combine: ' +
  5597. debug_op2str(taicpu(hp1).opcode) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + '; ' +
  5598. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(taicpu(p).oper[0]^.val) + ',' + debug_operstr(taicpu(p).oper[1]^) + ' -> ' +
  5599. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(ThisConst) + ' ' + debug_operstr(taicpu(p).oper[1]^), p)
  5600. else
  5601. DebugMsg(SPeepholeOptimization + 'Arithmetic combine: ' +
  5602. debug_op2str(taicpu(hp1).opcode) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + debug_operstr(taicpu(hp1).oper[1]^) + '; ' +
  5603. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(taicpu(p).oper[0]^.val) + ',' + debug_operstr(taicpu(p).oper[1]^) + ' -> ' +
  5604. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(ThisConst) + ' ' + debug_operstr(taicpu(p).oper[1]^), p);
  5605. RemoveInstruction(hp1);
  5606. taicpu(p).loadconst(0, ThisConst);
  5607. end;
  5608. Result := True;
  5609. end;
  5610. end;
  5611. function TX86AsmOptimizer.DoMovCmpMemOpt(var p : tai; const hp1: tai; UpdateTmpUsedRegs: Boolean) : Boolean;
  5612. begin
  5613. Result := False;
  5614. if UpdateTmpUsedRegs then
  5615. TransferUsedRegs(TmpUsedRegs);
  5616. if MatchOpType(taicpu(p),top_ref,top_reg) and
  5617. { The x86 assemblers have difficulty comparing values against absolute addresses }
  5618. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) and
  5619. (taicpu(hp1).oper[0]^.typ <> top_ref) and
  5620. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  5621. (
  5622. (
  5623. (taicpu(hp1).opcode = A_TEST)
  5624. ) or (
  5625. (taicpu(hp1).opcode = A_CMP) and
  5626. { A sanity check more than anything }
  5627. not MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg)
  5628. )
  5629. ) then
  5630. begin
  5631. { change
  5632. mov mem, %reg
  5633. cmp/test x, %reg / test %reg,%reg
  5634. (reg deallocated)
  5635. to
  5636. cmp/test x, mem / cmp 0, mem
  5637. }
  5638. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  5639. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) then
  5640. begin
  5641. { Convert test %reg,%reg or test $-1,%reg to cmp $0,mem }
  5642. if (taicpu(hp1).opcode = A_TEST) and
  5643. (
  5644. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) or
  5645. MatchOperand(taicpu(hp1).oper[0]^, -1)
  5646. ) then
  5647. begin
  5648. taicpu(hp1).opcode := A_CMP;
  5649. taicpu(hp1).loadconst(0, 0);
  5650. end;
  5651. taicpu(hp1).loadref(1, taicpu(p).oper[0]^.ref^);
  5652. DebugMsg(SPeepholeOptimization + 'MOV/CMP -> CMP (memory check)', p);
  5653. RemoveCurrentP(p, hp1);
  5654. Result := True;
  5655. Exit;
  5656. end;
  5657. end;
  5658. end;
  5659. function TX86AsmOptimizer.DoSETccLblRETOpt(var p: tai; const hp_label: tai_label) : Boolean;
  5660. var
  5661. hp_allocstart, hp_pos, hp2, hp3, hp4, hp5, hp6: tai;
  5662. ThisReg, SecondReg: TRegister;
  5663. JumpLoc: TAsmLabel;
  5664. NewSize: TOpSize;
  5665. begin
  5666. Result := False;
  5667. {
  5668. Convert:
  5669. j<c> .L1
  5670. .L2:
  5671. mov 1,reg
  5672. jmp .L3 (or ret, although it might not be a RET yet)
  5673. .L1:
  5674. mov 0,reg
  5675. jmp .L3 (or ret)
  5676. ( As long as .L3 <> .L1 or .L2)
  5677. To:
  5678. mov 0,reg
  5679. set<not(c)> reg
  5680. jmp .L3 (or ret)
  5681. .L2:
  5682. mov 1,reg
  5683. jmp .L3 (or ret)
  5684. .L1:
  5685. mov 0,reg
  5686. jmp .L3 (or ret)
  5687. }
  5688. if JumpTargetOp(taicpu(p))^.ref^.refaddr<>addr_full then
  5689. Exit;
  5690. JumpLoc := TAsmLabel(JumpTargetOp(taicpu(p))^.ref^.symbol);
  5691. if GetNextInstruction(hp_label, hp2) and
  5692. MatchInstruction(hp2,A_MOV,[]) and
  5693. (taicpu(hp2).oper[0]^.typ = top_const) and
  5694. (
  5695. (
  5696. (taicpu(hp2).oper[1]^.typ = top_reg)
  5697. {$ifdef i386}
  5698. { Under i386, ESI, EDI, EBP and ESP
  5699. don't have an 8-bit representation }
  5700. and not (getsupreg(taicpu(hp2).oper[1]^.reg) in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  5701. {$endif i386}
  5702. ) or (
  5703. {$ifdef i386}
  5704. (taicpu(hp2).oper[1]^.typ <> top_reg) and
  5705. {$endif i386}
  5706. (taicpu(hp2).opsize = S_B)
  5707. )
  5708. ) and
  5709. GetNextInstruction(hp2, hp3) and
  5710. MatchInstruction(hp3, A_JMP, A_RET, []) and
  5711. (
  5712. (taicpu(hp3).opcode=A_RET) or
  5713. (
  5714. (taicpu(hp3).oper[0]^.ref^.refaddr=addr_full) and
  5715. (tasmlabel(taicpu(hp3).oper[0]^.ref^.symbol)<>tai_label(hp_label).labsym)
  5716. )
  5717. ) and
  5718. GetNextInstruction(hp3, hp4) and
  5719. SkipAligns(hp4, hp4) and
  5720. (hp4.typ=ait_label) and
  5721. (tai_label(hp4).labsym=JumpLoc) and
  5722. (
  5723. not (cs_opt_size in current_settings.optimizerswitches) or
  5724. { If the initial jump is the label's only reference, then it will
  5725. become a dead label if the other conditions are met and hence
  5726. remove at least 2 instructions, including a jump }
  5727. (JumpLoc.getrefs = 1)
  5728. ) and
  5729. { Don't check if hp3 jumps to hp4 because this is a zero-distance jump
  5730. that will be optimised out }
  5731. GetNextInstruction(hp4, hp5) and
  5732. MatchInstruction(hp5,A_MOV,[taicpu(hp2).opsize]) and
  5733. (taicpu(hp5).oper[0]^.typ = top_const) and
  5734. (
  5735. ((taicpu(hp2).oper[0]^.val = 0) and (taicpu(hp5).oper[0]^.val = 1)) or
  5736. ((taicpu(hp2).oper[0]^.val = 1) and (taicpu(hp5).oper[0]^.val = 0))
  5737. ) and
  5738. MatchOperand(taicpu(hp2).oper[1]^,taicpu(hp5).oper[1]^) and
  5739. GetNextInstruction(hp5,hp6) and
  5740. (
  5741. (hp6.typ<>ait_label) or
  5742. SkipLabels(hp6, hp6)
  5743. ) and
  5744. (hp6.typ=ait_instruction) then
  5745. begin
  5746. { First, let's look at the two jumps that are hp3 and hp6 }
  5747. if not
  5748. (
  5749. (taicpu(hp6).opcode=taicpu(hp3).opcode) and { Both RET or both JMP to the same label }
  5750. (
  5751. (taicpu(hp6).opcode=A_RET) or
  5752. MatchOperand(taicpu(hp6).oper[0]^, taicpu(hp3).oper[0]^)
  5753. )
  5754. ) then
  5755. { If condition is False, then the JMP/RET instructions matched conventionally }
  5756. begin
  5757. { See if one of the jumps can be instantly converted into a RET }
  5758. if (taicpu(hp3).opcode=A_JMP) then
  5759. begin
  5760. { Reuse hp5 }
  5761. hp5 := getlabelwithsym(TAsmLabel(JumpTargetOp(taicpu(hp3))^.ref^.symbol));
  5762. { Make sure hp5 doesn't jump back to .L2 (infinite loop) }
  5763. if not Assigned(hp5) or (hp5=hp4) or not GetNextInstruction(hp5, hp5) then
  5764. Exit;
  5765. if MatchInstruction(hp5, A_RET, []) then
  5766. begin
  5767. DebugMsg(SPeepholeOptimization + 'Converted JMP to RET as part of SETcc optimisation (1st jump)', hp3);
  5768. ConvertJumpToRET(hp3, hp5);
  5769. Result := True;
  5770. end
  5771. else
  5772. Exit;
  5773. end;
  5774. if (taicpu(hp6).opcode=A_JMP) then
  5775. begin
  5776. { Reuse hp5 }
  5777. hp5 := getlabelwithsym(TAsmLabel(JumpTargetOp(taicpu(hp6))^.ref^.symbol));
  5778. if not Assigned(hp5) or not GetNextInstruction(hp5, hp5) then
  5779. Exit;
  5780. if MatchInstruction(hp5, A_RET, []) then
  5781. begin
  5782. DebugMsg(SPeepholeOptimization + 'Converted JMP to RET as part of SETcc optimisation (2nd jump)', hp6);
  5783. ConvertJumpToRET(hp6, hp5);
  5784. Result := True;
  5785. end
  5786. else
  5787. Exit;
  5788. end;
  5789. if not
  5790. (
  5791. (taicpu(hp6).opcode=taicpu(hp3).opcode) and { Both RET or both JMP to the same label }
  5792. (
  5793. (taicpu(hp6).opcode=A_RET) or
  5794. MatchOperand(taicpu(hp6).oper[0]^, taicpu(hp3).oper[0]^)
  5795. )
  5796. ) then
  5797. { Still doesn't match }
  5798. Exit;
  5799. end;
  5800. if (taicpu(hp2).oper[0]^.val = 1) then
  5801. begin
  5802. taicpu(p).condition := inverse_cond(taicpu(p).condition);
  5803. DebugMsg(SPeepholeOptimization + 'J(c)Mov1Jmp/RetMov0Jmp/Ret -> Set(~c)Jmp/Ret',p)
  5804. end
  5805. else
  5806. DebugMsg(SPeepholeOptimization + 'J(c)Mov0Jmp/RetMov1Jmp/Ret -> Set(c)Jmp/Ret',p);
  5807. if taicpu(hp2).opsize=S_B then
  5808. begin
  5809. if taicpu(hp2).oper[1]^.typ = top_reg then
  5810. begin
  5811. SecondReg := taicpu(hp2).oper[1]^.reg;
  5812. hp4:=taicpu.op_reg(A_SETcc, S_B, SecondReg);
  5813. end
  5814. else
  5815. begin
  5816. hp4:=taicpu.op_ref(A_SETcc, S_B, taicpu(hp2).oper[1]^.ref^);
  5817. SecondReg := NR_NO;
  5818. end;
  5819. hp_pos := p;
  5820. hp_allocstart := hp4;
  5821. end
  5822. else
  5823. begin
  5824. { Will be a register because the size can't be S_B otherwise }
  5825. SecondReg:=taicpu(hp2).oper[1]^.reg;
  5826. ThisReg:=newreg(R_INTREGISTER,getsupreg(SecondReg), R_SUBL);
  5827. hp4:=taicpu.op_reg(A_SETcc, S_B, ThisReg);
  5828. if (cs_opt_size in current_settings.optimizerswitches) then
  5829. begin
  5830. { Favour using MOVZX when optimising for size }
  5831. case taicpu(hp2).opsize of
  5832. S_W:
  5833. NewSize := S_BW;
  5834. S_L:
  5835. NewSize := S_BL;
  5836. {$ifdef x86_64}
  5837. S_Q:
  5838. begin
  5839. NewSize := S_BL;
  5840. { Will implicitly zero-extend to 64-bit }
  5841. setsubreg(SecondReg, R_SUBD);
  5842. end;
  5843. {$endif x86_64}
  5844. else
  5845. InternalError(2022101301);
  5846. end;
  5847. hp5:=taicpu.op_reg_reg(A_MOVZX, NewSize, ThisReg, SecondReg);
  5848. { Inserting it right before p will guarantee that the flags are also tracked }
  5849. Asml.InsertBefore(hp5, p);
  5850. { Make sure the SET instruction gets inserted before the MOVZX instruction }
  5851. hp_pos := hp5;
  5852. hp_allocstart := hp4;
  5853. end
  5854. else
  5855. begin
  5856. hp5:=taicpu.op_const_reg(A_MOV, taicpu(hp2).opsize, 0, SecondReg);
  5857. { Inserting it right before p will guarantee that the flags are also tracked }
  5858. Asml.InsertBefore(hp5, p);
  5859. hp_pos := p;
  5860. hp_allocstart := hp5;
  5861. end;
  5862. taicpu(hp5).fileinfo:=taicpu(p).fileinfo;
  5863. end;
  5864. taicpu(hp4).fileinfo := taicpu(p).fileinfo;
  5865. taicpu(hp4).condition := taicpu(p).condition;
  5866. asml.InsertBefore(hp4, hp_pos);
  5867. if taicpu(hp3).is_jmp then
  5868. begin
  5869. JumpLoc.decrefs;
  5870. MakeUnconditional(taicpu(p));
  5871. taicpu(p).loadref(0, JumpTargetOp(taicpu(hp3))^.ref^);
  5872. TAsmLabel(JumpTargetOp(taicpu(hp3))^.ref^.symbol).increfs;
  5873. end
  5874. else
  5875. ConvertJumpToRET(p, hp3);
  5876. if SecondReg <> NR_NO then
  5877. { Ensure the destination register is allocated over this region }
  5878. AllocRegBetween(SecondReg, hp_allocstart, p, UsedRegs);
  5879. if (JumpLoc.getrefs = 0) then
  5880. RemoveDeadCodeAfterJump(hp3);
  5881. Result:=true;
  5882. exit;
  5883. end;
  5884. end;
  5885. function TX86AsmOptimizer.OptPass1Sub(var p : tai) : boolean;
  5886. var
  5887. hp1, hp2: tai;
  5888. ActiveReg: TRegister;
  5889. OldOffset: asizeint;
  5890. ThisConst: TCGInt;
  5891. function RegDeallocated: Boolean;
  5892. begin
  5893. TransferUsedRegs(TmpUsedRegs);
  5894. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5895. Result := not(RegUsedAfterInstruction(ActiveReg,hp1,TmpUsedRegs))
  5896. end;
  5897. begin
  5898. Result:=false;
  5899. hp1 := nil;
  5900. { replace
  5901. subX const,%reg1
  5902. leaX (%reg1,%reg1,Y),%reg2 // Base or index might not be equal to reg1
  5903. dealloc %reg1
  5904. by
  5905. leaX -const-const*Y(%reg1,%reg1,Y),%reg2
  5906. }
  5907. if MatchOpType(taicpu(p),top_const,top_reg) then
  5908. begin
  5909. ActiveReg := taicpu(p).oper[1]^.reg;
  5910. { Ensures the entire register was updated }
  5911. if (taicpu(p).opsize >= S_L) and
  5912. GetNextInstructionUsingReg(p,hp1, ActiveReg) and
  5913. MatchInstruction(hp1,A_LEA,[]) and
  5914. (SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.base) or
  5915. SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.index)) and
  5916. (
  5917. { Cover the case where the register in the reference is also the destination register }
  5918. Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, ActiveReg) or
  5919. (
  5920. { Try to avoid the expensive check of RegUsedAfterInstruction if we know it will return False }
  5921. not SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ActiveReg) and
  5922. RegDeallocated
  5923. )
  5924. ) then
  5925. begin
  5926. OldOffset := taicpu(hp1).oper[0]^.ref^.offset;
  5927. if ActiveReg=taicpu(hp1).oper[0]^.ref^.base then
  5928. Dec(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val);
  5929. if ActiveReg=taicpu(hp1).oper[0]^.ref^.index then
  5930. Dec(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val*max(taicpu(hp1).oper[0]^.ref^.scalefactor,1));
  5931. {$ifdef x86_64}
  5932. if (taicpu(hp1).oper[0]^.ref^.offset > $7FFFFFFF) or (taicpu(hp1).oper[0]^.ref^.offset < -2147483648) then
  5933. begin
  5934. { Overflow; abort }
  5935. taicpu(hp1).oper[0]^.ref^.offset := OldOffset;
  5936. end
  5937. else
  5938. {$endif x86_64}
  5939. begin
  5940. DebugMsg(SPeepholeOptimization + 'SubLea2Lea done',p);
  5941. if not (cs_opt_level3 in current_settings.optimizerswitches) then
  5942. { hp1 is the immediate next instruction for sure - good for a quick speed boost }
  5943. RemoveCurrentP(p, hp1)
  5944. else
  5945. RemoveCurrentP(p);
  5946. result:=true;
  5947. Exit;
  5948. end;
  5949. end;
  5950. if (
  5951. { Save calling GetNextInstructionUsingReg again }
  5952. Assigned(hp1) or
  5953. GetNextInstructionUsingReg(p,hp1, ActiveReg)
  5954. ) and
  5955. MatchInstruction(hp1,A_SUB,[taicpu(p).opsize]) and
  5956. (taicpu(hp1).oper[1]^.reg = ActiveReg) then
  5957. begin
  5958. if taicpu(hp1).oper[0]^.typ = top_const then
  5959. begin
  5960. { Merge add const1,%reg; add const2,%reg to add const1+const2,%reg }
  5961. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val;
  5962. Result := True;
  5963. { Handle any overflows }
  5964. case taicpu(p).opsize of
  5965. S_B:
  5966. taicpu(p).oper[0]^.val := ThisConst and $FF;
  5967. S_W:
  5968. taicpu(p).oper[0]^.val := ThisConst and $FFFF;
  5969. S_L:
  5970. taicpu(p).oper[0]^.val := ThisConst and $FFFFFFFF;
  5971. {$ifdef x86_64}
  5972. S_Q:
  5973. if (ThisConst > $7FFFFFFF) or (ThisConst < -2147483648) then
  5974. { Overflow; abort }
  5975. Result := False
  5976. else
  5977. taicpu(p).oper[0]^.val := ThisConst;
  5978. {$endif x86_64}
  5979. else
  5980. InternalError(2021102611);
  5981. end;
  5982. { Result may get set to False again if the combined immediate overflows for S_Q sizes }
  5983. if Result then
  5984. begin
  5985. if (taicpu(p).oper[0]^.val < 0) and
  5986. (
  5987. ((taicpu(p).opsize = S_B) and (taicpu(p).oper[0]^.val <> -128)) or
  5988. ((taicpu(p).opsize = S_W) and (taicpu(p).oper[0]^.val <> -32768)) or
  5989. ((taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) and (taicpu(p).oper[0]^.val <> -2147483648))
  5990. ) then
  5991. begin
  5992. DebugMsg(SPeepholeOptimization + 'SUB; ADD/SUB -> ADD',p);
  5993. taicpu(p).opcode := A_SUB;
  5994. taicpu(p).oper[0]^.val := -taicpu(p).oper[0]^.val;
  5995. end
  5996. else
  5997. DebugMsg(SPeepholeOptimization + 'SUB; ADD/SUB -> SUB',p);
  5998. RemoveInstruction(hp1);
  5999. end;
  6000. end
  6001. else
  6002. begin
  6003. { Make doubly sure the flags aren't in use because the order of subtractions may affect them }
  6004. TransferUsedRegs(TmpUsedRegs);
  6005. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  6006. hp2 := p;
  6007. while not (cs_opt_level3 in current_settings.optimizerswitches) and
  6008. GetNextInstruction(hp2, hp2) and (hp2 <> hp1) do
  6009. UpdateUsedRegs(TmpUsedRegs, tai(hp2.next));
  6010. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  6011. begin
  6012. { Move the constant subtraction to after the reg/ref addition to improve optimisation }
  6013. DebugMsg(SPeepholeOptimization + 'Add/sub swap 1b done',p);
  6014. Asml.Remove(p);
  6015. Asml.InsertAfter(p, hp1);
  6016. p := hp1;
  6017. Result := True;
  6018. Exit;
  6019. end;
  6020. end;
  6021. end;
  6022. { * change "subl $2, %esp; pushw x" to "pushl x"}
  6023. { * change "sub/add const1, reg" or "dec reg" followed by
  6024. "sub const2, reg" to one "sub ..., reg" }
  6025. {$ifdef i386}
  6026. if (taicpu(p).oper[0]^.val = 2) and
  6027. (ActiveReg = NR_ESP) and
  6028. { Don't do the sub/push optimization if the sub }
  6029. { comes from setting up the stack frame (JM) }
  6030. (not(GetLastInstruction(p,hp1)) or
  6031. not(MatchInstruction(hp1,A_MOV,[S_L]) and
  6032. MatchOperand(taicpu(hp1).oper[0]^,NR_ESP) and
  6033. MatchOperand(taicpu(hp1).oper[0]^,NR_EBP))) then
  6034. begin
  6035. hp1 := tai(p.next);
  6036. while Assigned(hp1) and
  6037. (tai(hp1).typ in [ait_instruction]+SkipInstr) and
  6038. not RegReadByInstruction(NR_ESP,hp1) and
  6039. not RegModifiedByInstruction(NR_ESP,hp1) do
  6040. hp1 := tai(hp1.next);
  6041. if Assigned(hp1) and
  6042. MatchInstruction(hp1,A_PUSH,[S_W]) then
  6043. begin
  6044. taicpu(hp1).changeopsize(S_L);
  6045. if taicpu(hp1).oper[0]^.typ=top_reg then
  6046. setsubreg(taicpu(hp1).oper[0]^.reg,R_SUBWHOLE);
  6047. hp1 := tai(p.next);
  6048. RemoveCurrentp(p, hp1);
  6049. Result:=true;
  6050. exit;
  6051. end;
  6052. end;
  6053. {$endif i386}
  6054. if DoArithCombineOpt(p) then
  6055. Result:=true;
  6056. end;
  6057. end;
  6058. function TX86AsmOptimizer.OptPass1SHLSAL(var p : tai) : boolean;
  6059. var
  6060. TmpBool1,TmpBool2 : Boolean;
  6061. tmpref : treference;
  6062. hp1,hp2: tai;
  6063. mask, shiftval: tcgint;
  6064. begin
  6065. Result:=false;
  6066. { All these optimisations work on "shl/sal const,%reg" }
  6067. if not MatchOpType(taicpu(p),top_const,top_reg) then
  6068. Exit;
  6069. if (taicpu(p).opsize in [S_L{$ifdef x86_64},S_Q{$endif x86_64}]) and
  6070. (taicpu(p).oper[0]^.val <= 3) then
  6071. { Changes "shl const, %reg32; add const/reg, %reg32" to one lea statement }
  6072. begin
  6073. { should we check the next instruction? }
  6074. TmpBool1 := True;
  6075. { have we found an add/sub which could be
  6076. integrated in the lea? }
  6077. TmpBool2 := False;
  6078. reference_reset(tmpref,2,[]);
  6079. TmpRef.index := taicpu(p).oper[1]^.reg;
  6080. TmpRef.scalefactor := 1 shl taicpu(p).oper[0]^.val;
  6081. while TmpBool1 and
  6082. GetNextInstruction(p, hp1) and
  6083. (tai(hp1).typ = ait_instruction) and
  6084. ((((taicpu(hp1).opcode = A_ADD) or
  6085. (taicpu(hp1).opcode = A_SUB)) and
  6086. (taicpu(hp1).oper[1]^.typ = Top_Reg) and
  6087. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg)) or
  6088. (((taicpu(hp1).opcode = A_INC) or
  6089. (taicpu(hp1).opcode = A_DEC)) and
  6090. (taicpu(hp1).oper[0]^.typ = Top_Reg) and
  6091. (taicpu(hp1).oper[0]^.reg = taicpu(p).oper[1]^.reg)) or
  6092. ((taicpu(hp1).opcode = A_LEA) and
  6093. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  6094. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg))) and
  6095. (not GetNextInstruction(hp1,hp2) or
  6096. not instrReadsFlags(hp2)) Do
  6097. begin
  6098. TmpBool1 := False;
  6099. if taicpu(hp1).opcode=A_LEA then
  6100. begin
  6101. if (TmpRef.base = NR_NO) and
  6102. (taicpu(hp1).oper[0]^.ref^.symbol=nil) and
  6103. (taicpu(hp1).oper[0]^.ref^.relsymbol=nil) and
  6104. { Segment register isn't a concern here }
  6105. ((taicpu(hp1).oper[0]^.ref^.scalefactor=0) or
  6106. (taicpu(hp1).oper[0]^.ref^.scalefactor*tmpref.scalefactor<=8)) then
  6107. begin
  6108. TmpBool1 := True;
  6109. TmpBool2 := True;
  6110. inc(TmpRef.offset, taicpu(hp1).oper[0]^.ref^.offset);
  6111. if taicpu(hp1).oper[0]^.ref^.scalefactor<>0 then
  6112. tmpref.scalefactor:=tmpref.scalefactor*taicpu(hp1).oper[0]^.ref^.scalefactor;
  6113. TmpRef.base := taicpu(hp1).oper[0]^.ref^.base;
  6114. RemoveInstruction(hp1);
  6115. end
  6116. end
  6117. else if (taicpu(hp1).oper[0]^.typ = Top_Const) then
  6118. begin
  6119. TmpBool1 := True;
  6120. TmpBool2 := True;
  6121. case taicpu(hp1).opcode of
  6122. A_ADD:
  6123. inc(TmpRef.offset, longint(taicpu(hp1).oper[0]^.val));
  6124. A_SUB:
  6125. dec(TmpRef.offset, longint(taicpu(hp1).oper[0]^.val));
  6126. else
  6127. internalerror(2019050536);
  6128. end;
  6129. RemoveInstruction(hp1);
  6130. end
  6131. else
  6132. if (taicpu(hp1).oper[0]^.typ = Top_Reg) and
  6133. (((taicpu(hp1).opcode = A_ADD) and
  6134. (TmpRef.base = NR_NO)) or
  6135. (taicpu(hp1).opcode = A_INC) or
  6136. (taicpu(hp1).opcode = A_DEC)) then
  6137. begin
  6138. TmpBool1 := True;
  6139. TmpBool2 := True;
  6140. case taicpu(hp1).opcode of
  6141. A_ADD:
  6142. TmpRef.base := taicpu(hp1).oper[0]^.reg;
  6143. A_INC:
  6144. inc(TmpRef.offset);
  6145. A_DEC:
  6146. dec(TmpRef.offset);
  6147. else
  6148. internalerror(2019050535);
  6149. end;
  6150. RemoveInstruction(hp1);
  6151. end;
  6152. end;
  6153. if TmpBool2
  6154. {$ifndef x86_64}
  6155. or
  6156. ((current_settings.optimizecputype < cpu_Pentium2) and
  6157. (taicpu(p).oper[0]^.val <= 3) and
  6158. not(cs_opt_size in current_settings.optimizerswitches))
  6159. {$endif x86_64}
  6160. then
  6161. begin
  6162. if not(TmpBool2) and
  6163. (taicpu(p).oper[0]^.val=1) then
  6164. begin
  6165. taicpu(p).opcode := A_ADD;
  6166. taicpu(p).loadreg(0, taicpu(p).oper[1]^.reg);
  6167. end
  6168. else
  6169. begin
  6170. taicpu(p).opcode := A_LEA;
  6171. taicpu(p).loadref(0, TmpRef);
  6172. end;
  6173. DebugMsg(SPeepholeOptimization + 'ShlAddLeaSubIncDec2Lea',p);
  6174. Result := True;
  6175. end;
  6176. end
  6177. {$ifndef x86_64}
  6178. else if (current_settings.optimizecputype < cpu_Pentium2) then
  6179. begin
  6180. { changes "shl $1, %reg" to "add %reg, %reg", which is the same on a 386,
  6181. but faster on a 486, and Tairable in both U and V pipes on the Pentium
  6182. (unlike shl, which is only Tairable in the U pipe) }
  6183. if taicpu(p).oper[0]^.val=1 then
  6184. begin
  6185. taicpu(p).opcode := A_ADD;
  6186. taicpu(p).loadreg(0, taicpu(p).oper[1]^.reg);
  6187. Result := True;
  6188. end
  6189. { changes "shl $2, %reg" to "lea (,%reg,4), %reg"
  6190. "shl $3, %reg" to "lea (,%reg,8), %reg }
  6191. else if (taicpu(p).opsize = S_L) and
  6192. (taicpu(p).oper[0]^.val<= 3) then
  6193. begin
  6194. reference_reset(tmpref,2,[]);
  6195. TmpRef.index := taicpu(p).oper[1]^.reg;
  6196. TmpRef.scalefactor := 1 shl taicpu(p).oper[0]^.val;
  6197. taicpu(p).opcode := A_LEA;
  6198. taicpu(p).loadref(0, TmpRef);
  6199. Result := True;
  6200. end;
  6201. end
  6202. {$endif x86_64}
  6203. else if
  6204. GetNextInstruction(p, hp1) and (hp1.typ = ait_instruction) and MatchOpType(taicpu(hp1), top_const, top_reg) and
  6205. (
  6206. (
  6207. MatchInstruction(hp1, A_AND, [taicpu(p).opsize]) and
  6208. SetAndTest(hp1, hp2)
  6209. {$ifdef x86_64}
  6210. ) or
  6211. (
  6212. MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  6213. GetNextInstruction(hp1, hp2) and
  6214. MatchInstruction(hp2, A_AND, [taicpu(p).opsize]) and
  6215. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  6216. (taicpu(hp1).oper[1]^.reg = taicpu(hp2).oper[0]^.reg)
  6217. {$endif x86_64}
  6218. )
  6219. ) and
  6220. (taicpu(p).oper[1]^.reg = taicpu(hp2).oper[1]^.reg) then
  6221. begin
  6222. { Change:
  6223. shl x, %reg1
  6224. mov -(1<<x), %reg2
  6225. and %reg2, %reg1
  6226. Or:
  6227. shl x, %reg1
  6228. and -(1<<x), %reg1
  6229. To just:
  6230. shl x, %reg1
  6231. Since the and operation only zeroes bits that are already zero from the shl operation
  6232. }
  6233. case taicpu(p).oper[0]^.val of
  6234. 8:
  6235. mask:=$FFFFFFFFFFFFFF00;
  6236. 16:
  6237. mask:=$FFFFFFFFFFFF0000;
  6238. 32:
  6239. mask:=$FFFFFFFF00000000;
  6240. 63:
  6241. { Constant pre-calculated to prevent overflow errors with Int64 }
  6242. mask:=$8000000000000000;
  6243. else
  6244. begin
  6245. if taicpu(p).oper[0]^.val >= 64 then
  6246. { Shouldn't happen realistically, since the register
  6247. is guaranteed to be set to zero at this point }
  6248. mask := 0
  6249. else
  6250. mask := -(Int64(1 shl taicpu(p).oper[0]^.val));
  6251. end;
  6252. end;
  6253. if taicpu(hp1).oper[0]^.val = mask then
  6254. begin
  6255. { Everything checks out, perform the optimisation, as long as
  6256. the FLAGS register isn't being used}
  6257. TransferUsedRegs(TmpUsedRegs);
  6258. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  6259. {$ifdef x86_64}
  6260. if (hp1 <> hp2) then
  6261. begin
  6262. { "shl/mov/and" version }
  6263. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  6264. { Don't do the optimisation if the FLAGS register is in use }
  6265. if not(RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp2, TmpUsedRegs)) then
  6266. begin
  6267. DebugMsg(SPeepholeOptimization + 'ShlMovAnd2Shl', p);
  6268. { Don't remove the 'mov' instruction if its register is used elsewhere }
  6269. if not(RegUsedAfterInstruction(taicpu(hp1).oper[1]^.reg, hp2, TmpUsedRegs)) then
  6270. begin
  6271. RemoveInstruction(hp1);
  6272. Result := True;
  6273. end;
  6274. { Only set Result to True if the 'mov' instruction was removed }
  6275. RemoveInstruction(hp2);
  6276. end;
  6277. end
  6278. else
  6279. {$endif x86_64}
  6280. begin
  6281. { "shl/and" version }
  6282. { Don't do the optimisation if the FLAGS register is in use }
  6283. if not(RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)) then
  6284. begin
  6285. DebugMsg(SPeepholeOptimization + 'ShlAnd2Shl', p);
  6286. RemoveInstruction(hp1);
  6287. Result := True;
  6288. end;
  6289. end;
  6290. Exit;
  6291. end
  6292. else {$ifdef x86_64}if (hp1 = hp2) then{$endif x86_64}
  6293. begin
  6294. { Even if the mask doesn't allow for its removal, we might be
  6295. able to optimise the mask for the "shl/and" version, which
  6296. may permit other peephole optimisations }
  6297. {$ifdef DEBUG_AOPTCPU}
  6298. mask := taicpu(hp1).oper[0]^.val and mask;
  6299. if taicpu(hp1).oper[0]^.val <> mask then
  6300. begin
  6301. DebugMsg(
  6302. SPeepholeOptimization +
  6303. 'Changed mask from $' + debug_tostr(taicpu(hp1).oper[0]^.val) +
  6304. ' to $' + debug_tostr(mask) +
  6305. 'based on previous instruction (ShlAnd2ShlAnd)', hp1);
  6306. taicpu(hp1).oper[0]^.val := mask;
  6307. end;
  6308. {$else DEBUG_AOPTCPU}
  6309. { If debugging is off, just set the operand even if it's the same }
  6310. taicpu(hp1).oper[0]^.val := taicpu(hp1).oper[0]^.val and mask;
  6311. {$endif DEBUG_AOPTCPU}
  6312. end;
  6313. end;
  6314. {
  6315. change
  6316. shl/sal const,reg
  6317. <op> ...(...,reg,1),...
  6318. into
  6319. <op> ...(...,reg,1 shl const),...
  6320. if const in 1..3
  6321. }
  6322. if MatchOpType(taicpu(p), top_const, top_reg) and
  6323. (taicpu(p).oper[0]^.val in [1..3]) and
  6324. GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg) and
  6325. ((MatchInstruction(hp1,A_MOV,A_LEA,[]) and
  6326. MatchOpType(taicpu(hp1),top_ref,top_reg)) or
  6327. (MatchInstruction(hp1,A_FST,A_FSTP,A_FLD,[]) and
  6328. MatchOpType(taicpu(hp1),top_ref))
  6329. ) and
  6330. (taicpu(p).oper[1]^.reg=taicpu(hp1).oper[0]^.ref^.index) and
  6331. (taicpu(p).oper[1]^.reg<>taicpu(hp1).oper[0]^.ref^.base) and
  6332. (taicpu(hp1).oper[0]^.ref^.scalefactor in [0,1]) then
  6333. begin
  6334. TransferUsedRegs(TmpUsedRegs);
  6335. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  6336. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs)) then
  6337. begin
  6338. taicpu(hp1).oper[0]^.ref^.scalefactor:=1 shl taicpu(p).oper[0]^.val;
  6339. DebugMsg(SPeepholeOptimization + 'ShlOp2Op', p);
  6340. RemoveCurrentP(p);
  6341. Result:=true;
  6342. exit;
  6343. end;
  6344. end;
  6345. if MatchOpType(taicpu(p), top_const, top_reg) and
  6346. GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg) and
  6347. MatchInstruction(hp1,A_SHL,[taicpu(p).opsize]) and
  6348. MatchOpType(taicpu(hp1),top_const,top_reg) and
  6349. (taicpu(p).oper[1]^.reg=taicpu(hp1).oper[1]^.reg) then
  6350. begin
  6351. shiftval:=taicpu(p).oper[0]^.val+taicpu(hp1).oper[0]^.val;
  6352. if ((taicpu(p).opsize=S_B) and (shiftval>7)) or
  6353. ((taicpu(p).opsize=S_W) and (shiftval>15)) or
  6354. {$ifdef x86_64}
  6355. ((taicpu(p).opsize=S_Q) and (shiftval>63)) or
  6356. {$endif x86_64}
  6357. ((taicpu(p).opsize=S_L) and (shiftval>31)) then
  6358. begin
  6359. DebugMsg(SPeepholeOptimization + 'ShlShl2Mov', p);
  6360. taicpu(hp1).opcode:=A_MOV;
  6361. taicpu(hp1).oper[0]^.val:=0;
  6362. end
  6363. else
  6364. begin
  6365. DebugMsg(SPeepholeOptimization + 'ShlShl2Shl', p);
  6366. taicpu(hp1).oper[0]^.val:=shiftval;
  6367. end;
  6368. RemoveCurrentP(p);
  6369. Result:=true;
  6370. exit;
  6371. end;
  6372. end;
  6373. class function TX86AsmOptimizer.IsShrMovZFoldable(shr_size, movz_size: topsize; Shift: TCGInt): Boolean;
  6374. begin
  6375. case shr_size of
  6376. S_B:
  6377. { No valid combinations }
  6378. Result := False;
  6379. S_W:
  6380. Result := (Shift >= 8) and (movz_size = S_BW);
  6381. S_L:
  6382. Result :=
  6383. (Shift >= 24) { Any opsize is valid for this shift } or
  6384. ((Shift >= 16) and (movz_size = S_WL));
  6385. {$ifdef x86_64}
  6386. S_Q:
  6387. Result :=
  6388. (Shift >= 56) { Any opsize is valid for this shift } or
  6389. ((Shift >= 48) and (movz_size = S_WL));
  6390. {$endif x86_64}
  6391. else
  6392. InternalError(2022081510);
  6393. end;
  6394. end;
  6395. function TX86AsmOptimizer.OptPass1SHR(var p : tai) : boolean;
  6396. var
  6397. hp1, hp2: tai;
  6398. Shift: TCGInt;
  6399. LimitSize: Topsize;
  6400. DoNotMerge: Boolean;
  6401. begin
  6402. Result := False;
  6403. { All these optimisations work on "shr const,%reg" }
  6404. if not MatchOpType(taicpu(p), top_const, top_reg) then
  6405. Exit;
  6406. DoNotMerge := False;
  6407. Shift := taicpu(p).oper[0]^.val;
  6408. LimitSize := taicpu(p).opsize;
  6409. hp1 := p;
  6410. repeat
  6411. if not GetNextInstructionUsingReg(hp1, hp1, taicpu(p).oper[1]^.reg) or (hp1.typ <> ait_instruction) then
  6412. Exit;
  6413. case taicpu(hp1).opcode of
  6414. A_TEST, A_CMP, A_Jcc:
  6415. { Skip over conditional jumps and relevant comparisons }
  6416. Continue;
  6417. A_MOVZX:
  6418. if MatchOpType(taicpu(hp1), top_reg, top_reg) and
  6419. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(p).oper[1]^.reg) then
  6420. begin
  6421. { Since the original register is being read as is, subsequent
  6422. SHRs must not be merged at this point }
  6423. DoNotMerge := True;
  6424. if IsShrMovZFoldable(taicpu(p).opsize, taicpu(hp1).opsize, Shift) then
  6425. begin
  6426. if not SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg) then { Different register target }
  6427. begin
  6428. DebugMsg(SPeepholeOptimization + 'Converted MOVZX instruction to MOV since previous SHR makes zero-extension unnecessary (ShrMovz2ShrMov 1)', hp1);
  6429. taicpu(hp1).opcode := A_MOV;
  6430. setsubreg(taicpu(hp1).oper[0]^.reg, getsubreg(taicpu(hp1).oper[1]^.reg));
  6431. case taicpu(hp1).opsize of
  6432. S_BW:
  6433. taicpu(hp1).opsize := S_W;
  6434. S_BL, S_WL:
  6435. taicpu(hp1).opsize := S_L;
  6436. else
  6437. InternalError(2022081503);
  6438. end;
  6439. { p itself hasn't changed, so no need to set Result to True }
  6440. Include(OptsToCheck, aoc_ForceNewIteration);
  6441. { See if there's anything afterwards that can be
  6442. optimised, since the input register hasn't changed }
  6443. Continue;
  6444. end;
  6445. { NOTE: If the MOVZX instruction reads and writes the same
  6446. register, defer this to the post-peephole optimisation stage }
  6447. Exit;
  6448. end;
  6449. end;
  6450. A_SHL, A_SAL, A_SHR:
  6451. if (taicpu(hp1).opsize <= LimitSize) and
  6452. MatchOpType(taicpu(hp1), top_const, top_reg) and
  6453. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[1]^.reg) then
  6454. begin
  6455. { Make sure the sizes don't exceed the register size limit
  6456. (measured by the shift value falling below the limit) }
  6457. if taicpu(hp1).opsize < LimitSize then
  6458. LimitSize := taicpu(hp1).opsize;
  6459. if taicpu(hp1).opcode = A_SHR then
  6460. Inc(Shift, taicpu(hp1).oper[0]^.val)
  6461. else
  6462. begin
  6463. Dec(Shift, taicpu(hp1).oper[0]^.val);
  6464. DoNotMerge := True;
  6465. end;
  6466. if Shift < topsize2memsize[taicpu(p).opsize] - topsize2memsize[LimitSize] then
  6467. Exit;
  6468. { Since we've established that the combined shift is within
  6469. limits, we can actually combine the adjacent SHR
  6470. instructions even if they're different sizes }
  6471. if not DoNotMerge and (taicpu(hp1).opcode = A_SHR) then
  6472. begin
  6473. hp2 := tai(hp1.Previous);
  6474. DebugMsg(SPeepholeOptimization + 'ShrShr2Shr 1', p);
  6475. Inc(taicpu(p).oper[0]^.val, taicpu(hp1).oper[0]^.val);
  6476. RemoveInstruction(hp1);
  6477. hp1 := hp2;
  6478. { Though p has changed, only the constant has, and its
  6479. effects can still be detected on the next iteration of
  6480. the repeat..until loop }
  6481. Include(OptsToCheck, aoc_ForceNewIteration);
  6482. end;
  6483. { Move onto the next instruction }
  6484. Continue;
  6485. end;
  6486. else
  6487. ;
  6488. end;
  6489. Break;
  6490. until False;
  6491. end;
  6492. function TX86AsmOptimizer.CheckMemoryWrite(var first_mov, second_mov: taicpu): Boolean;
  6493. var
  6494. CurrentRef: TReference;
  6495. FullReg: TRegister;
  6496. hp1, hp2: tai;
  6497. begin
  6498. Result := False;
  6499. if (first_mov.opsize <> S_B) or (second_mov.opsize <> S_B) then
  6500. Exit;
  6501. { We assume you've checked if the operand is actually a reference by
  6502. this point. If it isn't, you'll most likely get an access violation }
  6503. CurrentRef := first_mov.oper[1]^.ref^;
  6504. { Memory must be aligned }
  6505. if (CurrentRef.offset mod 4) <> 0 then
  6506. Exit;
  6507. Inc(CurrentRef.offset);
  6508. CurrentRef.alignment := 1; { Otherwise references_equal will return False }
  6509. if MatchOperand(second_mov.oper[0]^, 0) and
  6510. references_equal(second_mov.oper[1]^.ref^, CurrentRef) and
  6511. GetNextInstruction(second_mov, hp1) and
  6512. (hp1.typ = ait_instruction) and
  6513. (taicpu(hp1).opcode = A_MOV) and
  6514. MatchOpType(taicpu(hp1), top_const, top_ref) and
  6515. (taicpu(hp1).oper[0]^.val = 0) then
  6516. begin
  6517. Inc(CurrentRef.offset);
  6518. CurrentRef.alignment := taicpu(hp1).oper[1]^.ref^.alignment; { Otherwise references_equal might return False }
  6519. FullReg := newreg(R_INTREGISTER,getsupreg(first_mov.oper[0]^.reg), R_SUBD);
  6520. if references_equal(taicpu(hp1).oper[1]^.ref^, CurrentRef) then
  6521. begin
  6522. case taicpu(hp1).opsize of
  6523. S_B:
  6524. if GetNextInstruction(hp1, hp2) and
  6525. MatchInstruction(taicpu(hp2), A_MOV, [S_B]) and
  6526. MatchOpType(taicpu(hp2), top_const, top_ref) and
  6527. (taicpu(hp2).oper[0]^.val = 0) then
  6528. begin
  6529. Inc(CurrentRef.offset);
  6530. CurrentRef.alignment := 1; { Otherwise references_equal will return False }
  6531. if references_equal(taicpu(hp2).oper[1]^.ref^, CurrentRef) and
  6532. (taicpu(hp2).opsize = S_B) then
  6533. begin
  6534. RemoveInstruction(hp1);
  6535. RemoveInstruction(hp2);
  6536. first_mov.opsize := S_L;
  6537. if first_mov.oper[0]^.typ = top_reg then
  6538. begin
  6539. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVb/MOVb -> MOVZX/MOVl', first_mov);
  6540. { Reuse second_mov as a MOVZX instruction }
  6541. second_mov.opcode := A_MOVZX;
  6542. second_mov.opsize := S_BL;
  6543. second_mov.loadreg(0, first_mov.oper[0]^.reg);
  6544. second_mov.loadreg(1, FullReg);
  6545. first_mov.oper[0]^.reg := FullReg;
  6546. asml.Remove(second_mov);
  6547. asml.InsertBefore(second_mov, first_mov);
  6548. end
  6549. else
  6550. { It's a value }
  6551. begin
  6552. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVb/MOVb -> MOVl', first_mov);
  6553. RemoveInstruction(second_mov);
  6554. end;
  6555. Result := True;
  6556. Exit;
  6557. end;
  6558. end;
  6559. S_W:
  6560. begin
  6561. RemoveInstruction(hp1);
  6562. first_mov.opsize := S_L;
  6563. if first_mov.oper[0]^.typ = top_reg then
  6564. begin
  6565. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVw -> MOVZX/MOVl', first_mov);
  6566. { Reuse second_mov as a MOVZX instruction }
  6567. second_mov.opcode := A_MOVZX;
  6568. second_mov.opsize := S_BL;
  6569. second_mov.loadreg(0, first_mov.oper[0]^.reg);
  6570. second_mov.loadreg(1, FullReg);
  6571. first_mov.oper[0]^.reg := FullReg;
  6572. asml.Remove(second_mov);
  6573. asml.InsertBefore(second_mov, first_mov);
  6574. end
  6575. else
  6576. { It's a value }
  6577. begin
  6578. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVw -> MOVl', first_mov);
  6579. RemoveInstruction(second_mov);
  6580. end;
  6581. Result := True;
  6582. Exit;
  6583. end;
  6584. else
  6585. ;
  6586. end;
  6587. end;
  6588. end;
  6589. end;
  6590. function TX86AsmOptimizer.OptPass1FSTP(var p: tai): boolean;
  6591. { returns true if a "continue" should be done after this optimization }
  6592. var
  6593. hp1, hp2, hp3: tai;
  6594. begin
  6595. Result := false;
  6596. hp3 := nil;
  6597. if MatchOpType(taicpu(p),top_ref) and
  6598. GetNextInstruction(p, hp1) and
  6599. (hp1.typ = ait_instruction) and
  6600. (((taicpu(hp1).opcode = A_FLD) and
  6601. (taicpu(p).opcode = A_FSTP)) or
  6602. ((taicpu(p).opcode = A_FISTP) and
  6603. (taicpu(hp1).opcode = A_FILD))) and
  6604. MatchOpType(taicpu(hp1),top_ref) and
  6605. (taicpu(hp1).opsize = taicpu(p).opsize) and
  6606. RefsEqual(taicpu(p).oper[0]^.ref^, taicpu(hp1).oper[0]^.ref^) then
  6607. begin
  6608. { replacing fstp f;fld f by fst f is only valid for extended because of rounding or if fastmath is on }
  6609. if ((taicpu(p).opsize=S_FX) or (cs_opt_fastmath in current_settings.optimizerswitches)) and
  6610. GetNextInstruction(hp1, hp2) and
  6611. (((hp2.typ = ait_instruction) and
  6612. IsExitCode(hp2) and
  6613. (taicpu(p).oper[0]^.ref^.base = current_procinfo.FramePointer) and
  6614. not(assigned(current_procinfo.procdef.funcretsym) and
  6615. (taicpu(p).oper[0]^.ref^.offset < tabstractnormalvarsym(current_procinfo.procdef.funcretsym).localloc.reference.offset)) and
  6616. (taicpu(p).oper[0]^.ref^.index = NR_NO)) or
  6617. { fstp <temp>
  6618. fld <temp>
  6619. <dealloc> <temp>
  6620. }
  6621. ((taicpu(p).oper[0]^.ref^.base = current_procinfo.FramePointer) and
  6622. (taicpu(p).oper[0]^.ref^.index = NR_NO) and
  6623. SetAndTest(FindTempDeAlloc(taicpu(p).oper[0]^.ref^.offset,tai(hp1.next)),hp2) and
  6624. (tai_tempalloc(hp2).temppos=taicpu(p).oper[0]^.ref^.offset) and
  6625. (((taicpu(p).opsize=S_FX) and (tai_tempalloc(hp2).tempsize=16)) or
  6626. ((taicpu(p).opsize in [S_IQ,S_FL]) and (tai_tempalloc(hp2).tempsize=8)) or
  6627. ((taicpu(p).opsize=S_FS) and (tai_tempalloc(hp2).tempsize=4))
  6628. )
  6629. )
  6630. ) then
  6631. begin
  6632. DebugMsg(SPeepholeOptimization + 'FstpFld2<Nop>',p);
  6633. RemoveInstruction(hp1);
  6634. RemoveCurrentP(p, hp2);
  6635. { first case: exit code }
  6636. if hp2.typ = ait_instruction then
  6637. RemoveLastDeallocForFuncRes(p);
  6638. Result := true;
  6639. end
  6640. else
  6641. { we can do this only in fast math mode as fstp is rounding ...
  6642. ... still disabled as it breaks the compiler and/or rtl }
  6643. if { (cs_opt_fastmath in current_settings.optimizerswitches) or }
  6644. { ... or if another fstp equal to the first one follows }
  6645. GetNextInstruction(hp1,hp2) and
  6646. (hp2.typ = ait_instruction) and
  6647. (taicpu(p).opcode=taicpu(hp2).opcode) and
  6648. (taicpu(p).opsize=taicpu(hp2).opsize) then
  6649. begin
  6650. if (taicpu(p).oper[0]^.ref^.base = current_procinfo.FramePointer) and
  6651. (taicpu(p).oper[0]^.ref^.index = NR_NO) and
  6652. SetAndTest(FindTempDeAlloc(taicpu(p).oper[0]^.ref^.offset,tai(hp2.next)),hp3) and
  6653. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[0]^) and
  6654. (tai_tempalloc(hp3).temppos=taicpu(p).oper[0]^.ref^.offset) and
  6655. (((taicpu(p).opsize=S_FX) and (tai_tempalloc(hp3).tempsize=16)) or
  6656. ((taicpu(p).opsize in [S_IQ,S_FL]) and (tai_tempalloc(hp3).tempsize=8)) or
  6657. ((taicpu(p).opsize=S_FS) and (tai_tempalloc(hp3).tempsize=4))
  6658. ) then
  6659. begin
  6660. DebugMsg(SPeepholeOptimization + 'FstpFldFstp2Fstp',p);
  6661. RemoveCurrentP(p,hp2);
  6662. RemoveInstruction(hp1);
  6663. Result := true;
  6664. end
  6665. else if { fst can't store an extended/comp value }
  6666. (taicpu(p).opsize <> S_FX) and
  6667. (taicpu(p).opsize <> S_IQ) then
  6668. begin
  6669. if (taicpu(p).opcode = A_FSTP) then
  6670. taicpu(p).opcode := A_FST
  6671. else
  6672. taicpu(p).opcode := A_FIST;
  6673. DebugMsg(SPeepholeOptimization + 'FstpFld2Fst',p);
  6674. RemoveInstruction(hp1);
  6675. Result := true;
  6676. end;
  6677. end;
  6678. end;
  6679. end;
  6680. function TX86AsmOptimizer.OptPass1FLD(var p : tai) : boolean;
  6681. var
  6682. hp1, hp2, hp3: tai;
  6683. begin
  6684. result:=false;
  6685. if MatchOpType(taicpu(p),top_reg) and
  6686. GetNextInstruction(p, hp1) and
  6687. (hp1.typ = Ait_Instruction) and
  6688. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  6689. (taicpu(hp1).oper[0]^.reg = NR_ST) and
  6690. (taicpu(hp1).oper[1]^.reg = NR_ST1) then
  6691. { change to
  6692. fld reg fxxx reg,st
  6693. fxxxp st, st1 (hp1)
  6694. Remark: non commutative operations must be reversed!
  6695. }
  6696. begin
  6697. case taicpu(hp1).opcode Of
  6698. A_FMULP,A_FADDP,
  6699. A_FSUBP,A_FDIVP,A_FSUBRP,A_FDIVRP:
  6700. begin
  6701. case taicpu(hp1).opcode Of
  6702. A_FADDP: taicpu(hp1).opcode := A_FADD;
  6703. A_FMULP: taicpu(hp1).opcode := A_FMUL;
  6704. A_FSUBP: taicpu(hp1).opcode := A_FSUBR;
  6705. A_FSUBRP: taicpu(hp1).opcode := A_FSUB;
  6706. A_FDIVP: taicpu(hp1).opcode := A_FDIVR;
  6707. A_FDIVRP: taicpu(hp1).opcode := A_FDIV;
  6708. else
  6709. internalerror(2019050534);
  6710. end;
  6711. taicpu(hp1).oper[0]^.reg := taicpu(p).oper[0]^.reg;
  6712. taicpu(hp1).oper[1]^.reg := NR_ST;
  6713. DebugMsg(SPeepholeOptimization + 'FldF*p2F*',hp1);
  6714. RemoveCurrentP(p, hp1);
  6715. Result:=true;
  6716. exit;
  6717. end;
  6718. else
  6719. ;
  6720. end;
  6721. end
  6722. else
  6723. if MatchOpType(taicpu(p),top_ref) and
  6724. GetNextInstruction(p, hp2) and
  6725. (hp2.typ = Ait_Instruction) and
  6726. MatchOpType(taicpu(hp2),top_reg,top_reg) and
  6727. (taicpu(p).opsize in [S_FS, S_FL]) and
  6728. (taicpu(hp2).oper[0]^.reg = NR_ST) and
  6729. (taicpu(hp2).oper[1]^.reg = NR_ST1) then
  6730. if GetLastInstruction(p, hp1) and
  6731. MatchInstruction(hp1,A_FLD,A_FST,[taicpu(p).opsize]) and
  6732. MatchOpType(taicpu(hp1),top_ref) and
  6733. RefsEqual(taicpu(p).oper[0]^.ref^, taicpu(hp1).oper[0]^.ref^) then
  6734. if ((taicpu(hp2).opcode = A_FMULP) or
  6735. (taicpu(hp2).opcode = A_FADDP)) then
  6736. { change to
  6737. fld/fst mem1 (hp1) fld/fst mem1
  6738. fld mem1 (p) fadd/
  6739. faddp/ fmul st, st
  6740. fmulp st, st1 (hp2) }
  6741. begin
  6742. DebugMsg(SPeepholeOptimization + 'Fld/FstFldFaddp/Fmulp2Fld/FstFadd/Fmul',hp1);
  6743. RemoveCurrentP(p, hp1);
  6744. if (taicpu(hp2).opcode = A_FADDP) then
  6745. taicpu(hp2).opcode := A_FADD
  6746. else
  6747. taicpu(hp2).opcode := A_FMUL;
  6748. taicpu(hp2).oper[1]^.reg := NR_ST;
  6749. end
  6750. else
  6751. { change to
  6752. fld/fst mem1 (hp1) fld/fst mem1
  6753. fld mem1 (p) fld st
  6754. }
  6755. begin
  6756. DebugMsg(SPeepholeOptimization + 'Fld/Fst<mem>Fld<mem>2Fld/Fst<mem>Fld<reg>',hp1);
  6757. taicpu(p).changeopsize(S_FL);
  6758. taicpu(p).loadreg(0,NR_ST);
  6759. end
  6760. else
  6761. begin
  6762. case taicpu(hp2).opcode Of
  6763. A_FMULP,A_FADDP,A_FSUBP,A_FDIVP,A_FSUBRP,A_FDIVRP:
  6764. { change to
  6765. fld/fst mem1 (hp1) fld/fst mem1
  6766. fld mem2 (p) fxxx mem2
  6767. fxxxp st, st1 (hp2) }
  6768. begin
  6769. case taicpu(hp2).opcode Of
  6770. A_FADDP: taicpu(p).opcode := A_FADD;
  6771. A_FMULP: taicpu(p).opcode := A_FMUL;
  6772. A_FSUBP: taicpu(p).opcode := A_FSUBR;
  6773. A_FSUBRP: taicpu(p).opcode := A_FSUB;
  6774. A_FDIVP: taicpu(p).opcode := A_FDIVR;
  6775. A_FDIVRP: taicpu(p).opcode := A_FDIV;
  6776. else
  6777. internalerror(2019050533);
  6778. end;
  6779. DebugMsg(SPeepholeOptimization + 'Fld/FstFldF*2Fld/FstF*',p);
  6780. RemoveInstruction(hp2);
  6781. end
  6782. else
  6783. ;
  6784. end
  6785. end
  6786. end;
  6787. function IsCmpSubset(cond1, cond2: TAsmCond): Boolean; inline;
  6788. begin
  6789. Result := condition_in(cond1, cond2) or
  6790. { Not strictly subsets due to the actual flags checked, but because we're
  6791. comparing integers, E is a subset of AE and GE and their aliases }
  6792. ((cond1 in [C_E, C_Z]) and (cond2 in [C_AE, C_NB, C_NC, C_GE, C_NL]));
  6793. end;
  6794. function TX86AsmOptimizer.OptPass1Cmp(var p: tai): boolean;
  6795. var
  6796. v: TCGInt;
  6797. hp1, hp2, p_dist, p_jump, hp1_dist, p_label, hp1_label: tai;
  6798. FirstMatch, TempBool: Boolean;
  6799. NewReg: TRegister;
  6800. JumpLabel, JumpLabel_dist, JumpLabel_far: TAsmLabel;
  6801. begin
  6802. Result:=false;
  6803. { All these optimisations need a next instruction }
  6804. if not GetNextInstruction(p, hp1) then
  6805. Exit;
  6806. { Search for:
  6807. cmp ###,###
  6808. j(c1) @lbl1
  6809. ...
  6810. @lbl:
  6811. cmp ###,### (same comparison as above)
  6812. j(c2) @lbl2
  6813. If c1 is a subset of c2, change to:
  6814. cmp ###,###
  6815. j(c1) @lbl2
  6816. (@lbl1 may become a dead label as a result)
  6817. }
  6818. { Also handle cases where there are multiple jumps in a row }
  6819. p_jump := hp1;
  6820. while Assigned(p_jump) and MatchInstruction(p_jump, A_JCC, []) do
  6821. begin
  6822. if IsJumpToLabel(taicpu(p_jump)) then
  6823. begin
  6824. { Do jump optimisations first in case the condition becomes
  6825. unnecessary }
  6826. TempBool := True;
  6827. if DoJumpOptimizations(p_jump, TempBool) or
  6828. not TempBool then
  6829. begin
  6830. if Assigned(p_jump) then
  6831. begin
  6832. hp1 := p_jump;
  6833. if (p_jump.typ in [ait_align]) then
  6834. SkipAligns(p_jump, p_jump);
  6835. { CollapseZeroDistJump will be set to the label after the
  6836. jump if it optimises, whether or not it's live or dead }
  6837. if (p_jump.typ in [ait_label]) and
  6838. not (tai_label(p_jump).labsym.is_used) then
  6839. GetNextInstruction(p_jump, p_jump);
  6840. end;
  6841. TransferUsedRegs(TmpUsedRegs);
  6842. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  6843. if not Assigned(p_jump) or
  6844. (
  6845. not MatchInstruction(p_jump, A_Jcc, A_SETcc, A_CMOVcc, []) and
  6846. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, p_jump, TmpUsedRegs)
  6847. ) then
  6848. begin
  6849. { No more conditional jumps; conditional statement is no longer required }
  6850. DebugMsg(SPeepholeOptimization + 'Removed unnecessary condition (Cmp2Nop)', p);
  6851. RemoveCurrentP(p);
  6852. Result := True;
  6853. Exit;
  6854. end;
  6855. hp1 := p_jump;
  6856. Include(OptsToCheck, aoc_ForceNewIteration);
  6857. Continue;
  6858. end;
  6859. JumpLabel := TAsmLabel(taicpu(p_jump).oper[0]^.ref^.symbol);
  6860. if GetNextInstruction(p_jump, hp2) and
  6861. (
  6862. OptimizeConditionalJump(JumpLabel, p_jump, hp2, TempBool) or
  6863. not TempBool
  6864. ) then
  6865. begin
  6866. hp1 := p_jump;
  6867. Include(OptsToCheck, aoc_ForceNewIteration);
  6868. Continue;
  6869. end;
  6870. p_label := nil;
  6871. if Assigned(JumpLabel) then
  6872. p_label := getlabelwithsym(JumpLabel);
  6873. if Assigned(p_label) and
  6874. GetNextInstruction(p_label, p_dist) and
  6875. MatchInstruction(p_dist, A_CMP, []) and
  6876. MatchOperand(taicpu(p_dist).oper[0]^, taicpu(p).oper[0]^) and
  6877. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p).oper[1]^) and
  6878. GetNextInstruction(p_dist, hp1_dist) and
  6879. MatchInstruction(hp1_dist, A_JCC, []) then { This doesn't have to be an explicit label }
  6880. begin
  6881. JumpLabel_dist := TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol);
  6882. if JumpLabel = JumpLabel_dist then
  6883. { This is an infinite loop }
  6884. Exit;
  6885. { Best optimisation when the first condition is a subset (or equal) of the second }
  6886. if IsCmpSubset(taicpu(p_jump).condition, taicpu(hp1_dist).condition) then
  6887. begin
  6888. { Any registers used here will already be allocated }
  6889. if Assigned(JumpLabel) then
  6890. JumpLabel.DecRefs;
  6891. DebugMsg(SPeepholeOptimization + 'CMP/Jcc/@Lbl/CMP/Jcc -> CMP/Jcc, redirecting first jump', p_jump);
  6892. taicpu(p_jump).loadref(0, taicpu(hp1_dist).oper[0]^.ref^); { This also increases the reference count }
  6893. Result := True;
  6894. { Don't exit yet. Since p and p_jump haven't actually been
  6895. removed, we can check for more on this iteration }
  6896. end
  6897. else if IsCmpSubset(taicpu(hp1_dist).condition, inverse_cond(taicpu(p_jump).condition)) and
  6898. GetNextInstruction(hp1_dist, hp1_label) and
  6899. SkipAligns(hp1_label, hp1_label) and
  6900. (hp1_label.typ = ait_label) then
  6901. begin
  6902. JumpLabel_far := tai_label(hp1_label).labsym;
  6903. if (JumpLabel_far = JumpLabel_dist) or (JumpLabel_far = JumpLabel) then
  6904. { This is an infinite loop }
  6905. Exit;
  6906. if Assigned(JumpLabel_far) then
  6907. begin
  6908. { In this situation, if the first jump branches, the second one will never,
  6909. branch so change the destination label to after the second jump }
  6910. DebugMsg(SPeepholeOptimization + 'CMP/Jcc/@Lbl/CMP/Jcc/@Lbl -> CMP/Jcc, redirecting first jump to 2nd label', p_jump);
  6911. if Assigned(JumpLabel) then
  6912. JumpLabel.DecRefs;
  6913. JumpLabel_far.IncRefs;
  6914. taicpu(p_jump).oper[0]^.ref^.symbol := JumpLabel_far;
  6915. Result := True;
  6916. { Don't exit yet. Since p and p_jump haven't actually been
  6917. removed, we can check for more on this iteration }
  6918. Continue;
  6919. end;
  6920. end;
  6921. end;
  6922. end;
  6923. { Search for:
  6924. cmp ###,###
  6925. j(c1) @lbl1
  6926. cmp ###,### (same as first)
  6927. Remove second cmp
  6928. }
  6929. if GetNextInstruction(p_jump, hp2) and
  6930. (
  6931. (
  6932. MatchInstruction(hp2, A_CMP, [taicpu(p).opsize]) and
  6933. (
  6934. (
  6935. MatchOpType(taicpu(p), top_const, top_reg) and
  6936. MatchOpType(taicpu(hp2), top_const, top_reg) and
  6937. (taicpu(hp2).oper[0]^.val = taicpu(p).oper[0]^.val) and
  6938. Reg1WriteOverwritesReg2Entirely(taicpu(hp2).oper[1]^.reg, taicpu(p).oper[1]^.reg)
  6939. ) or (
  6940. MatchOperand(taicpu(hp2).oper[0]^, taicpu(p).oper[0]^) and
  6941. MatchOperand(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^)
  6942. )
  6943. )
  6944. ) or (
  6945. { Also match cmp $0,%reg; jcc @lbl; test %reg,%reg }
  6946. MatchOperand(taicpu(p).oper[0]^, 0) and
  6947. (taicpu(p).oper[1]^.typ = top_reg) and
  6948. MatchInstruction(hp2, A_TEST, []) and
  6949. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  6950. (taicpu(hp2).oper[0]^.reg = taicpu(hp2).oper[1]^.reg) and
  6951. Reg1WriteOverwritesReg2Entirely(taicpu(hp2).oper[1]^.reg, taicpu(p).oper[1]^.reg)
  6952. )
  6953. ) then
  6954. begin
  6955. DebugMsg(SPeepholeOptimization + 'CMP/Jcc/CMP; removed superfluous CMP', hp2);
  6956. RemoveInstruction(hp2);
  6957. Result := True;
  6958. { Continue the while loop in case "Jcc/CMP" follows the second CMP that was just removed }
  6959. end;
  6960. GetNextInstruction(p_jump, p_jump);
  6961. end;
  6962. if (
  6963. { Don't call GetNextInstruction again if we already have it }
  6964. (hp1 = p_jump) or
  6965. GetNextInstruction(p, hp1)
  6966. ) and
  6967. MatchInstruction(hp1, A_Jcc, []) and
  6968. IsJumpToLabel(taicpu(hp1)) and
  6969. (taicpu(hp1).condition in [C_E, C_Z, C_NE, C_NZ]) and
  6970. GetNextInstruction(hp1, hp2) then
  6971. begin
  6972. {
  6973. cmp x, y (or "cmp y, x")
  6974. je @lbl
  6975. mov x, y
  6976. @lbl:
  6977. (x and y can be constants, registers or references)
  6978. Change to:
  6979. mov x, y (x and y will always be equal in the end)
  6980. @lbl: (may beceome a dead label)
  6981. Also:
  6982. cmp x, y (or "cmp y, x")
  6983. jne @lbl
  6984. mov x, y
  6985. @lbl:
  6986. (x and y can be constants, registers or references)
  6987. Change to:
  6988. Absolutely nothing! (Except @lbl if it's still live)
  6989. }
  6990. if MatchInstruction(hp2, A_MOV, [taicpu(p).opsize]) and
  6991. (
  6992. (
  6993. MatchOperand(taicpu(p).oper[0]^, taicpu(hp2).oper[0]^) and
  6994. MatchOperand(taicpu(p).oper[1]^, taicpu(hp2).oper[1]^)
  6995. ) or (
  6996. MatchOperand(taicpu(p).oper[0]^, taicpu(hp2).oper[1]^) and
  6997. MatchOperand(taicpu(p).oper[1]^, taicpu(hp2).oper[0]^)
  6998. )
  6999. ) and
  7000. GetNextInstruction(hp2, hp1_label) and
  7001. SkipAligns(hp1_label, hp1_label) and
  7002. (hp1_label.typ = ait_label) and
  7003. (tai_label(hp1_label).labsym = taicpu(hp1).oper[0]^.ref^.symbol) then
  7004. begin
  7005. tai_label(hp1_label).labsym.DecRefs;
  7006. if (taicpu(hp1).condition in [C_NE, C_NZ]) then
  7007. begin
  7008. DebugMsg(SPeepholeOptimization + 'CMP/JNE/MOV/@Lbl -> NOP, since the MOV is only executed if the operands are equal (CmpJneMov2Nop)', p);
  7009. RemoveInstruction(hp2);
  7010. hp2 := hp1_label; { So RemoveCurrentp below can be set to something valid }
  7011. end
  7012. else
  7013. DebugMsg(SPeepholeOptimization + 'CMP/JE/MOV/@Lbl -> MOV, since the MOV is only executed if the operands aren''t equal (CmpJeMov2Mov)', p);
  7014. RemoveInstruction(hp1);
  7015. RemoveCurrentp(p, hp2);
  7016. Result := True;
  7017. Exit;
  7018. end;
  7019. {
  7020. Try to optimise the following:
  7021. cmp $x,### ($x and $y can be registers or constants)
  7022. je @lbl1 (only reference)
  7023. cmp $y,### (### are identical)
  7024. @Lbl:
  7025. sete %reg1
  7026. Change to:
  7027. cmp $x,###
  7028. sete %reg2 (allocate new %reg2)
  7029. cmp $y,###
  7030. sete %reg1
  7031. orb %reg2,%reg1
  7032. (dealloc %reg2)
  7033. This adds an instruction (so don't perform under -Os), but it removes
  7034. a conditional branch.
  7035. }
  7036. if not (cs_opt_size in current_settings.optimizerswitches) and
  7037. MatchInstruction(hp2, A_CMP, A_TEST, [taicpu(p).opsize]) and
  7038. MatchOperand(taicpu(p).oper[1]^, taicpu(hp2).oper[1]^) and
  7039. { The first operand of CMP instructions can only be a register or
  7040. immediate anyway, so no need to check }
  7041. GetNextInstruction(hp2, p_label) and
  7042. (p_label.typ = ait_label) and
  7043. (tai_label(p_label).labsym.getrefs = 1) and
  7044. (JumpTargetOp(taicpu(hp1))^.ref^.symbol = tai_label(p_label).labsym) and
  7045. GetNextInstruction(p_label, p_dist) and
  7046. MatchInstruction(p_dist, A_SETcc, []) and
  7047. (taicpu(p_dist).condition in [C_E, C_Z]) and
  7048. (taicpu(p_dist).oper[0]^.typ = top_reg) then
  7049. begin
  7050. TransferUsedRegs(TmpUsedRegs);
  7051. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  7052. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  7053. UpdateUsedRegs(TmpUsedRegs, tai(p_label.Next));
  7054. UpdateUsedRegs(TmpUsedRegs, tai(p_dist.Next));
  7055. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  7056. { Get the instruction after the SETcc instruction so we can
  7057. allocate a new register over the entire range }
  7058. GetNextInstruction(p_dist, hp1_dist) then
  7059. begin
  7060. { Register can appear in p if it's not used afterwards, so only
  7061. allocate between hp1 and hp1_dist }
  7062. NewReg := GetIntRegisterBetween(R_SUBL, TmpUsedRegs, hp1, hp1_dist);
  7063. if NewReg <> NR_NO then
  7064. begin
  7065. DebugMsg(SPeepholeOptimization + 'CMP/JE/CMP/@Lbl/SETE -> CMP/SETE/CMP/SETE/OR, removing conditional branch', p);
  7066. { Change the jump instruction into a SETcc instruction }
  7067. taicpu(hp1).opcode := A_SETcc;
  7068. taicpu(hp1).opsize := S_B;
  7069. taicpu(hp1).loadreg(0, NewReg);
  7070. { This is now a dead label }
  7071. tai_label(p_label).labsym.decrefs;
  7072. { Prefer adding before the next instruction so the FLAGS
  7073. register is deallicated first }
  7074. AsmL.InsertBefore(
  7075. taicpu.op_reg_reg(A_OR, S_B, NewReg, taicpu(p_dist).oper[0]^.reg),
  7076. hp1_dist
  7077. );
  7078. Result := True;
  7079. { Don't exit yet, as p wasn't changed and hp1, while
  7080. modified, is still intact and might be optimised by the
  7081. SETcc optimisation below }
  7082. end;
  7083. end;
  7084. end;
  7085. end;
  7086. if taicpu(p).oper[0]^.typ = top_const then
  7087. begin
  7088. if (taicpu(p).oper[0]^.val = 0) and
  7089. (taicpu(p).oper[1]^.typ = top_reg) and
  7090. MatchInstruction(hp1,A_Jcc,A_SETcc,[]) then
  7091. begin
  7092. hp2 := p;
  7093. FirstMatch := True;
  7094. { When dealing with "cmp $0,%reg", only ZF and SF contain
  7095. anything meaningful once it's converted to "test %reg,%reg";
  7096. additionally, some jumps will always (or never) branch, so
  7097. evaluate every jump immediately following the
  7098. comparison, optimising the conditions if possible.
  7099. Similarly with SETcc... those that are always set to 0 or 1
  7100. are changed to MOV instructions }
  7101. while FirstMatch or { Saves calling GetNextInstruction unnecessarily }
  7102. (
  7103. GetNextInstruction(hp2, hp1) and
  7104. MatchInstruction(hp1,A_Jcc,A_SETcc,[])
  7105. ) do
  7106. begin
  7107. FirstMatch := False;
  7108. case taicpu(hp1).condition of
  7109. C_B, C_C, C_NAE, C_O:
  7110. { For B/NAE:
  7111. Will never branch since an unsigned integer can never be below zero
  7112. For C/O:
  7113. Result cannot overflow because 0 is being subtracted
  7114. }
  7115. begin
  7116. if taicpu(hp1).opcode = A_Jcc then
  7117. begin
  7118. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition B/C/NAE/O --> Never (jump removed)', hp1);
  7119. TAsmLabel(taicpu(hp1).oper[0]^.ref^.symbol).decrefs;
  7120. RemoveInstruction(hp1);
  7121. { Since hp1 was deleted, hp2 must not be updated }
  7122. Continue;
  7123. end
  7124. else
  7125. begin
  7126. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition B/C/NAE/O --> Never (set -> mov 0)', hp1);
  7127. { Convert "set(c) %reg" instruction to "movb 0,%reg" }
  7128. taicpu(hp1).opcode := A_MOV;
  7129. taicpu(hp1).ops := 2;
  7130. taicpu(hp1).condition := C_None;
  7131. taicpu(hp1).opsize := S_B;
  7132. taicpu(hp1).loadreg(1,taicpu(hp1).oper[0]^.reg);
  7133. taicpu(hp1).loadconst(0, 0);
  7134. end;
  7135. end;
  7136. C_BE, C_NA:
  7137. begin
  7138. { Will only branch if equal to zero }
  7139. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition BE/NA --> E', hp1);
  7140. taicpu(hp1).condition := C_E;
  7141. end;
  7142. C_A, C_NBE:
  7143. begin
  7144. { Will only branch if not equal to zero }
  7145. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition A/NBE --> NE', hp1);
  7146. taicpu(hp1).condition := C_NE;
  7147. end;
  7148. C_AE, C_NB, C_NC, C_NO:
  7149. begin
  7150. { Will always branch }
  7151. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition AE/NB/NC/NO --> Always', hp1);
  7152. if taicpu(hp1).opcode = A_Jcc then
  7153. begin
  7154. MakeUnconditional(taicpu(hp1));
  7155. { Any jumps/set that follow will now be dead code }
  7156. RemoveDeadCodeAfterJump(taicpu(hp1));
  7157. Break;
  7158. end
  7159. else
  7160. begin
  7161. { Convert "set(c) %reg" instruction to "movb 1,%reg" }
  7162. taicpu(hp1).opcode := A_MOV;
  7163. taicpu(hp1).ops := 2;
  7164. taicpu(hp1).condition := C_None;
  7165. taicpu(hp1).opsize := S_B;
  7166. taicpu(hp1).loadreg(1,taicpu(hp1).oper[0]^.reg);
  7167. taicpu(hp1).loadconst(0, 1);
  7168. end;
  7169. end;
  7170. C_None:
  7171. InternalError(2020012201);
  7172. C_P, C_PE, C_NP, C_PO:
  7173. { We can't handle parity checks and they should never be generated
  7174. after a general-purpose CMP (it's used in some floating-point
  7175. comparisons that don't use CMP) }
  7176. InternalError(2020012202);
  7177. else
  7178. { Zero/Equality, Sign, their complements and all of the
  7179. signed comparisons do not need to be converted };
  7180. end;
  7181. hp2 := hp1;
  7182. end;
  7183. { Convert the instruction to a TEST }
  7184. taicpu(p).opcode := A_TEST;
  7185. taicpu(p).loadreg(0,taicpu(p).oper[1]^.reg);
  7186. Result := True;
  7187. Exit;
  7188. end
  7189. else if (taicpu(p).oper[0]^.val = 1) and
  7190. MatchInstruction(hp1,A_Jcc,A_SETcc,[]) and
  7191. (taicpu(hp1).condition in [C_L, C_NL, C_NGE, C_GE]) then
  7192. begin
  7193. { Convert; To:
  7194. cmp $1,r/m cmp $0,r/m
  7195. jl @lbl jle @lbl
  7196. (Also do inverted conditions)
  7197. }
  7198. DebugMsg(SPeepholeOptimization + 'Cmp1Jl2Cmp0Jle', p);
  7199. taicpu(p).oper[0]^.val := 0;
  7200. if taicpu(hp1).condition in [C_L, C_NGE] then
  7201. taicpu(hp1).condition := C_LE
  7202. else
  7203. taicpu(hp1).condition := C_NLE;
  7204. { If the instruction is now "cmp $0,%reg", convert it to a
  7205. TEST (and effectively do the work of the "cmp $0,%reg" in
  7206. the block above)
  7207. }
  7208. if (taicpu(p).oper[1]^.typ = top_reg) then
  7209. begin
  7210. taicpu(p).opcode := A_TEST;
  7211. taicpu(p).loadreg(0,taicpu(p).oper[1]^.reg);
  7212. end;
  7213. Result := True;
  7214. Exit;
  7215. end
  7216. else if (taicpu(p).oper[1]^.typ = top_reg)
  7217. {$ifdef x86_64}
  7218. and (taicpu(p).opsize <> S_Q) { S_Q will never happen: cmp with 64 bit constants is not possible }
  7219. {$endif x86_64}
  7220. then
  7221. begin
  7222. { cmp register,$8000 neg register
  7223. je target --> jo target
  7224. .... only if register is deallocated before jump.}
  7225. case Taicpu(p).opsize of
  7226. S_B: v:=$80;
  7227. S_W: v:=$8000;
  7228. S_L: v:=qword($80000000);
  7229. else
  7230. internalerror(2013112905);
  7231. end;
  7232. if (taicpu(p).oper[0]^.val=v) and
  7233. MatchInstruction(hp1,A_Jcc,A_SETcc,[]) and
  7234. (Taicpu(hp1).condition in [C_E,C_NE]) then
  7235. begin
  7236. TransferUsedRegs(TmpUsedRegs);
  7237. UpdateUsedRegs(TmpUsedRegs,tai(p.next));
  7238. if not(RegInUsedRegs(Taicpu(p).oper[1]^.reg, TmpUsedRegs)) then
  7239. begin
  7240. DebugMsg(SPeepholeOptimization + 'CmpJe2NegJo done',p);
  7241. Taicpu(p).opcode:=A_NEG;
  7242. Taicpu(p).loadoper(0,Taicpu(p).oper[1]^);
  7243. Taicpu(p).clearop(1);
  7244. Taicpu(p).ops:=1;
  7245. if Taicpu(hp1).condition=C_E then
  7246. Taicpu(hp1).condition:=C_O
  7247. else
  7248. Taicpu(hp1).condition:=C_NO;
  7249. Result:=true;
  7250. exit;
  7251. end;
  7252. end;
  7253. end;
  7254. end;
  7255. if TrySwapMovCmp(p, hp1) then
  7256. begin
  7257. Result := True;
  7258. Exit;
  7259. end;
  7260. end;
  7261. function TX86AsmOptimizer.OptPass1PXor(var p: tai): boolean;
  7262. var
  7263. hp1: tai;
  7264. begin
  7265. {
  7266. remove the second (v)pxor from
  7267. pxor reg,reg
  7268. ...
  7269. pxor reg,reg
  7270. }
  7271. Result:=false;
  7272. if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^) and
  7273. MatchOpType(taicpu(p),top_reg,top_reg) and
  7274. GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[0]^.reg) and
  7275. MatchInstruction(hp1,taicpu(p).opcode,[taicpu(p).opsize]) and
  7276. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[0]^) and
  7277. MatchOperand(taicpu(hp1).oper[0]^,taicpu(hp1).oper[1]^) then
  7278. begin
  7279. DebugMsg(SPeepholeOptimization + 'PXorPXor2PXor done',hp1);
  7280. RemoveInstruction(hp1);
  7281. Result:=true;
  7282. Exit;
  7283. end
  7284. {
  7285. replace
  7286. pxor reg1,reg1
  7287. movapd/s reg1,reg2
  7288. dealloc reg1
  7289. by
  7290. pxor reg2,reg2
  7291. }
  7292. else if GetNextInstruction(p,hp1) and
  7293. { we mix single and double opperations here because we assume that the compiler
  7294. generates vmovapd only after double operations and vmovaps only after single operations }
  7295. MatchInstruction(hp1,A_MOVAPD,A_MOVAPS,[S_NO]) and
  7296. MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^) and
  7297. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  7298. (taicpu(p).oper[0]^.typ=top_reg) then
  7299. begin
  7300. TransferUsedRegs(TmpUsedRegs);
  7301. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  7302. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  7303. begin
  7304. taicpu(p).loadoper(0,taicpu(hp1).oper[1]^);
  7305. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  7306. DebugMsg(SPeepholeOptimization + 'PXorMovapd2PXor done',p);
  7307. RemoveInstruction(hp1);
  7308. result:=true;
  7309. end;
  7310. end;
  7311. end;
  7312. function TX86AsmOptimizer.OptPass1VPXor(var p: tai): boolean;
  7313. var
  7314. hp1: tai;
  7315. begin
  7316. {
  7317. remove the second (v)pxor from
  7318. (v)pxor reg,reg
  7319. ...
  7320. (v)pxor reg,reg
  7321. }
  7322. Result:=false;
  7323. if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^,taicpu(p).oper[2]^) and
  7324. MatchOpType(taicpu(p),top_reg,top_reg,top_reg) then
  7325. begin
  7326. if GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[0]^.reg) and
  7327. MatchInstruction(hp1,taicpu(p).opcode,[taicpu(p).opsize]) and
  7328. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[0]^) and
  7329. MatchOperand(taicpu(hp1).oper[0]^,taicpu(hp1).oper[1]^,taicpu(hp1).oper[2]^) then
  7330. begin
  7331. DebugMsg(SPeepholeOptimization + 'VPXorVPXor2VPXor done',hp1);
  7332. RemoveInstruction(hp1);
  7333. Result:=true;
  7334. Exit;
  7335. end;
  7336. {$ifdef x86_64}
  7337. {
  7338. replace
  7339. vpxor reg1,reg1,reg1
  7340. vmov reg,mem
  7341. by
  7342. movq $0,mem
  7343. }
  7344. if GetNextInstruction(p,hp1) and
  7345. MatchInstruction(hp1,A_VMOVSD,[]) and
  7346. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  7347. MatchOpType(taicpu(hp1),top_reg,top_ref) then
  7348. begin
  7349. TransferUsedRegs(TmpUsedRegs);
  7350. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  7351. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  7352. begin
  7353. taicpu(hp1).loadconst(0,0);
  7354. taicpu(hp1).opcode:=A_MOV;
  7355. taicpu(hp1).opsize:=S_Q;
  7356. DebugMsg(SPeepholeOptimization + 'VPXorVMov2Mov done',p);
  7357. RemoveCurrentP(p);
  7358. result:=true;
  7359. Exit;
  7360. end;
  7361. end;
  7362. {$endif x86_64}
  7363. end
  7364. {
  7365. replace
  7366. vpxor reg1,reg1,reg2
  7367. by
  7368. vpxor reg2,reg2,reg2
  7369. to avoid unncessary data dependencies
  7370. }
  7371. else if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^) and
  7372. MatchOpType(taicpu(p),top_reg,top_reg,top_reg) then
  7373. begin
  7374. DebugMsg(SPeepholeOptimization + 'VPXor2VPXor done',p);
  7375. { avoid unncessary data dependency }
  7376. taicpu(p).loadreg(0,taicpu(p).oper[2]^.reg);
  7377. taicpu(p).loadreg(1,taicpu(p).oper[2]^.reg);
  7378. result:=true;
  7379. exit;
  7380. end;
  7381. Result:=OptPass1VOP(p);
  7382. end;
  7383. function TX86AsmOptimizer.OptPass1Imul(var p: tai): boolean;
  7384. var
  7385. hp1 : tai;
  7386. begin
  7387. result:=false;
  7388. { replace
  7389. IMul const,%mreg1,%mreg2
  7390. Mov %reg2,%mreg3
  7391. dealloc %mreg3
  7392. by
  7393. Imul const,%mreg1,%mreg23
  7394. }
  7395. if (taicpu(p).ops=3) and
  7396. GetNextInstruction(p,hp1) and
  7397. MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) and
  7398. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  7399. (taicpu(hp1).oper[1]^.typ=top_reg) then
  7400. begin
  7401. TransferUsedRegs(TmpUsedRegs);
  7402. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  7403. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  7404. begin
  7405. taicpu(p).loadoper(2,taicpu(hp1).oper[1]^);
  7406. DebugMsg(SPeepholeOptimization + 'ImulMov2Imul done',p);
  7407. RemoveInstruction(hp1);
  7408. result:=true;
  7409. end;
  7410. end;
  7411. end;
  7412. function TX86AsmOptimizer.OptPass1SHXX(var p: tai): boolean;
  7413. var
  7414. hp1 : tai;
  7415. begin
  7416. result:=false;
  7417. { replace
  7418. IMul %reg0,%reg1,%reg2
  7419. Mov %reg2,%reg3
  7420. dealloc %reg2
  7421. by
  7422. Imul %reg0,%reg1,%reg3
  7423. }
  7424. if GetNextInstruction(p,hp1) and
  7425. MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) and
  7426. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  7427. (taicpu(hp1).oper[1]^.typ=top_reg) then
  7428. begin
  7429. TransferUsedRegs(TmpUsedRegs);
  7430. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  7431. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  7432. begin
  7433. taicpu(p).loadoper(2,taicpu(hp1).oper[1]^);
  7434. DebugMsg(SPeepholeOptimization + 'SHXXMov2SHXX done',p);
  7435. RemoveInstruction(hp1);
  7436. result:=true;
  7437. end;
  7438. end;
  7439. end;
  7440. function TX86AsmOptimizer.OptPass1_V_Cvtss2sd(var p: tai): boolean;
  7441. var
  7442. hp1: tai;
  7443. begin
  7444. Result:=false;
  7445. { get rid of
  7446. (v)cvtss2sd reg0,<reg1,>reg2
  7447. (v)cvtss2sd reg2,<reg2,>reg0
  7448. }
  7449. if GetNextInstruction(p,hp1) and
  7450. (((taicpu(p).opcode=A_CVTSS2SD) and MatchInstruction(hp1,A_CVTSD2SS,[taicpu(p).opsize]) and
  7451. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[1]^) and MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^)) or
  7452. ((taicpu(p).opcode=A_VCVTSS2SD) and MatchInstruction(hp1,A_VCVTSD2SS,[taicpu(p).opsize]) and
  7453. MatchOpType(taicpu(p),top_reg,top_reg,top_reg) and
  7454. MatchOpType(taicpu(hp1),top_reg,top_reg,top_reg) and
  7455. (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) and
  7456. (getsupreg(taicpu(hp1).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg)) and
  7457. (getsupreg(taicpu(p).oper[2]^.reg)=getsupreg(taicpu(hp1).oper[0]^.reg))
  7458. )
  7459. ) then
  7460. begin
  7461. if ((taicpu(p).opcode=A_CVTSS2SD) and (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg))) or
  7462. ((taicpu(p).opcode=A_VCVTSS2SD) and (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[2]^.reg))) then
  7463. begin
  7464. DebugMsg(SPeepholeOptimization + '(V)Cvtss2CvtSd(V)Cvtsd2ss2Nop done',p);
  7465. RemoveCurrentP(p);
  7466. RemoveInstruction(hp1);
  7467. end
  7468. else
  7469. begin
  7470. DebugMsg(SPeepholeOptimization + '(V)Cvtss2CvtSd(V)Cvtsd2ss2Vmovaps done',p);
  7471. if taicpu(hp1).opcode=A_CVTSD2SS then
  7472. begin
  7473. taicpu(p).loadreg(1,taicpu(hp1).oper[1]^.reg);
  7474. taicpu(p).opcode:=A_MOVAPS;
  7475. end
  7476. else
  7477. begin
  7478. taicpu(p).loadreg(1,taicpu(hp1).oper[2]^.reg);
  7479. taicpu(p).opcode:=A_VMOVAPS;
  7480. end;
  7481. taicpu(p).ops:=2;
  7482. RemoveInstruction(hp1);
  7483. end;
  7484. Result:=true;
  7485. Exit;
  7486. end;
  7487. end;
  7488. function TX86AsmOptimizer.OptPass1Jcc(var p : tai) : boolean;
  7489. var
  7490. hp1, hp2, hp3, hp4, hp5, hp6: tai;
  7491. ThisReg: TRegister;
  7492. begin
  7493. Result := False;
  7494. if not GetNextInstruction(p,hp1) then
  7495. Exit;
  7496. {
  7497. convert
  7498. j<c> .L1
  7499. mov 1,reg
  7500. jmp .L2
  7501. .L1
  7502. mov 0,reg
  7503. .L2
  7504. into
  7505. mov 0,reg
  7506. set<not(c)> reg
  7507. take care of alignment and that the mov 0,reg is not converted into a xor as this
  7508. would destroy the flag contents
  7509. Use MOVZX if size is preferred, since while mov 0,reg is bigger, it can be
  7510. executed at the same time as a previous comparison.
  7511. set<not(c)> reg
  7512. movzx reg, reg
  7513. }
  7514. if MatchInstruction(hp1,A_MOV,[]) and
  7515. (taicpu(hp1).oper[0]^.typ = top_const) and
  7516. (
  7517. (
  7518. (taicpu(hp1).oper[1]^.typ = top_reg)
  7519. {$ifdef i386}
  7520. { Under i386, ESI, EDI, EBP and ESP
  7521. don't have an 8-bit representation }
  7522. and not (getsupreg(taicpu(hp1).oper[1]^.reg) in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  7523. {$endif i386}
  7524. ) or (
  7525. {$ifdef i386}
  7526. (taicpu(hp1).oper[1]^.typ <> top_reg) and
  7527. {$endif i386}
  7528. (taicpu(hp1).opsize = S_B)
  7529. )
  7530. ) and
  7531. GetNextInstruction(hp1,hp2) and
  7532. MatchInstruction(hp2,A_JMP,[]) and (taicpu(hp2).oper[0]^.ref^.refaddr=addr_full) and
  7533. GetNextInstruction(hp2,hp3) and
  7534. SkipAligns(hp3, hp3) and
  7535. (hp3.typ=ait_label) and
  7536. (tasmlabel(taicpu(p).oper[0]^.ref^.symbol)=tai_label(hp3).labsym) and
  7537. GetNextInstruction(hp3,hp4) and
  7538. MatchInstruction(hp4,A_MOV,[taicpu(hp1).opsize]) and
  7539. (taicpu(hp4).oper[0]^.typ = top_const) and
  7540. (
  7541. ((taicpu(hp1).oper[0]^.val = 0) and (taicpu(hp4).oper[0]^.val = 1)) or
  7542. ((taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0))
  7543. ) and
  7544. MatchOperand(taicpu(hp1).oper[1]^,taicpu(hp4).oper[1]^) and
  7545. GetNextInstruction(hp4,hp5) and
  7546. SkipAligns(hp5, hp5) and
  7547. (hp5.typ=ait_label) and
  7548. (tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol)=tai_label(hp5).labsym) then
  7549. begin
  7550. if (taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0) then
  7551. taicpu(p).condition := inverse_cond(taicpu(p).condition);
  7552. tai_label(hp3).labsym.DecRefs;
  7553. { If this isn't the only reference to the middle label, we can
  7554. still make a saving - only that the first jump and everything
  7555. that follows will remain. }
  7556. if (tai_label(hp3).labsym.getrefs = 0) then
  7557. begin
  7558. if (taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0) then
  7559. DebugMsg(SPeepholeOptimization + 'J(c)Mov1JmpMov0 -> Set(~c)',p)
  7560. else
  7561. DebugMsg(SPeepholeOptimization + 'J(c)Mov0JmpMov1 -> Set(c)',p);
  7562. { remove jump, first label and second MOV (also catching any aligns) }
  7563. repeat
  7564. if not GetNextInstruction(hp2, hp3) then
  7565. InternalError(2021040810);
  7566. RemoveInstruction(hp2);
  7567. hp2 := hp3;
  7568. until hp2 = hp5;
  7569. { Don't decrement reference count before the removal loop
  7570. above, otherwise GetNextInstruction won't stop on the
  7571. the label }
  7572. tai_label(hp5).labsym.DecRefs;
  7573. end
  7574. else
  7575. begin
  7576. if (taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0) then
  7577. DebugMsg(SPeepholeOptimization + 'J(c)Mov1JmpMov0 -> Set(~c) (partial)',p)
  7578. else
  7579. DebugMsg(SPeepholeOptimization + 'J(c)Mov0JmpMov1 -> Set(c) (partial)',p);
  7580. end;
  7581. taicpu(p).opcode:=A_SETcc;
  7582. taicpu(p).opsize:=S_B;
  7583. taicpu(p).is_jmp:=False;
  7584. if taicpu(hp1).opsize=S_B then
  7585. begin
  7586. taicpu(p).loadoper(0, taicpu(hp1).oper[1]^);
  7587. if taicpu(hp1).oper[1]^.typ = top_reg then
  7588. AllocRegBetween(taicpu(hp1).oper[1]^.reg, p, hp2, UsedRegs);
  7589. RemoveInstruction(hp1);
  7590. end
  7591. else
  7592. begin
  7593. { Will be a register because the size can't be S_B otherwise }
  7594. ThisReg := newreg(R_INTREGISTER,getsupreg(taicpu(hp1).oper[1]^.reg), R_SUBL);
  7595. taicpu(p).loadreg(0, ThisReg);
  7596. AllocRegBetween(ThisReg, p, hp2, UsedRegs);
  7597. if (cs_opt_size in current_settings.optimizerswitches) and IsMOVZXAcceptable then
  7598. begin
  7599. case taicpu(hp1).opsize of
  7600. S_W:
  7601. taicpu(hp1).opsize := S_BW;
  7602. S_L:
  7603. taicpu(hp1).opsize := S_BL;
  7604. {$ifdef x86_64}
  7605. S_Q:
  7606. begin
  7607. taicpu(hp1).opsize := S_BL;
  7608. { Change the destination register to 32-bit }
  7609. taicpu(hp1).loadreg(1, newreg(R_INTREGISTER,getsupreg(ThisReg), R_SUBD));
  7610. end;
  7611. {$endif x86_64}
  7612. else
  7613. InternalError(2021040820);
  7614. end;
  7615. taicpu(hp1).opcode := A_MOVZX;
  7616. taicpu(hp1).loadreg(0, ThisReg);
  7617. end
  7618. else
  7619. begin
  7620. AllocRegBetween(NR_FLAGS,p,hp1,UsedRegs);
  7621. { hp1 is already a MOV instruction with the correct register }
  7622. taicpu(hp1).loadconst(0, 0);
  7623. { Inserting it right before p will guarantee that the flags are also tracked }
  7624. asml.Remove(hp1);
  7625. asml.InsertBefore(hp1, p);
  7626. end;
  7627. end;
  7628. Result:=true;
  7629. exit;
  7630. end
  7631. else if (hp1.typ = ait_label) then
  7632. Result := DoSETccLblRETOpt(p, tai_label(hp1));
  7633. end;
  7634. function TX86AsmOptimizer.OptPass1VMOVDQ(var p: tai): Boolean;
  7635. var
  7636. hp1, hp2, hp3: tai;
  7637. SourceRef, TargetRef: TReference;
  7638. CurrentReg: TRegister;
  7639. begin
  7640. { VMOVDQU/CMOVDQA shouldn't have even been generated }
  7641. if not UseAVX then
  7642. InternalError(2021100501);
  7643. Result := False;
  7644. { Look for the following to simplify:
  7645. vmovdqa/u x(mem1), %xmmreg
  7646. vmovdqa/u %xmmreg, y(mem2)
  7647. vmovdqa/u x+16(mem1), %xmmreg
  7648. vmovdqa/u %xmmreg, y+16(mem2)
  7649. Change to:
  7650. vmovdqa/u x(mem1), %ymmreg
  7651. vmovdqa/u %ymmreg, y(mem2)
  7652. vpxor %ymmreg, %ymmreg, %ymmreg
  7653. ( The VPXOR instruction is to zero the upper half, thus removing the
  7654. need to call the potentially expensive VZEROUPPER instruction. Other
  7655. peephole optimisations can remove VPXOR if it's unnecessary )
  7656. }
  7657. TransferUsedRegs(TmpUsedRegs);
  7658. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  7659. { NOTE: In the optimisations below, if the references dictate that an
  7660. aligned move is possible (i.e. VMOVDQA), the existing instructions
  7661. should already be VMOVDQA because if (x mod 32) = 0, then (x mod 16) = 0 }
  7662. if (taicpu(p).opsize = S_XMM) and
  7663. MatchOpType(taicpu(p), top_ref, top_reg) and
  7664. GetNextInstruction(p, hp1) and
  7665. MatchInstruction(hp1, A_VMOVDQA, A_VMOVDQU, [S_XMM]) and
  7666. MatchOpType(taicpu(hp1), top_reg, top_ref) and
  7667. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) then
  7668. begin
  7669. SourceRef := taicpu(p).oper[0]^.ref^;
  7670. TargetRef := taicpu(hp1).oper[1]^.ref^;
  7671. if GetNextInstruction(hp1, hp2) and
  7672. MatchInstruction(hp2, A_VMOVDQA, A_VMOVDQU, [S_XMM]) and
  7673. MatchOpType(taicpu(hp2), top_ref, top_reg) then
  7674. begin
  7675. { Delay calling GetNextInstruction(hp2, hp3) for as long as possible }
  7676. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  7677. Inc(SourceRef.offset, 16);
  7678. { Reuse the register in the first block move }
  7679. CurrentReg := newreg(R_MMREGISTER, getsupreg(taicpu(p).oper[1]^.reg), R_SUBMMY);
  7680. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) and
  7681. not RefsMightOverlap(taicpu(p).oper[0]^.ref^, TargetRef, 32) then
  7682. begin
  7683. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  7684. Inc(TargetRef.offset, 16);
  7685. if GetNextInstruction(hp2, hp3) and
  7686. MatchInstruction(hp3, A_VMOVDQA, A_VMOVDQU, [S_XMM]) and
  7687. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  7688. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  7689. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  7690. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  7691. begin
  7692. { Update the register tracking to the new size }
  7693. AllocRegBetween(CurrentReg, p, hp2, UsedRegs);
  7694. { Remember that the offsets are 16 ahead }
  7695. { Switch to unaligned if the memory isn't on a 32-byte boundary }
  7696. if not (
  7697. ((SourceRef.offset mod 32) = 16) and
  7698. (SourceRef.alignment >= 32) and ((SourceRef.alignment mod 32) = 0)
  7699. ) then
  7700. taicpu(p).opcode := A_VMOVDQU;
  7701. taicpu(p).opsize := S_YMM;
  7702. taicpu(p).oper[1]^.reg := CurrentReg;
  7703. if not (
  7704. ((TargetRef.offset mod 32) = 16) and
  7705. (TargetRef.alignment >= 32) and ((TargetRef.alignment mod 32) = 0)
  7706. ) then
  7707. taicpu(hp1).opcode := A_VMOVDQU;
  7708. taicpu(hp1).opsize := S_YMM;
  7709. taicpu(hp1).oper[0]^.reg := CurrentReg;
  7710. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(CurrentReg) + ' to merge a pair of memory moves (VmovdqxVmovdqxVmovdqxVmovdqx2VmovdqyVmovdqy 1)', p);
  7711. { If pi_uses_ymm is set, VZEROUPPER is present to do this for us }
  7712. if (pi_uses_ymm in current_procinfo.flags) then
  7713. RemoveInstruction(hp2)
  7714. else
  7715. begin
  7716. taicpu(hp2).opcode := A_VPXOR;
  7717. taicpu(hp2).opsize := S_YMM;
  7718. taicpu(hp2).loadreg(0, CurrentReg);
  7719. taicpu(hp2).loadreg(1, CurrentReg);
  7720. taicpu(hp2).loadreg(2, CurrentReg);
  7721. taicpu(hp2).ops := 3;
  7722. end;
  7723. RemoveInstruction(hp3);
  7724. Result := True;
  7725. Exit;
  7726. end;
  7727. end
  7728. else
  7729. begin
  7730. { See if the next references are 16 less rather than 16 greater }
  7731. Dec(SourceRef.offset, 32); { -16 the other way }
  7732. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) then
  7733. begin
  7734. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  7735. Dec(TargetRef.offset, 16); { Only 16, not 32, as it wasn't incremented unlike SourceRef }
  7736. if not RefsMightOverlap(SourceRef, TargetRef, 32) and
  7737. GetNextInstruction(hp2, hp3) and
  7738. MatchInstruction(hp3, A_MOV, [taicpu(p).opsize]) and
  7739. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  7740. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  7741. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  7742. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  7743. begin
  7744. { Update the register tracking to the new size }
  7745. AllocRegBetween(CurrentReg, hp2, hp3, UsedRegs);
  7746. { hp2 and hp3 are the starting offsets, so mod = 0 this time }
  7747. { Switch to unaligned if the memory isn't on a 32-byte boundary }
  7748. if not(
  7749. ((SourceRef.offset mod 32) = 0) and
  7750. (SourceRef.alignment >= 32) and ((SourceRef.alignment mod 32) = 0)
  7751. ) then
  7752. taicpu(hp2).opcode := A_VMOVDQU;
  7753. taicpu(hp2).opsize := S_YMM;
  7754. taicpu(hp2).oper[1]^.reg := CurrentReg;
  7755. if not (
  7756. ((TargetRef.offset mod 32) = 0) and
  7757. (TargetRef.alignment >= 32) and ((TargetRef.alignment mod 32) = 0)
  7758. ) then
  7759. taicpu(hp3).opcode := A_VMOVDQU;
  7760. taicpu(hp3).opsize := S_YMM;
  7761. taicpu(hp3).oper[0]^.reg := CurrentReg;
  7762. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(CurrentReg) + ' to merge a pair of memory moves (VmovdqxVmovdqxVmovdqxVmovdqx2VmovdqyVmovdqy 2)', p);
  7763. { If pi_uses_ymm is set, VZEROUPPER is present to do this for us }
  7764. if (pi_uses_ymm in current_procinfo.flags) then
  7765. RemoveInstruction(hp1)
  7766. else
  7767. begin
  7768. taicpu(hp1).opcode := A_VPXOR;
  7769. taicpu(hp1).opsize := S_YMM;
  7770. taicpu(hp1).loadreg(0, CurrentReg);
  7771. taicpu(hp1).loadreg(1, CurrentReg);
  7772. taicpu(hp1).loadreg(2, CurrentReg);
  7773. taicpu(hp1).ops := 3;
  7774. Asml.Remove(hp1);
  7775. Asml.InsertAfter(hp1, hp3); { Register deallocations will be after hp3 }
  7776. end;
  7777. RemoveCurrentP(p, hp2);
  7778. Result := True;
  7779. Exit;
  7780. end;
  7781. end;
  7782. end;
  7783. end;
  7784. end;
  7785. end;
  7786. function TX86AsmOptimizer.CheckJumpMovTransferOpt(var p: tai; hp1: tai; LoopCount: Integer; out Count: Integer): Boolean;
  7787. var
  7788. hp2, hp3, first_assignment: tai;
  7789. IncCount, OperIdx: Integer;
  7790. OrigLabel: TAsmLabel;
  7791. begin
  7792. Count := 0;
  7793. Result := False;
  7794. first_assignment := nil;
  7795. if (LoopCount >= 20) then
  7796. begin
  7797. { Guard against infinite loops }
  7798. Exit;
  7799. end;
  7800. if (taicpu(p).oper[0]^.typ <> top_ref) or
  7801. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) or
  7802. (taicpu(p).oper[0]^.ref^.base <> NR_NO) or
  7803. (taicpu(p).oper[0]^.ref^.index <> NR_NO) or
  7804. not (taicpu(p).oper[0]^.ref^.symbol is TAsmLabel) then
  7805. Exit;
  7806. OrigLabel := TAsmLabel(taicpu(p).oper[0]^.ref^.symbol);
  7807. {
  7808. change
  7809. jmp .L1
  7810. ...
  7811. .L1:
  7812. mov ##, ## ( multiple movs possible )
  7813. jmp/ret
  7814. into
  7815. mov ##, ##
  7816. jmp/ret
  7817. }
  7818. if not Assigned(hp1) then
  7819. begin
  7820. hp1 := GetLabelWithSym(OrigLabel);
  7821. if not Assigned(hp1) or not SkipLabels(hp1, hp1) then
  7822. Exit;
  7823. end;
  7824. hp2 := hp1;
  7825. while Assigned(hp2) do
  7826. begin
  7827. if Assigned(hp2) and (hp2.typ in [ait_label, ait_align]) then
  7828. SkipLabels(hp2,hp2);
  7829. if not Assigned(hp2) or (hp2.typ <> ait_instruction) then
  7830. Break;
  7831. case taicpu(hp2).opcode of
  7832. A_MOVSD:
  7833. begin
  7834. if taicpu(hp2).ops = 0 then
  7835. { Wrong MOVSD }
  7836. Break;
  7837. Inc(Count);
  7838. if Count >= 5 then
  7839. { Too many to be worthwhile }
  7840. Break;
  7841. GetNextInstruction(hp2, hp2);
  7842. Continue;
  7843. end;
  7844. A_MOV,
  7845. A_MOVD,
  7846. A_MOVQ,
  7847. A_MOVSX,
  7848. {$ifdef x86_64}
  7849. A_MOVSXD,
  7850. {$endif x86_64}
  7851. A_MOVZX,
  7852. A_MOVAPS,
  7853. A_MOVUPS,
  7854. A_MOVSS,
  7855. A_MOVAPD,
  7856. A_MOVUPD,
  7857. A_MOVDQA,
  7858. A_MOVDQU,
  7859. A_VMOVSS,
  7860. A_VMOVAPS,
  7861. A_VMOVUPS,
  7862. A_VMOVSD,
  7863. A_VMOVAPD,
  7864. A_VMOVUPD,
  7865. A_VMOVDQA,
  7866. A_VMOVDQU:
  7867. begin
  7868. Inc(Count);
  7869. if Count >= 5 then
  7870. { Too many to be worthwhile }
  7871. Break;
  7872. GetNextInstruction(hp2, hp2);
  7873. Continue;
  7874. end;
  7875. A_JMP:
  7876. begin
  7877. { Guard against infinite loops }
  7878. if taicpu(hp2).oper[0]^.ref^.symbol = OrigLabel then
  7879. Exit;
  7880. { Analyse this jump first in case it also duplicates assignments }
  7881. if CheckJumpMovTransferOpt(hp2, nil, LoopCount + 1, IncCount) then
  7882. begin
  7883. { Something did change! }
  7884. Result := True;
  7885. Inc(Count, IncCount);
  7886. if Count >= 5 then
  7887. begin
  7888. { Too many to be worthwhile }
  7889. Exit;
  7890. end;
  7891. if MatchInstruction(hp2, [A_JMP, A_RET], []) then
  7892. Break;
  7893. end;
  7894. Result := True;
  7895. Break;
  7896. end;
  7897. A_RET:
  7898. begin
  7899. Result := True;
  7900. Break;
  7901. end;
  7902. else
  7903. Break;
  7904. end;
  7905. end;
  7906. if Result then
  7907. begin
  7908. { A count of zero can happen when CheckJumpMovTransferOpt is called recursively }
  7909. if Count = 0 then
  7910. begin
  7911. Result := False;
  7912. Exit;
  7913. end;
  7914. hp3 := p;
  7915. DebugMsg(SPeepholeOptimization + 'Duplicated ' + debug_tostr(Count) + ' assignment(s) and redirected jump', p);
  7916. while True do
  7917. begin
  7918. if Assigned(hp1) and (hp1.typ in [ait_label, ait_align]) then
  7919. SkipLabels(hp1,hp1);
  7920. if (hp1.typ <> ait_instruction) then
  7921. InternalError(2021040720);
  7922. case taicpu(hp1).opcode of
  7923. A_JMP:
  7924. begin
  7925. { Change the original jump to the new destination }
  7926. OrigLabel.decrefs;
  7927. taicpu(hp1).oper[0]^.ref^.symbol.increfs;
  7928. taicpu(p).loadref(0, taicpu(hp1).oper[0]^.ref^);
  7929. { Set p to the first duplicated assignment so it can get optimised if needs be }
  7930. if not Assigned(first_assignment) then
  7931. InternalError(2021040810)
  7932. else
  7933. p := first_assignment;
  7934. Exit;
  7935. end;
  7936. A_RET:
  7937. begin
  7938. { Now change the jump into a RET instruction }
  7939. ConvertJumpToRET(p, hp1);
  7940. { Set p to the first duplicated assignment so it can get optimised if needs be }
  7941. if not Assigned(first_assignment) then
  7942. InternalError(2021040811)
  7943. else
  7944. p := first_assignment;
  7945. Exit;
  7946. end;
  7947. else
  7948. begin
  7949. { Duplicate the MOV instruction }
  7950. hp3:=tai(hp1.getcopy);
  7951. if first_assignment = nil then
  7952. first_assignment := hp3;
  7953. asml.InsertBefore(hp3, p);
  7954. { Make sure the compiler knows about any final registers written here }
  7955. for OperIdx := 0 to taicpu(hp3).ops - 1 do
  7956. with taicpu(hp3).oper[OperIdx]^ do
  7957. begin
  7958. case typ of
  7959. top_ref:
  7960. begin
  7961. if (ref^.base <> NR_NO) and
  7962. (getsupreg(ref^.base) <> RS_ESP) and
  7963. (getsupreg(ref^.base) <> RS_EBP)
  7964. {$ifdef x86_64} and (ref^.base <> NR_RIP) {$endif x86_64}
  7965. then
  7966. AllocRegBetween(ref^.base, hp3, tai(p.Next), UsedRegs);
  7967. if (ref^.index <> NR_NO) and
  7968. (getsupreg(ref^.index) <> RS_ESP) and
  7969. (getsupreg(ref^.index) <> RS_EBP)
  7970. {$ifdef x86_64} and (ref^.index <> NR_RIP) {$endif x86_64} and
  7971. (ref^.index <> ref^.base) then
  7972. AllocRegBetween(ref^.index, hp3, tai(p.Next), UsedRegs);
  7973. end;
  7974. top_reg:
  7975. AllocRegBetween(reg, hp3, tai(p.Next), UsedRegs);
  7976. else
  7977. ;
  7978. end;
  7979. end;
  7980. end;
  7981. end;
  7982. if not GetNextInstruction(hp1, hp1) then
  7983. { Should have dropped out earlier }
  7984. InternalError(2021040710);
  7985. end;
  7986. end;
  7987. end;
  7988. const
  7989. WriteOp: array[0..3] of set of TInsChange = (
  7990. [Ch_Wop1, Ch_RWop1, Ch_Mop1],
  7991. [Ch_Wop2, Ch_RWop2, Ch_Mop2],
  7992. [Ch_Wop3, Ch_RWop3, Ch_Mop3],
  7993. [Ch_Wop4, Ch_RWop4, Ch_Mop4]);
  7994. RegWriteFlags: array[0..7] of set of TInsChange = (
  7995. { The order is important: EAX, ECX, EDX, EBX, ESI, EDI, EBP, ESP }
  7996. [Ch_WEAX, Ch_RWEAX, Ch_MEAX{$ifdef x86_64}, Ch_WRAX, Ch_RWRAX, Ch_MRAX{$endif x86_64}],
  7997. [Ch_WECX, Ch_RWECX, Ch_MECX{$ifdef x86_64}, Ch_WRCX, Ch_RWRCX, Ch_MRCX{$endif x86_64}],
  7998. [Ch_WEDX, Ch_RWEDX, Ch_MEDX{$ifdef x86_64}, Ch_WRDX, Ch_RWRDX, Ch_MRDX{$endif x86_64}],
  7999. [Ch_WEBX, Ch_RWEBX, Ch_MEBX{$ifdef x86_64}, Ch_WRBX, Ch_RWRBX, Ch_MRBX{$endif x86_64}],
  8000. [Ch_WESI, Ch_RWESI, Ch_MESI{$ifdef x86_64}, Ch_WRSI, Ch_RWRSI, Ch_MRSI{$endif x86_64}],
  8001. [Ch_WEDI, Ch_RWEDI, Ch_MEDI{$ifdef x86_64}, Ch_WRDI, Ch_RWRDI, Ch_MRDI{$endif x86_64}],
  8002. [Ch_WEBP, Ch_RWEBP, Ch_MEBP{$ifdef x86_64}, Ch_WRBP, Ch_RWRBP, Ch_MRBP{$endif x86_64}],
  8003. [Ch_WESP, Ch_RWESP, Ch_MESP{$ifdef x86_64}, Ch_WRSP, Ch_RWRSP, Ch_MRSP{$endif x86_64}]);
  8004. function TX86AsmOptimizer.TrySwapMovOp(var p, hp1: tai): Boolean;
  8005. var
  8006. hp2: tai;
  8007. X: Integer;
  8008. begin
  8009. { If we have something like:
  8010. op ###,###
  8011. mov ###,###
  8012. Try to move the MOV instruction to before OP as long as OP and MOV don't
  8013. interfere in regards to what they write to.
  8014. NOTE: p must be a 2-operand instruction
  8015. }
  8016. Result := False;
  8017. if (hp1.typ <> ait_instruction) or
  8018. taicpu(hp1).is_jmp or
  8019. RegInInstruction(NR_DEFAULTFLAGS, hp1) then
  8020. Exit;
  8021. { NOP is a pipeline fence, likely marking the beginning of the function
  8022. epilogue, so drop out. Similarly, drop out if POP or RET are
  8023. encountered }
  8024. if MatchInstruction(hp1, A_NOP, A_POP, A_RET, []) then
  8025. Exit;
  8026. if (taicpu(hp1).opcode = A_MOVSD) and
  8027. (taicpu(hp1).ops = 0) then
  8028. { Wrong MOVSD }
  8029. Exit;
  8030. { Check for writes to specific registers first }
  8031. { EAX, ECX, EDX, EBX, ESI, EDI, EBP, ESP in that order }
  8032. for X := 0 to 7 do
  8033. if (RegWriteFlags[X] * InsProp[taicpu(hp1).opcode].Ch <> [])
  8034. and RegInInstruction(newreg(R_INTREGISTER, TSuperRegister(X), R_SUBWHOLE), p) then
  8035. Exit;
  8036. for X := 0 to taicpu(hp1).ops - 1 do
  8037. begin
  8038. { Check to see if this operand writes to something }
  8039. if ((WriteOp[X] * InsProp[taicpu(hp1).opcode].Ch) <> []) and
  8040. { And matches something in the CMP/TEST instruction }
  8041. (
  8042. MatchOperand(taicpu(hp1).oper[X]^, taicpu(p).oper[0]^) or
  8043. MatchOperand(taicpu(hp1).oper[X]^, taicpu(p).oper[1]^) or
  8044. (
  8045. { If it's a register, make sure the register written to doesn't
  8046. appear in the cmp instruction as part of a reference }
  8047. (taicpu(hp1).oper[X]^.typ = top_reg) and
  8048. RegInInstruction(taicpu(hp1).oper[X]^.reg, p)
  8049. )
  8050. ) then
  8051. Exit;
  8052. end;
  8053. { Check p to make sure it doesn't write to something that affects hp1 }
  8054. { Check for writes to specific registers first }
  8055. { EAX, ECX, EDX, EBX, ESI, EDI, EBP, ESP in that order }
  8056. for X := 0 to 7 do
  8057. if (RegWriteFlags[X] * InsProp[taicpu(p).opcode].Ch <> [])
  8058. and RegInInstruction(newreg(R_INTREGISTER, TSuperRegister(X), R_SUBWHOLE), hp1) then
  8059. Exit;
  8060. for X := 0 to taicpu(p).ops - 1 do
  8061. begin
  8062. { Check to see if this operand writes to something }
  8063. if ((WriteOp[X] * InsProp[taicpu(p).opcode].Ch) <> []) and
  8064. { And matches something in hp1 }
  8065. (taicpu(p).oper[X]^.typ = top_reg) and
  8066. RegInInstruction(taicpu(p).oper[X]^.reg, hp1) then
  8067. Exit;
  8068. end;
  8069. { The instruction can be safely moved }
  8070. asml.Remove(hp1);
  8071. { Try to insert after the last instructions where the FLAGS register is not
  8072. yet in use, so "mov $0,%reg" can be optimised into "xor %reg,%reg" later }
  8073. if SetAndTest(FindRegAllocBackward(NR_DEFAULTFLAGS, tai(p.Previous)), hp2) then
  8074. asml.InsertBefore(hp1, hp2)
  8075. { Failing that, try to insert after the last instructions where the
  8076. FLAGS register is not yet in use }
  8077. else if GetLastInstruction(p, hp2) and
  8078. (
  8079. (hp2.typ <> ait_instruction) or
  8080. { Don't insert after an instruction that uses the flags when p doesn't use them }
  8081. RegInInstruction(NR_DEFAULTFLAGS, p) or
  8082. not RegInInstruction(NR_DEFAULTFLAGS, hp2)
  8083. ) then
  8084. asml.InsertAfter(hp1, hp2)
  8085. else
  8086. { Note, if p.Previous is nil (even if it should logically never be the
  8087. case), FindRegAllocBackward immediately exits with False and so we
  8088. safely land here (we can't just pass p because FindRegAllocBackward
  8089. immediately exits on an instruction). [Kit] }
  8090. asml.InsertBefore(hp1, p);
  8091. DebugMsg(SPeepholeOptimization + 'Swapped ' + debug_op2str(taicpu(p).opcode) + ' and ' + debug_op2str(taicpu(hp1).opcode) + ' instructions to improve optimisation potential', hp1);
  8092. { We can't trust UsedRegs because we're looking backwards, although we
  8093. know the registers are allocated after p at the very least, so manually
  8094. create tai_regalloc objects if needed }
  8095. for X := 0 to taicpu(hp1).ops - 1 do
  8096. case taicpu(hp1).oper[X]^.typ of
  8097. top_reg:
  8098. begin
  8099. asml.InsertBefore(tai_regalloc.alloc(taicpu(hp1).oper[X]^.reg, nil), hp1);
  8100. IncludeRegInUsedRegs(taicpu(hp1).oper[X]^.reg, UsedRegs);
  8101. AllocRegBetween(taicpu(hp1).oper[X]^.reg, hp1, p, UsedRegs);
  8102. end;
  8103. top_ref:
  8104. begin
  8105. if taicpu(hp1).oper[X]^.ref^.base <> NR_NO then
  8106. begin
  8107. asml.InsertBefore(tai_regalloc.alloc(taicpu(hp1).oper[X]^.ref^.base, nil), hp1);
  8108. IncludeRegInUsedRegs(taicpu(hp1).oper[X]^.ref^.base, UsedRegs);
  8109. AllocRegBetween(taicpu(hp1).oper[X]^.ref^.base, hp1, p, UsedRegs);
  8110. end;
  8111. if taicpu(hp1).oper[X]^.ref^.index <> NR_NO then
  8112. begin
  8113. asml.InsertBefore(tai_regalloc.alloc(taicpu(hp1).oper[X]^.ref^.index, nil), hp1);
  8114. IncludeRegInUsedRegs(taicpu(hp1).oper[X]^.ref^.index, UsedRegs);
  8115. AllocRegBetween(taicpu(hp1).oper[X]^.ref^.index, hp1, p, UsedRegs);
  8116. end;
  8117. end;
  8118. else
  8119. ;
  8120. end;
  8121. Result := True;
  8122. end;
  8123. function TX86AsmOptimizer.TrySwapMovCmp(var p, hp1: tai): Boolean;
  8124. var
  8125. hp2: tai;
  8126. X: Integer;
  8127. begin
  8128. { If we have something like:
  8129. cmp ###,%reg1
  8130. mov 0,%reg2
  8131. And no modified registers are shared, move the instruction to before
  8132. the comparison as this means it can be optimised without worrying
  8133. about the FLAGS register. (CMP/MOV is generated by
  8134. "J(c)Mov1JmpMov0 -> Set(~c)", among other things).
  8135. As long as the second instruction doesn't use the flags or one of the
  8136. registers used by CMP or TEST (also check any references that use the
  8137. registers), then it can be moved prior to the comparison.
  8138. }
  8139. Result := False;
  8140. if not TrySwapMovOp(p, hp1) then
  8141. Exit;
  8142. if taicpu(hp1).opcode = A_LEA then
  8143. { The flags will be overwritten by the CMP/TEST instruction }
  8144. ConvertLEA(taicpu(hp1));
  8145. Result := True;
  8146. { Can we move it one further back? }
  8147. if GetLastInstruction(hp1, hp2) and (hp2.typ = ait_instruction) and
  8148. { Check to see if CMP/TEST is a comparison against zero }
  8149. (
  8150. (
  8151. (taicpu(p).opcode = A_CMP) and
  8152. MatchOperand(taicpu(p).oper[0]^, 0)
  8153. ) or
  8154. (
  8155. (taicpu(p).opcode = A_TEST) and
  8156. (
  8157. OpsEqual(taicpu(p).oper[0]^, taicpu(p).oper[1]^) or
  8158. MatchOperand(taicpu(p).oper[0]^, -1)
  8159. )
  8160. )
  8161. ) and
  8162. { These instructions set the zero flag if the result is zero }
  8163. MatchInstruction(hp2, [A_ADD, A_SUB, A_OR, A_XOR, A_AND, A_POPCNT, A_LZCNT], []) and
  8164. OpsEqual(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^) then
  8165. { Looks like we can - if successful, this benefits PostPeepholeOptTestOr }
  8166. TrySwapMovOp(hp2, hp1);
  8167. end;
  8168. function TX86AsmOptimizer.OptPass2MOV(var p : tai) : boolean;
  8169. function IsXCHGAcceptable: Boolean; inline;
  8170. begin
  8171. { Always accept if optimising for size }
  8172. Result := (cs_opt_size in current_settings.optimizerswitches) or
  8173. { From the Pentium M onwards, XCHG only has a latency of 2 rather
  8174. than 3, so it becomes a saving compared to three MOVs with two of
  8175. them able to execute simultaneously. [Kit] }
  8176. (CPUX86_HINT_FAST_XCHG in cpu_optimization_hints[current_settings.optimizecputype]);
  8177. end;
  8178. var
  8179. NewRef: TReference;
  8180. hp1, hp2, hp3, hp4: Tai;
  8181. {$ifndef x86_64}
  8182. OperIdx: Integer;
  8183. {$endif x86_64}
  8184. NewInstr : Taicpu;
  8185. NewAligh : Tai_align;
  8186. DestLabel: TAsmLabel;
  8187. function TryMovArith2Lea(InputInstr: tai): Boolean;
  8188. var
  8189. NextInstr: tai;
  8190. begin
  8191. Result := False;
  8192. UpdateUsedRegs(TmpUsedRegs, tai(InputInstr.Next));
  8193. if not GetNextInstruction(InputInstr, NextInstr) or
  8194. (
  8195. { The FLAGS register isn't always tracked properly, so do not
  8196. perform this optimisation if a conditional statement follows }
  8197. not RegReadByInstruction(NR_DEFAULTFLAGS, NextInstr) and
  8198. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, NextInstr, TmpUsedRegs)
  8199. ) then
  8200. begin
  8201. reference_reset(NewRef, 1, []);
  8202. NewRef.base := taicpu(p).oper[0]^.reg;
  8203. NewRef.scalefactor := 1;
  8204. if taicpu(InputInstr).opcode = A_ADD then
  8205. begin
  8206. DebugMsg(SPeepholeOptimization + 'MovAdd2Lea', p);
  8207. NewRef.offset := taicpu(InputInstr).oper[0]^.val;
  8208. end
  8209. else
  8210. begin
  8211. DebugMsg(SPeepholeOptimization + 'MovSub2Lea', p);
  8212. NewRef.offset := -taicpu(InputInstr).oper[0]^.val;
  8213. end;
  8214. taicpu(p).opcode := A_LEA;
  8215. taicpu(p).loadref(0, NewRef);
  8216. RemoveInstruction(InputInstr);
  8217. Result := True;
  8218. end;
  8219. end;
  8220. begin
  8221. Result:=false;
  8222. { This optimisation adds an instruction, so only do it for speed }
  8223. if not (cs_opt_size in current_settings.optimizerswitches) and
  8224. MatchOpType(taicpu(p), top_const, top_reg) and
  8225. (taicpu(p).oper[0]^.val = 0) then
  8226. begin
  8227. { To avoid compiler warning }
  8228. DestLabel := nil;
  8229. if (p.typ <> ait_instruction) or (taicpu(p).oper[1]^.typ <> top_reg) then
  8230. InternalError(2021040750);
  8231. if not GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[1]^.reg) then
  8232. Exit;
  8233. case hp1.typ of
  8234. ait_align,
  8235. ait_label:
  8236. begin
  8237. { Change:
  8238. mov $0,%reg mov $0,%reg
  8239. @Lbl1: @Lbl1:
  8240. test %reg,%reg / cmp $0,%reg test %reg,%reg / mov $0,%reg
  8241. je @Lbl2 jne @Lbl2
  8242. To: To:
  8243. mov $0,%reg mov $0,%reg
  8244. jmp @Lbl2 jmp @Lbl3
  8245. (align) (align)
  8246. @Lbl1: @Lbl1:
  8247. test %reg,%reg / cmp $0,%reg test %reg,%reg / cmp $0,%reg
  8248. je @Lbl2 je @Lbl2
  8249. @Lbl3: <-- Only if label exists
  8250. (Not if it's optimised for size)
  8251. }
  8252. if not SkipAligns(hp1, hp1) or not GetNextInstruction(hp1, hp2) then
  8253. Exit;
  8254. if (hp2.typ = ait_instruction) and
  8255. (
  8256. { Register sizes must exactly match }
  8257. (
  8258. (taicpu(hp2).opcode = A_CMP) and
  8259. MatchOperand(taicpu(hp2).oper[0]^, 0) and
  8260. MatchOperand(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^.reg)
  8261. ) or (
  8262. (taicpu(hp2).opcode = A_TEST) and
  8263. MatchOperand(taicpu(hp2).oper[0]^, taicpu(p).oper[1]^.reg) and
  8264. MatchOperand(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^.reg)
  8265. )
  8266. ) and GetNextInstruction(hp2, hp3) and
  8267. (hp3.typ = ait_instruction) and
  8268. (taicpu(hp3).opcode = A_JCC) and
  8269. (taicpu(hp3).oper[0]^.typ=top_ref) and (taicpu(hp3).oper[0]^.ref^.refaddr=addr_full) and (taicpu(hp3).oper[0]^.ref^.base=NR_NO) and
  8270. (taicpu(hp3).oper[0]^.ref^.index=NR_NO) and (taicpu(hp3).oper[0]^.ref^.symbol is tasmlabel) then
  8271. begin
  8272. { Check condition of jump }
  8273. { Always true? }
  8274. if condition_in(C_E, taicpu(hp3).condition) then
  8275. begin
  8276. { Copy label symbol and obtain matching label entry for the
  8277. conditional jump, as this will be our destination}
  8278. DestLabel := tasmlabel(taicpu(hp3).oper[0]^.ref^.symbol);
  8279. DebugMsg(SPeepholeOptimization + 'Mov0LblCmp0Je -> Mov0JmpLblCmp0Je', p);
  8280. Result := True;
  8281. end
  8282. { Always false? }
  8283. else if condition_in(C_NE, taicpu(hp3).condition) and GetNextInstruction(hp3, hp2) then
  8284. begin
  8285. { This is only worth it if there's a jump to take }
  8286. case hp2.typ of
  8287. ait_instruction:
  8288. begin
  8289. if taicpu(hp2).opcode = A_JMP then
  8290. begin
  8291. DestLabel := tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol);
  8292. { An unconditional jump follows the conditional jump which will always be false,
  8293. so use this jump's destination for the new jump }
  8294. DebugMsg(SPeepholeOptimization + 'Mov0LblCmp0Jne -> Mov0JmpLblCmp0Jne (with JMP)', p);
  8295. Result := True;
  8296. end
  8297. else if taicpu(hp2).opcode = A_JCC then
  8298. begin
  8299. DestLabel := tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol);
  8300. if condition_in(C_E, taicpu(hp2).condition) then
  8301. begin
  8302. { A second conditional jump follows the conditional jump which will always be false,
  8303. while the second jump is always True, so use this jump's destination for the new jump }
  8304. DebugMsg(SPeepholeOptimization + 'Mov0LblCmp0Jne -> Mov0JmpLblCmp0Jne (with second Jcc)', p);
  8305. Result := True;
  8306. end;
  8307. { Don't risk it if the jump isn't always true (Result remains False) }
  8308. end;
  8309. end;
  8310. else
  8311. { If anything else don't optimise };
  8312. end;
  8313. end;
  8314. if Result then
  8315. begin
  8316. { Just so we have something to insert as a paremeter}
  8317. reference_reset(NewRef, 1, []);
  8318. NewInstr := taicpu.op_ref(A_JMP, S_NO, NewRef);
  8319. { Now actually load the correct parameter (this also
  8320. increases the reference count) }
  8321. NewInstr.loadsymbol(0, DestLabel, 0);
  8322. if (cs_opt_level3 in current_settings.optimizerswitches) then
  8323. begin
  8324. { Get instruction before original label (may not be p under -O3) }
  8325. if not GetLastInstruction(hp1, hp2) then
  8326. { Shouldn't fail here }
  8327. InternalError(2021040701);
  8328. { Before the aligns too }
  8329. while (hp2.typ = ait_align) do
  8330. if not GetLastInstruction(hp2, hp2) then
  8331. { Shouldn't fail here }
  8332. InternalError(2021040702);
  8333. end
  8334. else
  8335. hp2 := p;
  8336. taicpu(NewInstr).fileinfo := taicpu(hp2).fileinfo;
  8337. AsmL.InsertAfter(NewInstr, hp2);
  8338. { Add new alignment field }
  8339. (* AsmL.InsertAfter(
  8340. cai_align.create_max(
  8341. current_settings.alignment.jumpalign,
  8342. current_settings.alignment.jumpalignskipmax
  8343. ),
  8344. NewInstr
  8345. ); *)
  8346. end;
  8347. Exit;
  8348. end;
  8349. end;
  8350. else
  8351. ;
  8352. end;
  8353. end;
  8354. if not GetNextInstruction(p, hp1) then
  8355. Exit;
  8356. if MatchInstruction(hp1, A_CMP, A_TEST, [taicpu(p).opsize])
  8357. and DoMovCmpMemOpt(p, hp1, True) then
  8358. begin
  8359. Result := True;
  8360. Exit;
  8361. end
  8362. else if MatchInstruction(hp1, A_JMP, [S_NO]) then
  8363. begin
  8364. { Sometimes the MOVs that OptPass2JMP produces can be improved
  8365. further, but we can't just put this jump optimisation in pass 1
  8366. because it tends to perform worse when conditional jumps are
  8367. nearby (e.g. when converting CMOV instructions). [Kit] }
  8368. if OptPass2JMP(hp1) then
  8369. { call OptPass1MOV once to potentially merge any MOVs that were created }
  8370. Result := OptPass1MOV(p)
  8371. { OptPass2MOV will now exit but will be called again if OptPass1MOV
  8372. returned True and the instruction is still a MOV, thus checking
  8373. the optimisations below }
  8374. { If OptPass2JMP returned False, no optimisations were done to
  8375. the jump and there are no further optimisations that can be done
  8376. to the MOV instruction on this pass }
  8377. end
  8378. else if MatchOpType(taicpu(p),top_reg,top_reg) and
  8379. (taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) and
  8380. MatchInstruction(hp1,A_ADD,A_SUB,[taicpu(p).opsize]) and
  8381. (taicpu(hp1).oper[1]^.typ = top_reg) and
  8382. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  8383. begin
  8384. { Change:
  8385. movl/q %reg1,%reg2 movl/q %reg1,%reg2
  8386. addl/q $x,%reg2 subl/q $x,%reg2
  8387. To:
  8388. leal/q x(%reg1),%reg2 leal/q -x(%reg1),%reg2
  8389. }
  8390. if (taicpu(hp1).oper[0]^.typ = top_const) and
  8391. { be lazy, checking separately for sub would be slightly better }
  8392. (abs(taicpu(hp1).oper[0]^.val)<=$7fffffff) then
  8393. begin
  8394. TransferUsedRegs(TmpUsedRegs);
  8395. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  8396. if TryMovArith2Lea(hp1) then
  8397. begin
  8398. Result := True;
  8399. Exit;
  8400. end
  8401. end
  8402. else if not RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^) and
  8403. GetNextInstructionUsingReg(hp1, hp2, taicpu(p).oper[1]^.reg) and
  8404. { Same as above, but also adds or subtracts to %reg2 in between.
  8405. It's still valid as long as the flags aren't in use }
  8406. MatchInstruction(hp2,A_ADD,A_SUB,[taicpu(p).opsize]) and
  8407. MatchOpType(taicpu(hp2), top_const, top_reg) and
  8408. (taicpu(hp2).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  8409. { be lazy, checking separately for sub would be slightly better }
  8410. (abs(taicpu(hp2).oper[0]^.val)<=$7fffffff) then
  8411. begin
  8412. TransferUsedRegs(TmpUsedRegs);
  8413. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  8414. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  8415. if TryMovArith2Lea(hp2) then
  8416. begin
  8417. Result := True;
  8418. Exit;
  8419. end;
  8420. end;
  8421. end
  8422. else if MatchOpType(taicpu(p),top_reg,top_reg) and
  8423. {$ifdef x86_64}
  8424. MatchInstruction(hp1,A_MOVZX,A_MOVSX,A_MOVSXD,[]) and
  8425. {$else x86_64}
  8426. MatchInstruction(hp1,A_MOVZX,A_MOVSX,[]) and
  8427. {$endif x86_64}
  8428. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  8429. (taicpu(hp1).oper[0]^.reg = taicpu(p).oper[1]^.reg) then
  8430. { mov reg1, reg2 mov reg1, reg2
  8431. movzx/sx reg2, reg3 to movzx/sx reg1, reg3}
  8432. begin
  8433. taicpu(hp1).oper[0]^.reg := taicpu(p).oper[0]^.reg;
  8434. DebugMsg(SPeepholeOptimization + 'mov %reg1,%reg2; movzx/sx %reg2,%reg3 -> mov %reg1,%reg2;movzx/sx %reg1,%reg3',p);
  8435. { Don't remove the MOV command without first checking that reg2 isn't used afterwards,
  8436. or unless supreg(reg3) = supreg(reg2)). [Kit] }
  8437. TransferUsedRegs(TmpUsedRegs);
  8438. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  8439. if (getsupreg(taicpu(p).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) or
  8440. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs)
  8441. then
  8442. begin
  8443. RemoveCurrentP(p, hp1);
  8444. Result:=true;
  8445. end;
  8446. exit;
  8447. end
  8448. else if MatchOpType(taicpu(p),top_reg,top_reg) and
  8449. IsXCHGAcceptable and
  8450. { XCHG doesn't support 8-byte registers }
  8451. (taicpu(p).opsize <> S_B) and
  8452. MatchInstruction(hp1, A_MOV, []) and
  8453. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  8454. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[0]^.reg) and
  8455. GetNextInstruction(hp1, hp2) and
  8456. MatchInstruction(hp2, A_MOV, []) and
  8457. { Don't need to call MatchOpType for hp2 because the operand matches below cover for it }
  8458. MatchOperand(taicpu(hp2).oper[0]^, taicpu(p).oper[1]^.reg) and
  8459. MatchOperand(taicpu(hp2).oper[1]^, taicpu(hp1).oper[0]^.reg) then
  8460. begin
  8461. { mov %reg1,%reg2
  8462. mov %reg3,%reg1 -> xchg %reg3,%reg1
  8463. mov %reg2,%reg3
  8464. (%reg2 not used afterwards)
  8465. Note that xchg takes 3 cycles to execute, and generally mov's take
  8466. only one cycle apiece, but the first two mov's can be executed in
  8467. parallel, only taking 2 cycles overall. Older processors should
  8468. therefore only optimise for size. [Kit]
  8469. }
  8470. TransferUsedRegs(TmpUsedRegs);
  8471. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  8472. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  8473. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp2, TmpUsedRegs) then
  8474. begin
  8475. DebugMsg(SPeepholeOptimization + 'MovMovMov2XChg', p);
  8476. AllocRegBetween(taicpu(hp2).oper[1]^.reg, p, hp1, UsedRegs);
  8477. taicpu(hp1).opcode := A_XCHG;
  8478. RemoveCurrentP(p, hp1);
  8479. RemoveInstruction(hp2);
  8480. Result := True;
  8481. Exit;
  8482. end;
  8483. end
  8484. else if MatchOpType(taicpu(p),top_reg,top_reg) and
  8485. MatchInstruction(hp1, A_SAR, []) then
  8486. begin
  8487. if MatchOperand(taicpu(hp1).oper[0]^, 31) then
  8488. begin
  8489. { the use of %edx also covers the opsize being S_L }
  8490. if MatchOperand(taicpu(hp1).oper[1]^, NR_EDX) then
  8491. begin
  8492. { Note it has to be specifically "movl %eax,%edx", and those specific sub-registers }
  8493. if (taicpu(p).oper[0]^.reg = NR_EAX) and
  8494. (taicpu(p).oper[1]^.reg = NR_EDX) then
  8495. begin
  8496. { Change:
  8497. movl %eax,%edx
  8498. sarl $31,%edx
  8499. To:
  8500. cltd
  8501. }
  8502. DebugMsg(SPeepholeOptimization + 'MovSar2Cltd', p);
  8503. RemoveInstruction(hp1);
  8504. taicpu(p).opcode := A_CDQ;
  8505. taicpu(p).opsize := S_NO;
  8506. taicpu(p).clearop(1);
  8507. taicpu(p).clearop(0);
  8508. taicpu(p).ops:=0;
  8509. Result := True;
  8510. end
  8511. else if (cs_opt_size in current_settings.optimizerswitches) and
  8512. (taicpu(p).oper[0]^.reg = NR_EDX) and
  8513. (taicpu(p).oper[1]^.reg = NR_EAX) then
  8514. begin
  8515. { Change:
  8516. movl %edx,%eax
  8517. sarl $31,%edx
  8518. To:
  8519. movl %edx,%eax
  8520. cltd
  8521. Note that this creates a dependency between the two instructions,
  8522. so only perform if optimising for size.
  8523. }
  8524. DebugMsg(SPeepholeOptimization + 'MovSar2MovCltd', p);
  8525. taicpu(hp1).opcode := A_CDQ;
  8526. taicpu(hp1).opsize := S_NO;
  8527. taicpu(hp1).clearop(1);
  8528. taicpu(hp1).clearop(0);
  8529. taicpu(hp1).ops:=0;
  8530. end;
  8531. {$ifndef x86_64}
  8532. end
  8533. { Don't bother if CMOV is supported, because a more optimal
  8534. sequence would have been generated for the Abs() intrinsic }
  8535. else if not(CPUX86_HAS_CMOV in cpu_capabilities[current_settings.cputype]) and
  8536. { the use of %eax also covers the opsize being S_L }
  8537. MatchOperand(taicpu(hp1).oper[1]^, NR_EAX) and
  8538. (taicpu(p).oper[0]^.reg = NR_EAX) and
  8539. (taicpu(p).oper[1]^.reg = NR_EDX) and
  8540. GetNextInstruction(hp1, hp2) and
  8541. MatchInstruction(hp2, A_XOR, [S_L]) and
  8542. MatchOperand(taicpu(hp2).oper[0]^, NR_EAX) and
  8543. MatchOperand(taicpu(hp2).oper[1]^, NR_EDX) and
  8544. GetNextInstruction(hp2, hp3) and
  8545. MatchInstruction(hp3, A_SUB, [S_L]) and
  8546. MatchOperand(taicpu(hp3).oper[0]^, NR_EAX) and
  8547. MatchOperand(taicpu(hp3).oper[1]^, NR_EDX) then
  8548. begin
  8549. { Change:
  8550. movl %eax,%edx
  8551. sarl $31,%eax
  8552. xorl %eax,%edx
  8553. subl %eax,%edx
  8554. (Instruction that uses %edx)
  8555. (%eax deallocated)
  8556. (%edx deallocated)
  8557. To:
  8558. cltd
  8559. xorl %edx,%eax <-- Note the registers have swapped
  8560. subl %edx,%eax
  8561. (Instruction that uses %eax) <-- %eax rather than %edx
  8562. }
  8563. TransferUsedRegs(TmpUsedRegs);
  8564. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  8565. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  8566. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  8567. if not RegUsedAfterInstruction(NR_EAX, hp3, TmpUsedRegs) then
  8568. begin
  8569. if GetNextInstruction(hp3, hp4) and
  8570. not RegModifiedByInstruction(NR_EDX, hp4) and
  8571. not RegUsedAfterInstruction(NR_EDX, hp4, TmpUsedRegs) then
  8572. begin
  8573. DebugMsg(SPeepholeOptimization + 'abs() intrinsic optimisation', p);
  8574. taicpu(p).opcode := A_CDQ;
  8575. taicpu(p).clearop(1);
  8576. taicpu(p).clearop(0);
  8577. taicpu(p).ops:=0;
  8578. RemoveInstruction(hp1);
  8579. taicpu(hp2).loadreg(0, NR_EDX);
  8580. taicpu(hp2).loadreg(1, NR_EAX);
  8581. taicpu(hp3).loadreg(0, NR_EDX);
  8582. taicpu(hp3).loadreg(1, NR_EAX);
  8583. AllocRegBetween(NR_EAX, hp3, hp4, TmpUsedRegs);
  8584. { Convert references in the following instruction (hp4) from %edx to %eax }
  8585. for OperIdx := 0 to taicpu(hp4).ops - 1 do
  8586. with taicpu(hp4).oper[OperIdx]^ do
  8587. case typ of
  8588. top_reg:
  8589. if getsupreg(reg) = RS_EDX then
  8590. reg := newreg(R_INTREGISTER,RS_EAX,getsubreg(reg));
  8591. top_ref:
  8592. begin
  8593. if getsupreg(reg) = RS_EDX then
  8594. ref^.base := newreg(R_INTREGISTER,RS_EAX,getsubreg(reg));
  8595. if getsupreg(reg) = RS_EDX then
  8596. ref^.index := newreg(R_INTREGISTER,RS_EAX,getsubreg(reg));
  8597. end;
  8598. else
  8599. ;
  8600. end;
  8601. end;
  8602. end;
  8603. {$else x86_64}
  8604. end;
  8605. end
  8606. else if MatchOperand(taicpu(hp1).oper[0]^, 63) and
  8607. { the use of %rdx also covers the opsize being S_Q }
  8608. MatchOperand(taicpu(hp1).oper[1]^, NR_RDX) then
  8609. begin
  8610. { Note it has to be specifically "movq %rax,%rdx", and those specific sub-registers }
  8611. if (taicpu(p).oper[0]^.reg = NR_RAX) and
  8612. (taicpu(p).oper[1]^.reg = NR_RDX) then
  8613. begin
  8614. { Change:
  8615. movq %rax,%rdx
  8616. sarq $63,%rdx
  8617. To:
  8618. cqto
  8619. }
  8620. DebugMsg(SPeepholeOptimization + 'MovSar2Cqto', p);
  8621. RemoveInstruction(hp1);
  8622. taicpu(p).opcode := A_CQO;
  8623. taicpu(p).opsize := S_NO;
  8624. taicpu(p).clearop(1);
  8625. taicpu(p).clearop(0);
  8626. taicpu(p).ops:=0;
  8627. Result := True;
  8628. end
  8629. else if (cs_opt_size in current_settings.optimizerswitches) and
  8630. (taicpu(p).oper[0]^.reg = NR_RDX) and
  8631. (taicpu(p).oper[1]^.reg = NR_RAX) then
  8632. begin
  8633. { Change:
  8634. movq %rdx,%rax
  8635. sarq $63,%rdx
  8636. To:
  8637. movq %rdx,%rax
  8638. cqto
  8639. Note that this creates a dependency between the two instructions,
  8640. so only perform if optimising for size.
  8641. }
  8642. DebugMsg(SPeepholeOptimization + 'MovSar2MovCqto', p);
  8643. taicpu(hp1).opcode := A_CQO;
  8644. taicpu(hp1).opsize := S_NO;
  8645. taicpu(hp1).clearop(1);
  8646. taicpu(hp1).clearop(0);
  8647. taicpu(hp1).ops:=0;
  8648. {$endif x86_64}
  8649. end;
  8650. end;
  8651. end
  8652. else if MatchInstruction(hp1, A_MOV, []) and
  8653. (taicpu(hp1).oper[1]^.typ = top_reg) then
  8654. { Though "GetNextInstruction" could be factored out, along with
  8655. the instructions that depend on hp2, it is an expensive call that
  8656. should be delayed for as long as possible, hence we do cheaper
  8657. checks first that are likely to be False. [Kit] }
  8658. begin
  8659. if (
  8660. (
  8661. MatchOperand(taicpu(p).oper[1]^, NR_EDX) and
  8662. (taicpu(hp1).oper[1]^.reg = NR_EAX) and
  8663. (
  8664. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  8665. MatchOperand(taicpu(hp1).oper[0]^, NR_EDX)
  8666. )
  8667. ) or
  8668. (
  8669. MatchOperand(taicpu(p).oper[1]^, NR_EAX) and
  8670. (taicpu(hp1).oper[1]^.reg = NR_EDX) and
  8671. (
  8672. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  8673. MatchOperand(taicpu(hp1).oper[0]^, NR_EAX)
  8674. )
  8675. )
  8676. ) and
  8677. GetNextInstruction(hp1, hp2) and
  8678. MatchInstruction(hp2, A_SAR, []) and
  8679. MatchOperand(taicpu(hp2).oper[0]^, 31) then
  8680. begin
  8681. if MatchOperand(taicpu(hp2).oper[1]^, NR_EDX) then
  8682. begin
  8683. { Change:
  8684. movl r/m,%edx movl r/m,%eax movl r/m,%edx movl r/m,%eax
  8685. movl %edx,%eax or movl %eax,%edx or movl r/m,%eax or movl r/m,%edx
  8686. sarl $31,%edx sarl $31,%edx sarl $31,%edx sarl $31,%edx
  8687. To:
  8688. movl r/m,%eax <- Note the change in register
  8689. cltd
  8690. }
  8691. DebugMsg(SPeepholeOptimization + 'MovMovSar2MovCltd', p);
  8692. AllocRegBetween(NR_EAX, p, hp1, UsedRegs);
  8693. taicpu(p).loadreg(1, NR_EAX);
  8694. taicpu(hp1).opcode := A_CDQ;
  8695. taicpu(hp1).clearop(1);
  8696. taicpu(hp1).clearop(0);
  8697. taicpu(hp1).ops:=0;
  8698. RemoveInstruction(hp2);
  8699. (*
  8700. {$ifdef x86_64}
  8701. end
  8702. else if MatchOperand(taicpu(hp2).oper[1]^, NR_RDX) and
  8703. { This code sequence does not get generated - however it might become useful
  8704. if and when 128-bit signed integer types make an appearance, so the code
  8705. is kept here for when it is eventually needed. [Kit] }
  8706. (
  8707. (
  8708. (taicpu(hp1).oper[1]^.reg = NR_RAX) and
  8709. (
  8710. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  8711. MatchOperand(taicpu(hp1).oper[0]^, NR_RDX)
  8712. )
  8713. ) or
  8714. (
  8715. (taicpu(hp1).oper[1]^.reg = NR_RDX) and
  8716. (
  8717. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  8718. MatchOperand(taicpu(hp1).oper[0]^, NR_RAX)
  8719. )
  8720. )
  8721. ) and
  8722. GetNextInstruction(hp1, hp2) and
  8723. MatchInstruction(hp2, A_SAR, [S_Q]) and
  8724. MatchOperand(taicpu(hp2).oper[0]^, 63) and
  8725. MatchOperand(taicpu(hp2).oper[1]^, NR_RDX) then
  8726. begin
  8727. { Change:
  8728. movq r/m,%rdx movq r/m,%rax movq r/m,%rdx movq r/m,%rax
  8729. movq %rdx,%rax or movq %rax,%rdx or movq r/m,%rax or movq r/m,%rdx
  8730. sarq $63,%rdx sarq $63,%rdx sarq $63,%rdx sarq $63,%rdx
  8731. To:
  8732. movq r/m,%rax <- Note the change in register
  8733. cqto
  8734. }
  8735. DebugMsg(SPeepholeOptimization + 'MovMovSar2MovCqto', p);
  8736. AllocRegBetween(NR_RAX, p, hp1, UsedRegs);
  8737. taicpu(p).loadreg(1, NR_RAX);
  8738. taicpu(hp1).opcode := A_CQO;
  8739. taicpu(hp1).clearop(1);
  8740. taicpu(hp1).clearop(0);
  8741. taicpu(hp1).ops:=0;
  8742. RemoveInstruction(hp2);
  8743. {$endif x86_64}
  8744. *)
  8745. end;
  8746. end;
  8747. {$ifdef x86_64}
  8748. end
  8749. else if (taicpu(p).opsize = S_L) and
  8750. (taicpu(p).oper[1]^.typ = top_reg) and
  8751. (
  8752. MatchInstruction(hp1, A_MOV,[]) and
  8753. (taicpu(hp1).opsize = S_L) and
  8754. (taicpu(hp1).oper[1]^.typ = top_reg)
  8755. ) and (
  8756. GetNextInstruction(hp1, hp2) and
  8757. (tai(hp2).typ=ait_instruction) and
  8758. (taicpu(hp2).opsize = S_Q) and
  8759. (
  8760. (
  8761. MatchInstruction(hp2, A_ADD,[]) and
  8762. (taicpu(hp2).opsize = S_Q) and
  8763. (taicpu(hp2).oper[0]^.typ = top_reg) and (taicpu(hp2).oper[1]^.typ = top_reg) and
  8764. (
  8765. (
  8766. (getsupreg(taicpu(hp2).oper[0]^.reg) = getsupreg(taicpu(p).oper[1]^.reg)) and
  8767. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg))
  8768. ) or (
  8769. (getsupreg(taicpu(hp2).oper[0]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) and
  8770. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(p).oper[1]^.reg))
  8771. )
  8772. )
  8773. ) or (
  8774. MatchInstruction(hp2, A_LEA,[]) and
  8775. (taicpu(hp2).oper[0]^.ref^.offset = 0) and
  8776. (taicpu(hp2).oper[0]^.ref^.scalefactor <= 1) and
  8777. (
  8778. (
  8779. (getsupreg(taicpu(hp2).oper[0]^.ref^.base) = getsupreg(taicpu(p).oper[1]^.reg)) and
  8780. (getsupreg(taicpu(hp2).oper[0]^.ref^.index) = getsupreg(taicpu(hp1).oper[1]^.reg))
  8781. ) or (
  8782. (getsupreg(taicpu(hp2).oper[0]^.ref^.base) = getsupreg(taicpu(hp1).oper[1]^.reg)) and
  8783. (getsupreg(taicpu(hp2).oper[0]^.ref^.index) = getsupreg(taicpu(p).oper[1]^.reg))
  8784. )
  8785. ) and (
  8786. (
  8787. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg))
  8788. ) or (
  8789. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(p).oper[1]^.reg))
  8790. )
  8791. )
  8792. )
  8793. )
  8794. ) and (
  8795. GetNextInstruction(hp2, hp3) and
  8796. MatchInstruction(hp3, A_SHR,[]) and
  8797. (taicpu(hp3).opsize = S_Q) and
  8798. (taicpu(hp3).oper[0]^.typ = top_const) and (taicpu(hp2).oper[1]^.typ = top_reg) and
  8799. (taicpu(hp3).oper[0]^.val = 1) and
  8800. (taicpu(hp3).oper[1]^.reg = taicpu(hp2).oper[1]^.reg)
  8801. ) then
  8802. begin
  8803. { Change movl x, reg1d movl x, reg1d
  8804. movl y, reg2d movl y, reg2d
  8805. addq reg2q,reg1q or leaq (reg1q,reg2q),reg1q
  8806. shrq $1, reg1q shrq $1, reg1q
  8807. ( reg1d and reg2d can be switched around in the first two instructions )
  8808. To movl x, reg1d
  8809. addl y, reg1d
  8810. rcrl $1, reg1d
  8811. This corresponds to the common expression (x + y) shr 1, where
  8812. x and y are Cardinals (replacing "shr 1" with "div 2" produces
  8813. smaller code, but won't account for x + y causing an overflow). [Kit]
  8814. }
  8815. DebugMsg(SPeepholeOptimization + 'MovMov*Shr2MovMov*Rcr', p);
  8816. if (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) then
  8817. { Change first MOV command to have the same register as the final output }
  8818. taicpu(p).oper[1]^.reg := taicpu(hp1).oper[1]^.reg
  8819. else
  8820. taicpu(hp1).oper[1]^.reg := taicpu(p).oper[1]^.reg;
  8821. { Change second MOV command to an ADD command. This is easier than
  8822. converting the existing command because it means we don't have to
  8823. touch 'y', which might be a complicated reference, and also the
  8824. fact that the third command might either be ADD or LEA. [Kit] }
  8825. taicpu(hp1).opcode := A_ADD;
  8826. { Delete old ADD/LEA instruction }
  8827. RemoveInstruction(hp2);
  8828. { Convert "shrq $1, reg1q" to "rcr $1, reg1d" }
  8829. taicpu(hp3).opcode := A_RCR;
  8830. taicpu(hp3).changeopsize(S_L);
  8831. setsubreg(taicpu(hp3).oper[1]^.reg, R_SUBD);
  8832. {$endif x86_64}
  8833. end;
  8834. if FuncMov2Func(p, hp1) then
  8835. begin
  8836. Result := True;
  8837. Exit;
  8838. end;
  8839. end;
  8840. {$push}
  8841. {$q-}{$r-}
  8842. function TX86AsmOptimizer.OptPass2Movx(var p : tai) : boolean;
  8843. var
  8844. ThisReg: TRegister;
  8845. MinSize, MaxSize, TryShiftDown, TargetSize: TOpSize;
  8846. TargetSubReg: TSubRegister;
  8847. hp1, hp2: tai;
  8848. RegInUse, RegChanged, p_removed, hp1_removed: Boolean;
  8849. { Store list of found instructions so we don't have to call
  8850. GetNextInstructionUsingReg multiple times }
  8851. InstrList: array of taicpu;
  8852. InstrMax, Index: Integer;
  8853. UpperLimit, SignedUpperLimit, SignedUpperLimitBottom,
  8854. LowerLimit, SignedLowerLimit, SignedLowerLimitBottom,
  8855. TryShiftDownLimit, TryShiftDownSignedLimit, TryShiftDownSignedLimitLower,
  8856. WorkingValue: TCgInt;
  8857. PreMessage: string;
  8858. { Data flow analysis }
  8859. TestValMin, TestValMax, TestValSignedMax: TCgInt;
  8860. BitwiseOnly, OrXorUsed,
  8861. ShiftDownOverflow, UpperSignedOverflow, UpperUnsignedOverflow, LowerSignedOverflow, LowerUnsignedOverflow: Boolean;
  8862. function CheckOverflowConditions: Boolean;
  8863. begin
  8864. Result := True;
  8865. if (TestValSignedMax > SignedUpperLimit) then
  8866. UpperSignedOverflow := True;
  8867. if (TestValSignedMax > SignedLowerLimit) or (TestValSignedMax < SignedLowerLimitBottom) then
  8868. LowerSignedOverflow := True;
  8869. if (TestValMin > LowerLimit) or (TestValMax > LowerLimit) then
  8870. LowerUnsignedOverflow := True;
  8871. if (TestValMin > UpperLimit) or (TestValMax > UpperLimit) or (TestValSignedMax > UpperLimit) or
  8872. (TestValMin < SignedUpperLimitBottom) or (TestValMax < SignedUpperLimitBottom) or (TestValSignedMax < SignedUpperLimitBottom) then
  8873. begin
  8874. { Absolute overflow }
  8875. Result := False;
  8876. Exit;
  8877. end;
  8878. if not ShiftDownOverflow and (TryShiftDown <> S_NO) and
  8879. ((TestValMin > TryShiftDownLimit) or (TestValMax > TryShiftDownLimit)) then
  8880. ShiftDownOverflow := True;
  8881. if (TestValMin < 0) or (TestValMax < 0) then
  8882. begin
  8883. LowerUnsignedOverflow := True;
  8884. UpperUnsignedOverflow := True;
  8885. end;
  8886. end;
  8887. function AdjustInitialLoadAndSize: Boolean;
  8888. begin
  8889. Result := False;
  8890. if not p_removed then
  8891. begin
  8892. if TargetSize = MinSize then
  8893. begin
  8894. { Convert the input MOVZX to a MOV }
  8895. if (taicpu(p).oper[0]^.typ = top_reg) and
  8896. SuperRegistersEqual(taicpu(p).oper[0]^.reg, ThisReg) then
  8897. begin
  8898. { Or remove it completely! }
  8899. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 1', p);
  8900. RemoveCurrentP(p);
  8901. p_removed := True;
  8902. end
  8903. else
  8904. begin
  8905. DebugMsg(SPeepholeOptimization + 'Movzx2Mov 1', p);
  8906. taicpu(p).opcode := A_MOV;
  8907. taicpu(p).oper[1]^.reg := ThisReg;
  8908. taicpu(p).opsize := TargetSize;
  8909. end;
  8910. Result := True;
  8911. end
  8912. else if TargetSize <> MaxSize then
  8913. begin
  8914. case MaxSize of
  8915. S_L:
  8916. if TargetSize = S_W then
  8917. begin
  8918. DebugMsg(SPeepholeOptimization + 'movzbl2movzbw', p);
  8919. taicpu(p).opsize := S_BW;
  8920. taicpu(p).oper[1]^.reg := ThisReg;
  8921. Result := True;
  8922. end
  8923. else
  8924. InternalError(2020112341);
  8925. S_W:
  8926. if TargetSize = S_L then
  8927. begin
  8928. DebugMsg(SPeepholeOptimization + 'movzbw2movzbl', p);
  8929. taicpu(p).opsize := S_BL;
  8930. taicpu(p).oper[1]^.reg := ThisReg;
  8931. Result := True;
  8932. end
  8933. else
  8934. InternalError(2020112342);
  8935. else
  8936. ;
  8937. end;
  8938. end
  8939. else if not hp1_removed and not RegInUse then
  8940. begin
  8941. { If we have something like:
  8942. movzbl (oper),%regd
  8943. add x, %regd
  8944. movzbl %regb, %regd
  8945. We can reduce the register size to the input of the final
  8946. movzbl instruction. Overflows won't have any effect.
  8947. }
  8948. if (taicpu(p).opsize in [S_BW, S_BL]) and
  8949. (taicpu(hp1).opsize in [S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}]) then
  8950. begin
  8951. TargetSize := S_B;
  8952. setsubreg(ThisReg, R_SUBL);
  8953. Result := True;
  8954. end
  8955. else if (taicpu(p).opsize = S_WL) and
  8956. (taicpu(hp1).opsize in [S_WL{$ifdef x86_64}, S_BQ{$endif x86_64}]) then
  8957. begin
  8958. TargetSize := S_W;
  8959. setsubreg(ThisReg, R_SUBW);
  8960. Result := True;
  8961. end;
  8962. if Result then
  8963. begin
  8964. { Convert the input MOVZX to a MOV }
  8965. if (taicpu(p).oper[0]^.typ = top_reg) and
  8966. SuperRegistersEqual(taicpu(p).oper[0]^.reg, ThisReg) then
  8967. begin
  8968. { Or remove it completely! }
  8969. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 1a', p);
  8970. RemoveCurrentP(p);
  8971. p_removed := True;
  8972. end
  8973. else
  8974. begin
  8975. DebugMsg(SPeepholeOptimization + 'Movzx2Mov 1a', p);
  8976. taicpu(p).opcode := A_MOV;
  8977. taicpu(p).oper[1]^.reg := ThisReg;
  8978. taicpu(p).opsize := TargetSize;
  8979. end;
  8980. end;
  8981. end;
  8982. end;
  8983. end;
  8984. procedure AdjustFinalLoad;
  8985. begin
  8986. if not LowerUnsignedOverflow then
  8987. begin
  8988. if ((TargetSize = S_L) and (taicpu(hp1).opsize in [S_L, S_BL, S_WL])) or
  8989. ((TargetSize = S_W) and (taicpu(hp1).opsize in [S_W, S_BW])) then
  8990. begin
  8991. { Convert the output MOVZX to a MOV }
  8992. if SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  8993. begin
  8994. { Make sure the zero-expansion covers at least the minimum size (fixes i40003) }
  8995. if (MinSize = S_B) or
  8996. (not ShiftDownOverflow and (TryShiftDown = S_B)) or
  8997. ((MinSize = S_W) and (taicpu(hp1).opsize = S_WL)) then
  8998. begin
  8999. { Remove it completely! }
  9000. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 2', hp1);
  9001. { Be careful; if p = hp1 and p was also removed, p
  9002. will become a dangling pointer }
  9003. if p = hp1 then
  9004. begin
  9005. RemoveCurrentp(p); { p = hp1 and will then become the next instruction }
  9006. p_removed := True;
  9007. end
  9008. else
  9009. RemoveInstruction(hp1);
  9010. hp1_removed := True;
  9011. end;
  9012. end
  9013. else
  9014. begin
  9015. DebugMsg(SPeepholeOptimization + 'Movzx2Mov 2', hp1);
  9016. taicpu(hp1).opcode := A_MOV;
  9017. taicpu(hp1).oper[0]^.reg := ThisReg;
  9018. taicpu(hp1).opsize := TargetSize;
  9019. end;
  9020. end
  9021. else if (TargetSize = S_B) and (MaxSize = S_W) and (taicpu(hp1).opsize = S_WL) then
  9022. begin
  9023. { Need to change the size of the output }
  9024. DebugMsg(SPeepholeOptimization + 'movzwl2movzbl 2', hp1);
  9025. taicpu(hp1).oper[0]^.reg := ThisReg;
  9026. taicpu(hp1).opsize := S_BL;
  9027. end;
  9028. end;
  9029. end;
  9030. function CompressInstructions: Boolean;
  9031. var
  9032. LocalIndex: Integer;
  9033. begin
  9034. Result := False;
  9035. { The objective here is to try to find a combination that
  9036. removes one of the MOV/Z instructions. }
  9037. if (
  9038. (taicpu(p).oper[0]^.typ <> top_reg) or
  9039. not SuperRegistersEqual(taicpu(p).oper[0]^.reg, ThisReg)
  9040. ) and
  9041. (taicpu(hp1).oper[1]^.typ = top_reg) and
  9042. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  9043. begin
  9044. { Make a preference to remove the second MOVZX instruction }
  9045. case taicpu(hp1).opsize of
  9046. S_BL, S_WL:
  9047. begin
  9048. TargetSize := S_L;
  9049. TargetSubReg := R_SUBD;
  9050. end;
  9051. S_BW:
  9052. begin
  9053. TargetSize := S_W;
  9054. TargetSubReg := R_SUBW;
  9055. end;
  9056. else
  9057. InternalError(2020112302);
  9058. end;
  9059. end
  9060. else
  9061. begin
  9062. if LowerUnsignedOverflow and not UpperUnsignedOverflow then
  9063. begin
  9064. { Exceeded lower bound but not upper bound }
  9065. TargetSize := MaxSize;
  9066. end
  9067. else if not LowerUnsignedOverflow then
  9068. begin
  9069. { Size didn't exceed lower bound }
  9070. TargetSize := MinSize;
  9071. end
  9072. else
  9073. Exit;
  9074. end;
  9075. case TargetSize of
  9076. S_B:
  9077. TargetSubReg := R_SUBL;
  9078. S_W:
  9079. TargetSubReg := R_SUBW;
  9080. S_L:
  9081. TargetSubReg := R_SUBD;
  9082. else
  9083. InternalError(2020112350);
  9084. end;
  9085. { Update the register to its new size }
  9086. setsubreg(ThisReg, TargetSubReg);
  9087. RegInUse := False;
  9088. if not SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  9089. begin
  9090. { Check to see if the active register is used afterwards;
  9091. if not, we can change it and make a saving. }
  9092. TransferUsedRegs(TmpUsedRegs);
  9093. { The target register may be marked as in use to cross
  9094. a jump to a distant label, so exclude it }
  9095. ExcludeRegFromUsedRegs(taicpu(hp1).oper[1]^.reg, TmpUsedRegs);
  9096. hp2 := p;
  9097. repeat
  9098. { Explicitly check for the excluded register (don't include the first
  9099. instruction as it may be reading from here }
  9100. if ((p <> hp2) and (RegInInstruction(taicpu(hp1).oper[1]^.reg, hp2))) or
  9101. RegInUsedRegs(taicpu(hp1).oper[1]^.reg, TmpUsedRegs) then
  9102. begin
  9103. RegInUse := True;
  9104. Break;
  9105. end;
  9106. UpdateUsedRegs(TmpUsedRegs, tai(hp2.next));
  9107. if not GetNextInstruction(hp2, hp2) then
  9108. InternalError(2020112340);
  9109. until (hp2 = hp1);
  9110. if not RegInUse and RegUsedAfterInstruction(ThisReg, hp1, TmpUsedRegs) then
  9111. { We might still be able to get away with this }
  9112. RegInUse := not
  9113. (
  9114. GetNextInstructionUsingReg(hp1, hp2, ThisReg) and
  9115. (hp2.typ = ait_instruction) and
  9116. (
  9117. { Under -O1 and -O2, GetNextInstructionUsingReg may return an
  9118. instruction that doesn't actually contain ThisReg }
  9119. (cs_opt_level3 in current_settings.optimizerswitches) or
  9120. RegInInstruction(ThisReg, hp2)
  9121. ) and
  9122. RegLoadedWithNewValue(ThisReg, hp2)
  9123. );
  9124. if not RegInUse then
  9125. begin
  9126. { Force the register size to the same as this instruction so it can be removed}
  9127. if (taicpu(hp1).opsize in [S_L, S_BL, S_WL]) then
  9128. begin
  9129. TargetSize := S_L;
  9130. TargetSubReg := R_SUBD;
  9131. end
  9132. else if (taicpu(hp1).opsize in [S_W, S_BW]) then
  9133. begin
  9134. TargetSize := S_W;
  9135. TargetSubReg := R_SUBW;
  9136. end;
  9137. ThisReg := taicpu(hp1).oper[1]^.reg;
  9138. setsubreg(ThisReg, TargetSubReg);
  9139. RegChanged := True;
  9140. DebugMsg(SPeepholeOptimization + 'Simplified register usage so ' + debug_regname(ThisReg) + ' = ' + debug_regname(taicpu(p).oper[1]^.reg), p);
  9141. TransferUsedRegs(TmpUsedRegs);
  9142. AllocRegBetween(ThisReg, p, hp1, TmpUsedRegs);
  9143. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 3', hp1);
  9144. if p = hp1 then
  9145. begin
  9146. RemoveCurrentp(p); { p = hp1 and will then become the next instruction }
  9147. p_removed := True;
  9148. end
  9149. else
  9150. RemoveInstruction(hp1);
  9151. hp1_removed := True;
  9152. { Instruction will become "mov %reg,%reg" }
  9153. if not p_removed and (taicpu(p).opcode = A_MOV) and
  9154. MatchOperand(taicpu(p).oper[0]^, ThisReg) then
  9155. begin
  9156. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 6', p);
  9157. RemoveCurrentP(p);
  9158. p_removed := True;
  9159. end
  9160. else
  9161. taicpu(p).oper[1]^.reg := ThisReg;
  9162. Result := True;
  9163. end
  9164. else
  9165. begin
  9166. if TargetSize <> MaxSize then
  9167. begin
  9168. { Since the register is in use, we have to force it to
  9169. MaxSize otherwise part of it may become undefined later on }
  9170. TargetSize := MaxSize;
  9171. case TargetSize of
  9172. S_B:
  9173. TargetSubReg := R_SUBL;
  9174. S_W:
  9175. TargetSubReg := R_SUBW;
  9176. S_L:
  9177. TargetSubReg := R_SUBD;
  9178. else
  9179. InternalError(2020112351);
  9180. end;
  9181. setsubreg(ThisReg, TargetSubReg);
  9182. end;
  9183. AdjustFinalLoad;
  9184. end;
  9185. end
  9186. else
  9187. AdjustFinalLoad;
  9188. Result := AdjustInitialLoadAndSize or Result;
  9189. { Now go through every instruction we found and change the
  9190. size. If TargetSize = MaxSize, then almost no changes are
  9191. needed and Result can remain False if it hasn't been set
  9192. yet.
  9193. If RegChanged is True, then the register requires changing
  9194. and so the point about TargetSize = MaxSize doesn't apply. }
  9195. if ((TargetSize <> MaxSize) or RegChanged) and (InstrMax >= 0) then
  9196. begin
  9197. for LocalIndex := 0 to InstrMax do
  9198. begin
  9199. { If p_removed is true, then the original MOV/Z was removed
  9200. and removing the AND instruction may not be safe if it
  9201. appears first }
  9202. if (InstrList[LocalIndex].oper[InstrList[LocalIndex].ops - 1]^.typ <> top_reg) then
  9203. InternalError(2020112310);
  9204. if InstrList[LocalIndex].oper[0]^.typ = top_reg then
  9205. InstrList[LocalIndex].oper[0]^.reg := ThisReg;
  9206. InstrList[LocalIndex].oper[InstrList[LocalIndex].ops - 1]^.reg := ThisReg;
  9207. InstrList[LocalIndex].opsize := TargetSize;
  9208. end;
  9209. Result := True;
  9210. end;
  9211. end;
  9212. begin
  9213. Result := False;
  9214. p_removed := False;
  9215. hp1_removed := False;
  9216. ThisReg := taicpu(p).oper[1]^.reg;
  9217. { Check for:
  9218. movs/z ###,%ecx (or %cx or %rcx)
  9219. ...
  9220. shl/shr/sar/rcl/rcr/ror/rol %cl,###
  9221. (dealloc %ecx)
  9222. Change to:
  9223. mov ###,%cl (if ### = %cl, then remove completely)
  9224. ...
  9225. shl/shr/sar/rcl/rcr/ror/rol %cl,###
  9226. }
  9227. if (getsupreg(ThisReg) = RS_ECX) and
  9228. GetNextInstructionUsingReg(p, hp1, NR_ECX) and
  9229. (hp1.typ = ait_instruction) and
  9230. (
  9231. { Under -O1 and -O2, GetNextInstructionUsingReg may return an
  9232. instruction that doesn't actually contain ECX }
  9233. (cs_opt_level3 in current_settings.optimizerswitches) or
  9234. RegInInstruction(NR_ECX, hp1) or
  9235. (
  9236. { It's common for the shift/rotate's read/write register to be
  9237. initialised in between, so under -O2 and under, search ahead
  9238. one more instruction
  9239. }
  9240. GetNextInstruction(hp1, hp1) and
  9241. (hp1.typ = ait_instruction) and
  9242. RegInInstruction(NR_ECX, hp1)
  9243. )
  9244. ) and
  9245. MatchInstruction(hp1, [A_SHL, A_SHR, A_SAR, A_ROR, A_ROL, A_RCR, A_RCL], []) and
  9246. (taicpu(hp1).oper[0]^.typ = top_reg) { This is enough to determine that it's %cl } then
  9247. begin
  9248. TransferUsedRegs(TmpUsedRegs);
  9249. hp2 := p;
  9250. repeat
  9251. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  9252. until not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  9253. if not RegUsedAfterInstruction(NR_CL, hp1, TmpUsedRegs) then
  9254. begin
  9255. case taicpu(p).opsize of
  9256. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  9257. if MatchOperand(taicpu(p).oper[0]^, NR_CL) then
  9258. begin
  9259. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 3a', p);
  9260. RemoveCurrentP(p);
  9261. end
  9262. else
  9263. begin
  9264. taicpu(p).opcode := A_MOV;
  9265. taicpu(p).opsize := S_B;
  9266. taicpu(p).oper[1]^.reg := NR_CL;
  9267. DebugMsg(SPeepholeOptimization + 'MovxOp2MovOp 1', p);
  9268. end;
  9269. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  9270. if MatchOperand(taicpu(p).oper[0]^, NR_CX) then
  9271. begin
  9272. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 3b', p);
  9273. RemoveCurrentP(p);
  9274. end
  9275. else
  9276. begin
  9277. taicpu(p).opcode := A_MOV;
  9278. taicpu(p).opsize := S_W;
  9279. taicpu(p).oper[1]^.reg := NR_CX;
  9280. DebugMsg(SPeepholeOptimization + 'MovxOp2MovOp 2', p);
  9281. end;
  9282. {$ifdef x86_64}
  9283. S_LQ:
  9284. if MatchOperand(taicpu(p).oper[0]^, NR_ECX) then
  9285. begin
  9286. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 3c', p);
  9287. RemoveCurrentP(p);
  9288. end
  9289. else
  9290. begin
  9291. taicpu(p).opcode := A_MOV;
  9292. taicpu(p).opsize := S_L;
  9293. taicpu(p).oper[1]^.reg := NR_ECX;
  9294. DebugMsg(SPeepholeOptimization + 'MovxOp2MovOp 3', p);
  9295. end;
  9296. {$endif x86_64}
  9297. else
  9298. InternalError(2021120401);
  9299. end;
  9300. Result := True;
  9301. Exit;
  9302. end;
  9303. end;
  9304. { This is anything but quick! }
  9305. if not(cs_opt_level2 in current_settings.optimizerswitches) then
  9306. Exit;
  9307. SetLength(InstrList, 0);
  9308. InstrMax := -1;
  9309. case taicpu(p).opsize of
  9310. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  9311. begin
  9312. {$if defined(i386) or defined(i8086)}
  9313. { If the target size is 8-bit, make sure we can actually encode it }
  9314. if not (GetSupReg(ThisReg) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX]) then
  9315. Exit;
  9316. {$endif i386 or i8086}
  9317. LowerLimit := $FF;
  9318. SignedLowerLimit := $7F;
  9319. SignedLowerLimitBottom := -128;
  9320. MinSize := S_B;
  9321. if taicpu(p).opsize = S_BW then
  9322. begin
  9323. MaxSize := S_W;
  9324. UpperLimit := $FFFF;
  9325. SignedUpperLimit := $7FFF;
  9326. SignedUpperLimitBottom := -32768;
  9327. end
  9328. else
  9329. begin
  9330. { Keep at a 32-bit limit for BQ as well since one can't really optimise otherwise }
  9331. MaxSize := S_L;
  9332. UpperLimit := $FFFFFFFF;
  9333. SignedUpperLimit := $7FFFFFFF;
  9334. SignedUpperLimitBottom := -2147483648;
  9335. end;
  9336. end;
  9337. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  9338. begin
  9339. { Keep at a 32-bit limit for WQ as well since one can't really optimise otherwise }
  9340. LowerLimit := $FFFF;
  9341. SignedLowerLimit := $7FFF;
  9342. SignedLowerLimitBottom := -32768;
  9343. UpperLimit := $FFFFFFFF;
  9344. SignedUpperLimit := $7FFFFFFF;
  9345. SignedUpperLimitBottom := -2147483648;
  9346. MinSize := S_W;
  9347. MaxSize := S_L;
  9348. end;
  9349. {$ifdef x86_64}
  9350. S_LQ:
  9351. begin
  9352. { Both the lower and upper limits are set to 32-bit. If a limit
  9353. is breached, then optimisation is impossible }
  9354. LowerLimit := $FFFFFFFF;
  9355. SignedLowerLimit := $7FFFFFFF;
  9356. SignedLowerLimitBottom := -2147483648;
  9357. UpperLimit := $FFFFFFFF;
  9358. SignedUpperLimit := $7FFFFFFF;
  9359. SignedUpperLimitBottom := -2147483648;
  9360. MinSize := S_L;
  9361. MaxSize := S_L;
  9362. end;
  9363. {$endif x86_64}
  9364. else
  9365. InternalError(2020112301);
  9366. end;
  9367. TestValMin := 0;
  9368. TestValMax := LowerLimit;
  9369. TestValSignedMax := SignedLowerLimit;
  9370. TryShiftDownLimit := LowerLimit;
  9371. TryShiftDown := S_NO;
  9372. ShiftDownOverflow := False;
  9373. RegChanged := False;
  9374. BitwiseOnly := True;
  9375. OrXorUsed := False;
  9376. UpperSignedOverflow := False;
  9377. LowerSignedOverflow := False;
  9378. UpperUnsignedOverflow := False;
  9379. LowerUnsignedOverflow := False;
  9380. hp1 := p;
  9381. while GetNextInstructionUsingReg(hp1, hp1, ThisReg) and
  9382. (hp1.typ = ait_instruction) and
  9383. (
  9384. { Under -O1 and -O2, GetNextInstructionUsingReg may return an
  9385. instruction that doesn't actually contain ThisReg }
  9386. (cs_opt_level3 in current_settings.optimizerswitches) or
  9387. { This allows this Movx optimisation to work through the SETcc instructions
  9388. inserted by the 'CMP/JE/CMP/@Lbl/SETE -> CMP/SETE/CMP/SETE/OR'
  9389. optimisation on -O1 and -O2 (on -O3, GetNextInstructionUsingReg will
  9390. skip over these SETcc instructions). }
  9391. (taicpu(hp1).opcode = A_SETcc) or
  9392. RegInInstruction(ThisReg, hp1)
  9393. ) do
  9394. begin
  9395. case taicpu(hp1).opcode of
  9396. A_INC,A_DEC:
  9397. begin
  9398. { Has to be an exact match on the register }
  9399. if not MatchOperand(taicpu(hp1).oper[0]^, ThisReg) then
  9400. Break;
  9401. if taicpu(hp1).opcode = A_INC then
  9402. begin
  9403. Inc(TestValMin);
  9404. Inc(TestValMax);
  9405. Inc(TestValSignedMax);
  9406. end
  9407. else
  9408. begin
  9409. Dec(TestValMin);
  9410. Dec(TestValMax);
  9411. Dec(TestValSignedMax);
  9412. end;
  9413. end;
  9414. A_TEST, A_CMP:
  9415. begin
  9416. if (
  9417. { Too high a risk of non-linear behaviour that breaks DFA
  9418. here, unless it's cmp $0,%reg, which is equivalent to
  9419. test %reg,%reg }
  9420. OrXorUsed and
  9421. (taicpu(hp1).opcode = A_CMP) and
  9422. not Matchoperand(taicpu(hp1).oper[0]^, 0)
  9423. ) or
  9424. (taicpu(hp1).oper[1]^.typ <> top_reg) or
  9425. { Has to be an exact match on the register }
  9426. (taicpu(hp1).oper[1]^.reg <> ThisReg) or
  9427. (
  9428. { Permit "test %reg,%reg" }
  9429. (taicpu(hp1).opcode = A_TEST) and
  9430. (taicpu(hp1).oper[0]^.typ = top_reg) and
  9431. (taicpu(hp1).oper[0]^.reg <> ThisReg)
  9432. ) or
  9433. (taicpu(hp1).oper[0]^.typ <> top_const) or
  9434. { Make sure the comparison value is not smaller than the
  9435. smallest allowed signed value for the minimum size (e.g.
  9436. -128 for 8-bit) }
  9437. not (
  9438. ((taicpu(hp1).oper[0]^.val and LowerLimit) = taicpu(hp1).oper[0]^.val) or
  9439. { Is it in the negative range? }
  9440. (
  9441. (taicpu(hp1).oper[0]^.val < 0) and
  9442. (taicpu(hp1).oper[0]^.val >= SignedLowerLimitBottom)
  9443. )
  9444. ) then
  9445. Break;
  9446. { Check to see if the active register is used afterwards }
  9447. TransferUsedRegs(TmpUsedRegs);
  9448. IncludeRegInUsedRegs(ThisReg, TmpUsedRegs);
  9449. if not RegUsedAfterInstruction(ThisReg, hp1, TmpUsedRegs) then
  9450. begin
  9451. { Make sure the comparison or any previous instructions
  9452. hasn't pushed the test values outside of the range of
  9453. MinSize }
  9454. if LowerUnsignedOverflow and not UpperUnsignedOverflow then
  9455. begin
  9456. { Exceeded lower bound but not upper bound }
  9457. Exit;
  9458. end
  9459. else if not LowerSignedOverflow or not LowerUnsignedOverflow then
  9460. begin
  9461. { Size didn't exceed lower bound }
  9462. TargetSize := MinSize;
  9463. end
  9464. else
  9465. Break;
  9466. case TargetSize of
  9467. S_B:
  9468. TargetSubReg := R_SUBL;
  9469. S_W:
  9470. TargetSubReg := R_SUBW;
  9471. S_L:
  9472. TargetSubReg := R_SUBD;
  9473. else
  9474. InternalError(2021051002);
  9475. end;
  9476. if TargetSize <> MaxSize then
  9477. begin
  9478. { Update the register to its new size }
  9479. setsubreg(ThisReg, TargetSubReg);
  9480. DebugMsg(SPeepholeOptimization + 'CMP instruction resized thanks to register size optimisation (see MOV/Z assignment above)', hp1);
  9481. taicpu(hp1).oper[1]^.reg := ThisReg;
  9482. taicpu(hp1).opsize := TargetSize;
  9483. { Convert the input MOVZX to a MOV if necessary }
  9484. AdjustInitialLoadAndSize;
  9485. if (InstrMax >= 0) then
  9486. begin
  9487. for Index := 0 to InstrMax do
  9488. begin
  9489. { If p_removed is true, then the original MOV/Z was removed
  9490. and removing the AND instruction may not be safe if it
  9491. appears first }
  9492. if (InstrList[Index].oper[InstrList[Index].ops - 1]^.typ <> top_reg) then
  9493. InternalError(2020112311);
  9494. if InstrList[Index].oper[0]^.typ = top_reg then
  9495. InstrList[Index].oper[0]^.reg := ThisReg;
  9496. InstrList[Index].oper[InstrList[Index].ops - 1]^.reg := ThisReg;
  9497. InstrList[Index].opsize := MinSize;
  9498. end;
  9499. end;
  9500. Result := True;
  9501. end;
  9502. Exit;
  9503. end;
  9504. end;
  9505. A_SETcc:
  9506. begin
  9507. { This allows this Movx optimisation to work through the SETcc instructions
  9508. inserted by the 'CMP/JE/CMP/@Lbl/SETE -> CMP/SETE/CMP/SETE/OR'
  9509. optimisation on -O1 and -O2 (on -O3, GetNextInstructionUsingReg will
  9510. skip over these SETcc instructions). }
  9511. if (cs_opt_level3 in current_settings.optimizerswitches) or
  9512. { Of course, break out if the current register is used }
  9513. RegInOp(ThisReg, taicpu(hp1).oper[0]^) then
  9514. Break
  9515. else
  9516. { We must use Continue so the instruction doesn't get added
  9517. to InstrList }
  9518. Continue;
  9519. end;
  9520. A_ADD,A_SUB,A_AND,A_OR,A_XOR,A_SHL,A_SHR,A_SAR:
  9521. begin
  9522. if
  9523. (taicpu(hp1).oper[1]^.typ <> top_reg) or
  9524. { Has to be an exact match on the register }
  9525. (taicpu(hp1).oper[1]^.reg <> ThisReg) or not
  9526. (
  9527. (
  9528. (taicpu(hp1).oper[0]^.typ = top_const) and
  9529. (
  9530. (
  9531. (taicpu(hp1).opcode = A_SHL) and
  9532. (
  9533. ((MinSize = S_B) and (taicpu(hp1).oper[0]^.val < 8)) or
  9534. ((MinSize = S_W) and (taicpu(hp1).oper[0]^.val < 16)) or
  9535. ((MinSize = S_L) and (taicpu(hp1).oper[0]^.val < 32))
  9536. )
  9537. ) or (
  9538. (taicpu(hp1).opcode <> A_SHL) and
  9539. (
  9540. ((taicpu(hp1).oper[0]^.val and UpperLimit) = taicpu(hp1).oper[0]^.val) or
  9541. { Is it in the negative range? }
  9542. (((not taicpu(hp1).oper[0]^.val) and (UpperLimit shr 1)) = (not taicpu(hp1).oper[0]^.val))
  9543. )
  9544. )
  9545. )
  9546. ) or (
  9547. MatchOperand(taicpu(hp1).oper[0]^, taicpu(hp1).oper[1]^.reg) and
  9548. ((taicpu(hp1).opcode = A_ADD) or (taicpu(hp1).opcode = A_AND) or (taicpu(hp1).opcode = A_SUB))
  9549. )
  9550. ) then
  9551. Break;
  9552. { Only process OR and XOR if there are only bitwise operations,
  9553. since otherwise they can too easily fool the data flow
  9554. analysis (they can cause non-linear behaviour) }
  9555. case taicpu(hp1).opcode of
  9556. A_ADD:
  9557. begin
  9558. if OrXorUsed then
  9559. { Too high a risk of non-linear behaviour that breaks DFA here }
  9560. Break
  9561. else
  9562. BitwiseOnly := False;
  9563. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  9564. begin
  9565. TestValMin := TestValMin * 2;
  9566. TestValMax := TestValMax * 2;
  9567. TestValSignedMax := TestValSignedMax * 2;
  9568. end
  9569. else
  9570. begin
  9571. WorkingValue := taicpu(hp1).oper[0]^.val;
  9572. TestValMin := TestValMin + WorkingValue;
  9573. TestValMax := TestValMax + WorkingValue;
  9574. TestValSignedMax := TestValSignedMax + WorkingValue;
  9575. end;
  9576. end;
  9577. A_SUB:
  9578. begin
  9579. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  9580. begin
  9581. TestValMin := 0;
  9582. TestValMax := 0;
  9583. TestValSignedMax := 0;
  9584. end
  9585. else
  9586. begin
  9587. if OrXorUsed then
  9588. { Too high a risk of non-linear behaviour that breaks DFA here }
  9589. Break
  9590. else
  9591. BitwiseOnly := False;
  9592. WorkingValue := taicpu(hp1).oper[0]^.val;
  9593. TestValMin := TestValMin - WorkingValue;
  9594. TestValMax := TestValMax - WorkingValue;
  9595. TestValSignedMax := TestValSignedMax - WorkingValue;
  9596. end;
  9597. end;
  9598. A_AND:
  9599. if (taicpu(hp1).oper[0]^.typ = top_const) then
  9600. begin
  9601. { we might be able to go smaller if AND appears first }
  9602. if InstrMax = -1 then
  9603. case MinSize of
  9604. S_B:
  9605. ;
  9606. S_W:
  9607. if ((taicpu(hp1).oper[0]^.val and $FF) = taicpu(hp1).oper[0]^.val) or
  9608. ((not(taicpu(hp1).oper[0]^.val) and $7F) = (not taicpu(hp1).oper[0]^.val)) then
  9609. begin
  9610. TryShiftDown := S_B;
  9611. TryShiftDownLimit := $FF;
  9612. end;
  9613. S_L:
  9614. if ((taicpu(hp1).oper[0]^.val and $FF) = taicpu(hp1).oper[0]^.val) or
  9615. ((not(taicpu(hp1).oper[0]^.val) and $7F) = (not taicpu(hp1).oper[0]^.val)) then
  9616. begin
  9617. TryShiftDown := S_B;
  9618. TryShiftDownLimit := $FF;
  9619. end
  9620. else if ((taicpu(hp1).oper[0]^.val and $FFFF) = taicpu(hp1).oper[0]^.val) or
  9621. ((not(taicpu(hp1).oper[0]^.val) and $7FFF) = (not taicpu(hp1).oper[0]^.val)) then
  9622. begin
  9623. TryShiftDown := S_W;
  9624. TryShiftDownLimit := $FFFF;
  9625. end;
  9626. else
  9627. InternalError(2020112320);
  9628. end;
  9629. WorkingValue := taicpu(hp1).oper[0]^.val;
  9630. TestValMin := TestValMin and WorkingValue;
  9631. TestValMax := TestValMax and WorkingValue;
  9632. TestValSignedMax := TestValSignedMax and WorkingValue;
  9633. end;
  9634. A_OR:
  9635. begin
  9636. if not BitwiseOnly then
  9637. Break;
  9638. OrXorUsed := True;
  9639. WorkingValue := taicpu(hp1).oper[0]^.val;
  9640. TestValMin := TestValMin or WorkingValue;
  9641. TestValMax := TestValMax or WorkingValue;
  9642. TestValSignedMax := TestValSignedMax or WorkingValue;
  9643. end;
  9644. A_XOR:
  9645. begin
  9646. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  9647. begin
  9648. TestValMin := 0;
  9649. TestValMax := 0;
  9650. TestValSignedMax := 0;
  9651. end
  9652. else
  9653. begin
  9654. if not BitwiseOnly then
  9655. Break;
  9656. OrXorUsed := True;
  9657. WorkingValue := taicpu(hp1).oper[0]^.val;
  9658. TestValMin := TestValMin xor WorkingValue;
  9659. TestValMax := TestValMax xor WorkingValue;
  9660. TestValSignedMax := TestValSignedMax xor WorkingValue;
  9661. end;
  9662. end;
  9663. A_SHL:
  9664. begin
  9665. BitwiseOnly := False;
  9666. WorkingValue := taicpu(hp1).oper[0]^.val;
  9667. TestValMin := TestValMin shl WorkingValue;
  9668. TestValMax := TestValMax shl WorkingValue;
  9669. TestValSignedMax := TestValSignedMax shl WorkingValue;
  9670. end;
  9671. A_SHR,
  9672. { The first instruction was MOVZX, so the value won't be negative }
  9673. A_SAR:
  9674. begin
  9675. if InstrMax <> -1 then
  9676. BitwiseOnly := False
  9677. else
  9678. { we might be able to go smaller if SHR appears first }
  9679. case MinSize of
  9680. S_B:
  9681. ;
  9682. S_W:
  9683. if (taicpu(hp1).oper[0]^.val >= 8) then
  9684. begin
  9685. TryShiftDown := S_B;
  9686. TryShiftDownLimit := $FF;
  9687. TryShiftDownSignedLimit := $7F;
  9688. TryShiftDownSignedLimitLower := -128;
  9689. end;
  9690. S_L:
  9691. if (taicpu(hp1).oper[0]^.val >= 24) then
  9692. begin
  9693. TryShiftDown := S_B;
  9694. TryShiftDownLimit := $FF;
  9695. TryShiftDownSignedLimit := $7F;
  9696. TryShiftDownSignedLimitLower := -128;
  9697. end
  9698. else if (taicpu(hp1).oper[0]^.val >= 16) then
  9699. begin
  9700. TryShiftDown := S_W;
  9701. TryShiftDownLimit := $FFFF;
  9702. TryShiftDownSignedLimit := $7FFF;
  9703. TryShiftDownSignedLimitLower := -32768;
  9704. end;
  9705. else
  9706. InternalError(2020112321);
  9707. end;
  9708. WorkingValue := taicpu(hp1).oper[0]^.val;
  9709. if taicpu(hp1).opcode = A_SAR then
  9710. begin
  9711. TestValMin := SarInt64(TestValMin, WorkingValue);
  9712. TestValMax := SarInt64(TestValMax, WorkingValue);
  9713. TestValSignedMax := SarInt64(TestValSignedMax, WorkingValue);
  9714. end
  9715. else
  9716. begin
  9717. TestValMin := TestValMin shr WorkingValue;
  9718. TestValMax := TestValMax shr WorkingValue;
  9719. TestValSignedMax := TestValSignedMax shr WorkingValue;
  9720. end;
  9721. end;
  9722. else
  9723. InternalError(2020112303);
  9724. end;
  9725. end;
  9726. (*
  9727. A_IMUL:
  9728. case taicpu(hp1).ops of
  9729. 2:
  9730. begin
  9731. if not MatchOpType(hp1, top_reg, top_reg) or
  9732. { Has to be an exact match on the register }
  9733. (taicpu(hp1).oper[0]^.reg <> ThisReg) or
  9734. (taicpu(hp1).oper[1]^.reg <> ThisReg) then
  9735. Break;
  9736. TestValMin := TestValMin * TestValMin;
  9737. TestValMax := TestValMax * TestValMax;
  9738. TestValSignedMax := TestValSignedMax * TestValMax;
  9739. end;
  9740. 3:
  9741. begin
  9742. if not MatchOpType(hp1, top_const, top_reg, top_reg) or
  9743. { Has to be an exact match on the register }
  9744. (taicpu(hp1).oper[1]^.reg <> ThisReg) or
  9745. (taicpu(hp1).oper[2]^.reg <> ThisReg) or
  9746. ((taicpu(hp1).oper[0]^.val and UpperLimit) = taicpu(hp1).oper[0]^.val) or
  9747. { Is it in the negative range? }
  9748. (((not taicpu(hp1).oper[0]^.val) and (UpperLimit shr 1)) = (not taicpu(hp1).oper[0]^.val)) then
  9749. Break;
  9750. TestValMin := TestValMin * taicpu(hp1).oper[0]^.val;
  9751. TestValMax := TestValMax * taicpu(hp1).oper[0]^.val;
  9752. TestValSignedMax := TestValSignedMax * taicpu(hp1).oper[0]^.val;
  9753. end;
  9754. else
  9755. Break;
  9756. end;
  9757. A_IDIV:
  9758. case taicpu(hp1).ops of
  9759. 3:
  9760. begin
  9761. if not MatchOpType(hp1, top_const, top_reg, top_reg) or
  9762. { Has to be an exact match on the register }
  9763. (taicpu(hp1).oper[1]^.reg <> ThisReg) or
  9764. (taicpu(hp1).oper[2]^.reg <> ThisReg) or
  9765. ((taicpu(hp1).oper[0]^.val and UpperLimit) = taicpu(hp1).oper[0]^.val) or
  9766. { Is it in the negative range? }
  9767. (((not taicpu(hp1).oper[0]^.val) and (UpperLimit shr 1)) = (not taicpu(hp1).oper[0]^.val)) then
  9768. Break;
  9769. TestValMin := TestValMin div taicpu(hp1).oper[0]^.val;
  9770. TestValMax := TestValMax div taicpu(hp1).oper[0]^.val;
  9771. TestValSignedMax := TestValSignedMax div taicpu(hp1).oper[0]^.val;
  9772. end;
  9773. else
  9774. Break;
  9775. end;
  9776. *)
  9777. A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  9778. begin
  9779. { If there are no instructions in between, then we might be able to make a saving }
  9780. if UpperSignedOverflow or (taicpu(hp1).oper[0]^.typ <> top_reg) or (taicpu(hp1).oper[0]^.reg <> ThisReg) then
  9781. Break;
  9782. { We have something like:
  9783. movzbw %dl,%dx
  9784. ...
  9785. movswl %dx,%edx
  9786. Change the latter to a zero-extension then enter the
  9787. A_MOVZX case branch.
  9788. }
  9789. {$ifdef x86_64}
  9790. if (taicpu(hp1).opsize = S_LQ) and SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  9791. begin
  9792. { this becomes a zero extension from 32-bit to 64-bit, but
  9793. the upper 32 bits are already zero, so just delete the
  9794. instruction }
  9795. DebugMsg(SPeepholeOptimization + 'MovzMovsxd2MovzNop', hp1);
  9796. RemoveInstruction(hp1);
  9797. Result := True;
  9798. Exit;
  9799. end
  9800. else
  9801. {$endif x86_64}
  9802. begin
  9803. DebugMsg(SPeepholeOptimization + 'MovzMovs2MovzMovz', hp1);
  9804. taicpu(hp1).opcode := A_MOVZX;
  9805. {$ifdef x86_64}
  9806. case taicpu(hp1).opsize of
  9807. S_BQ:
  9808. begin
  9809. taicpu(hp1).opsize := S_BL;
  9810. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  9811. end;
  9812. S_WQ:
  9813. begin
  9814. taicpu(hp1).opsize := S_WL;
  9815. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  9816. end;
  9817. S_LQ:
  9818. begin
  9819. taicpu(hp1).opcode := A_MOV;
  9820. taicpu(hp1).opsize := S_L;
  9821. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  9822. { In this instance, we need to break out because the
  9823. instruction is no longer MOVZX or MOVSXD }
  9824. Result := True;
  9825. Exit;
  9826. end;
  9827. else
  9828. ;
  9829. end;
  9830. {$endif x86_64}
  9831. Result := CompressInstructions;
  9832. Exit;
  9833. end;
  9834. end;
  9835. A_MOVZX:
  9836. begin
  9837. if UpperUnsignedOverflow or (taicpu(hp1).oper[0]^.typ <> top_reg) then
  9838. Break;
  9839. if (InstrMax = -1) then
  9840. begin
  9841. if SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, ThisReg) then
  9842. begin
  9843. { Optimise around i40003 }
  9844. if SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) and
  9845. (taicpu(p).opsize = S_WL) and (taicpu(hp1).opsize = S_BL)
  9846. {$ifndef x86_64}
  9847. and (
  9848. (taicpu(p).oper[0]^.typ <> top_reg) or
  9849. { Cannot encode byte-sized ESI, EDI, EBP or ESP under i386 }
  9850. (GetSupReg(taicpu(p).oper[0]^.reg) in [RS_EAX, RS_EBX, RS_ECX, RS_EDX])
  9851. )
  9852. {$endif not x86_64}
  9853. then
  9854. begin
  9855. if (taicpu(p).oper[0]^.typ = top_reg) then
  9856. setsubreg(taicpu(p).oper[0]^.reg, R_SUBL);
  9857. DebugMsg(SPeepholeOptimization + 'movzwl2movzbl 1', p);
  9858. taicpu(p).opsize := S_BL;
  9859. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 2a', hp1);
  9860. RemoveInstruction(hp1);
  9861. Result := True;
  9862. Exit;
  9863. end;
  9864. end
  9865. else
  9866. begin
  9867. { Will return false if the second parameter isn't ThisReg
  9868. (can happen on -O2 and under) }
  9869. if Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, ThisReg) then
  9870. begin
  9871. { The two MOVZX instructions are adjacent, so remove the first one }
  9872. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 5', p);
  9873. RemoveCurrentP(p);
  9874. Result := True;
  9875. Exit;
  9876. end;
  9877. Break;
  9878. end;
  9879. end;
  9880. Result := CompressInstructions;
  9881. Exit;
  9882. end;
  9883. else
  9884. { This includes ADC, SBB and IDIV }
  9885. Break;
  9886. end;
  9887. if not CheckOverflowConditions then
  9888. Break;
  9889. { Contains highest index (so instruction count - 1) }
  9890. Inc(InstrMax);
  9891. if InstrMax > High(InstrList) then
  9892. SetLength(InstrList, InstrMax + LIST_STEP_SIZE);
  9893. InstrList[InstrMax] := taicpu(hp1);
  9894. end;
  9895. end;
  9896. {$pop}
  9897. function TX86AsmOptimizer.OptPass2Imul(var p : tai) : boolean;
  9898. var
  9899. hp1 : tai;
  9900. begin
  9901. Result:=false;
  9902. if (taicpu(p).ops >= 2) and
  9903. ((taicpu(p).oper[0]^.typ = top_const) or
  9904. ((taicpu(p).oper[0]^.typ = top_ref) and (taicpu(p).oper[0]^.ref^.refaddr=addr_full))) and
  9905. (taicpu(p).oper[1]^.typ = top_reg) and
  9906. ((taicpu(p).ops = 2) or
  9907. ((taicpu(p).oper[2]^.typ = top_reg) and
  9908. (taicpu(p).oper[2]^.reg = taicpu(p).oper[1]^.reg))) and
  9909. GetLastInstruction(p,hp1) and
  9910. MatchInstruction(hp1,A_MOV,[]) and
  9911. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  9912. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  9913. begin
  9914. TransferUsedRegs(TmpUsedRegs);
  9915. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,p,TmpUsedRegs)) or
  9916. ((taicpu(p).ops = 3) and (taicpu(p).oper[1]^.reg=taicpu(p).oper[2]^.reg)) then
  9917. { change
  9918. mov reg1,reg2
  9919. imul y,reg2 to imul y,reg1,reg2 }
  9920. begin
  9921. taicpu(p).ops := 3;
  9922. taicpu(p).loadreg(2,taicpu(p).oper[1]^.reg);
  9923. taicpu(p).loadreg(1,taicpu(hp1).oper[0]^.reg);
  9924. DebugMsg(SPeepholeOptimization + 'MovImul2Imul done',p);
  9925. RemoveInstruction(hp1);
  9926. result:=true;
  9927. end;
  9928. end;
  9929. end;
  9930. procedure TX86AsmOptimizer.ConvertJumpToRET(const p: tai; const ret_p: tai);
  9931. var
  9932. ThisLabel: TAsmLabel;
  9933. begin
  9934. ThisLabel := tasmlabel(taicpu(p).oper[0]^.ref^.symbol);
  9935. ThisLabel.decrefs;
  9936. taicpu(p).condition := C_None;
  9937. taicpu(p).opcode := A_RET;
  9938. taicpu(p).is_jmp := false;
  9939. taicpu(p).ops := taicpu(ret_p).ops;
  9940. case taicpu(ret_p).ops of
  9941. 0:
  9942. taicpu(p).clearop(0);
  9943. 1:
  9944. taicpu(p).loadconst(0,taicpu(ret_p).oper[0]^.val);
  9945. else
  9946. internalerror(2016041301);
  9947. end;
  9948. { If the original label is now dead, it might turn out that the label
  9949. immediately follows p. As a result, everything beyond it, which will
  9950. be just some final register configuration and a RET instruction, is
  9951. now dead code. [Kit] }
  9952. { NOTE: This is much faster than introducing a OptPass2RET routine and
  9953. running RemoveDeadCodeAfterJump for each RET instruction, because
  9954. this optimisation rarely happens and most RETs appear at the end of
  9955. routines where there is nothing that can be stripped. [Kit] }
  9956. if not ThisLabel.is_used then
  9957. RemoveDeadCodeAfterJump(p);
  9958. end;
  9959. function TX86AsmOptimizer.OptPass2SETcc(var p: tai): boolean;
  9960. var
  9961. hp1,hp2,next: tai; SetC, JumpC: TAsmCond;
  9962. Unconditional, PotentialModified: Boolean;
  9963. OperPtr: POper;
  9964. NewRef: TReference;
  9965. InstrList: array of taicpu;
  9966. InstrMax, Index: Integer;
  9967. const
  9968. {$ifdef DEBUG_AOPTCPU}
  9969. SNoFlags: shortstring = ' so the flags aren''t modified';
  9970. {$else DEBUG_AOPTCPU}
  9971. SNoFlags = '';
  9972. {$endif DEBUG_AOPTCPU}
  9973. begin
  9974. Result:=false;
  9975. if MatchOpType(taicpu(p),top_reg) and GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[0]^.reg) then
  9976. begin
  9977. if MatchInstruction(hp1, A_TEST, [S_B]) and
  9978. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  9979. (taicpu(hp1).oper[0]^.reg = taicpu(hp1).oper[1]^.reg) and
  9980. (taicpu(p).oper[0]^.reg = taicpu(hp1).oper[1]^.reg) and
  9981. GetNextInstruction(hp1, hp2) and
  9982. MatchInstruction(hp2, A_Jcc, A_SETcc, []) then
  9983. { Change from: To:
  9984. set(C) %reg j(~C) label
  9985. test %reg,%reg/cmp $0,%reg
  9986. je label
  9987. set(C) %reg j(C) label
  9988. test %reg,%reg/cmp $0,%reg
  9989. jne label
  9990. (Also do something similar with sete/setne instead of je/jne)
  9991. }
  9992. begin
  9993. { Before we do anything else, we need to check the instructions
  9994. in between SETcc and TEST to make sure they don't modify the
  9995. FLAGS register - if -O2 or under, there won't be any
  9996. instructions between SET and TEST }
  9997. TransferUsedRegs(TmpUsedRegs);
  9998. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  9999. if (cs_opt_level3 in current_settings.optimizerswitches) then
  10000. begin
  10001. next := p;
  10002. SetLength(InstrList, 0);
  10003. InstrMax := -1;
  10004. PotentialModified := False;
  10005. { Make a note of every instruction that modifies the FLAGS
  10006. register }
  10007. while GetNextInstruction(next, next) and (next <> hp1) do
  10008. begin
  10009. if next.typ <> ait_instruction then
  10010. { GetNextInstructionUsingReg should have returned False }
  10011. InternalError(2021051701);
  10012. if RegModifiedByInstruction(NR_DEFAULTFLAGS, next) then
  10013. begin
  10014. case taicpu(next).opcode of
  10015. A_SETcc,
  10016. A_CMOVcc,
  10017. A_Jcc:
  10018. begin
  10019. if PotentialModified then
  10020. { Not safe because the flags were modified earlier }
  10021. Exit
  10022. else
  10023. { Condition is the same as the initial SETcc, so this is safe
  10024. (don't add to instruction list though) }
  10025. Continue;
  10026. end;
  10027. A_ADD:
  10028. begin
  10029. if (taicpu(next).opsize = S_B) or
  10030. { LEA doesn't support 8-bit operands }
  10031. (taicpu(next).oper[1]^.typ <> top_reg) or
  10032. { Must write to a register }
  10033. (taicpu(next).oper[0]^.typ = top_ref) then
  10034. { Require a constant or a register }
  10035. Exit;
  10036. PotentialModified := True;
  10037. end;
  10038. A_SUB:
  10039. begin
  10040. if (taicpu(next).opsize = S_B) or
  10041. { LEA doesn't support 8-bit operands }
  10042. (taicpu(next).oper[1]^.typ <> top_reg) or
  10043. { Must write to a register }
  10044. (taicpu(next).oper[0]^.typ <> top_const) or
  10045. (taicpu(next).oper[0]^.val = $80000000) then
  10046. { Can't subtract a register with LEA - also
  10047. check that the value isn't -2^31, as this
  10048. can't be negated }
  10049. Exit;
  10050. PotentialModified := True;
  10051. end;
  10052. A_SAL,
  10053. A_SHL:
  10054. begin
  10055. if (taicpu(next).opsize = S_B) or
  10056. { LEA doesn't support 8-bit operands }
  10057. (taicpu(next).oper[1]^.typ <> top_reg) or
  10058. { Must write to a register }
  10059. (taicpu(next).oper[0]^.typ <> top_const) or
  10060. (taicpu(next).oper[0]^.val < 0) or
  10061. (taicpu(next).oper[0]^.val > 3) then
  10062. Exit;
  10063. PotentialModified := True;
  10064. end;
  10065. A_IMUL:
  10066. begin
  10067. if (taicpu(next).ops <> 3) or
  10068. (taicpu(next).oper[1]^.typ <> top_reg) or
  10069. { Must write to a register }
  10070. (taicpu(next).oper[2]^.val in [2,3,4,5,8,9]) then
  10071. { We can convert "imul x,%reg1,%reg2" (where x = 2, 4 or 8)
  10072. to "lea (%reg1,x),%reg2". If x = 3, 5 or 9, we can
  10073. change this to "lea (%reg1,%reg1,(x-1)),%reg2" }
  10074. Exit
  10075. else
  10076. PotentialModified := True;
  10077. end;
  10078. else
  10079. { Don't know how to change this, so abort }
  10080. Exit;
  10081. end;
  10082. { Contains highest index (so instruction count - 1) }
  10083. Inc(InstrMax);
  10084. if InstrMax > High(InstrList) then
  10085. SetLength(InstrList, InstrMax + LIST_STEP_SIZE);
  10086. InstrList[InstrMax] := taicpu(next);
  10087. end;
  10088. UpdateUsedRegs(TmpUsedRegs, tai(next.next));
  10089. end;
  10090. if not Assigned(next) or (next <> hp1) then
  10091. { It should be equal to hp1 }
  10092. InternalError(2021051702);
  10093. { Cycle through each instruction and check to see if we can
  10094. change them to versions that don't modify the flags }
  10095. if (InstrMax >= 0) then
  10096. begin
  10097. for Index := 0 to InstrMax do
  10098. case InstrList[Index].opcode of
  10099. A_ADD:
  10100. begin
  10101. DebugMsg(SPeepholeOptimization + 'ADD -> LEA' + SNoFlags, InstrList[Index]);
  10102. InstrList[Index].opcode := A_LEA;
  10103. reference_reset(NewRef, 1, []);
  10104. NewRef.base := InstrList[Index].oper[1]^.reg;
  10105. if InstrList[Index].oper[0]^.typ = top_reg then
  10106. begin
  10107. NewRef.index := InstrList[Index].oper[0]^.reg;
  10108. NewRef.scalefactor := 1;
  10109. end
  10110. else
  10111. NewRef.offset := InstrList[Index].oper[0]^.val;
  10112. InstrList[Index].loadref(0, NewRef);
  10113. end;
  10114. A_SUB:
  10115. begin
  10116. DebugMsg(SPeepholeOptimization + 'SUB -> LEA' + SNoFlags, InstrList[Index]);
  10117. InstrList[Index].opcode := A_LEA;
  10118. reference_reset(NewRef, 1, []);
  10119. NewRef.base := InstrList[Index].oper[1]^.reg;
  10120. NewRef.offset := -InstrList[Index].oper[0]^.val;
  10121. InstrList[Index].loadref(0, NewRef);
  10122. end;
  10123. A_SHL,
  10124. A_SAL:
  10125. begin
  10126. DebugMsg(SPeepholeOptimization + 'SHL -> LEA' + SNoFlags, InstrList[Index]);
  10127. InstrList[Index].opcode := A_LEA;
  10128. reference_reset(NewRef, 1, []);
  10129. NewRef.index := InstrList[Index].oper[1]^.reg;
  10130. NewRef.scalefactor := 1 shl (InstrList[Index].oper[0]^.val);
  10131. InstrList[Index].loadref(0, NewRef);
  10132. end;
  10133. A_IMUL:
  10134. begin
  10135. DebugMsg(SPeepholeOptimization + 'IMUL -> LEA' + SNoFlags, InstrList[Index]);
  10136. InstrList[Index].opcode := A_LEA;
  10137. reference_reset(NewRef, 1, []);
  10138. NewRef.index := InstrList[Index].oper[1]^.reg;
  10139. case InstrList[Index].oper[0]^.val of
  10140. 2, 4, 8:
  10141. NewRef.scalefactor := InstrList[Index].oper[0]^.val;
  10142. else {3, 5 and 9}
  10143. begin
  10144. NewRef.scalefactor := InstrList[Index].oper[0]^.val - 1;
  10145. NewRef.base := InstrList[Index].oper[1]^.reg;
  10146. end;
  10147. end;
  10148. InstrList[Index].loadref(0, NewRef);
  10149. end;
  10150. else
  10151. InternalError(2021051710);
  10152. end;
  10153. end;
  10154. { Mark the FLAGS register as used across this whole block }
  10155. AllocRegBetween(NR_DEFAULTFLAGS, p, hp1, UsedRegs);
  10156. end;
  10157. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  10158. JumpC := taicpu(hp2).condition;
  10159. Unconditional := False;
  10160. if conditions_equal(JumpC, C_E) then
  10161. SetC := inverse_cond(taicpu(p).condition)
  10162. else if conditions_equal(JumpC, C_NE) then
  10163. SetC := taicpu(p).condition
  10164. else
  10165. { We've got something weird here (and inefficent) }
  10166. begin
  10167. DebugMsg('DEBUG: Inefficient jump - check code generation', p);
  10168. SetC := C_NONE;
  10169. { JAE/JNB will always branch (use 'condition_in', since C_AE <> C_NB normally) }
  10170. if condition_in(C_AE, JumpC) then
  10171. Unconditional := True
  10172. else
  10173. { Not sure what to do with this jump - drop out }
  10174. Exit;
  10175. end;
  10176. RemoveInstruction(hp1);
  10177. if Unconditional then
  10178. MakeUnconditional(taicpu(hp2))
  10179. else
  10180. begin
  10181. if SetC = C_NONE then
  10182. InternalError(2018061402);
  10183. taicpu(hp2).SetCondition(SetC);
  10184. end;
  10185. { as hp2 is a jump, we cannot use RegUsedAfterInstruction but we have to check if it is included in
  10186. TmpUsedRegs }
  10187. if not TmpUsedRegs[getregtype(taicpu(p).oper[0]^.reg)].IsUsed(taicpu(p).oper[0]^.reg) then
  10188. begin
  10189. RemoveCurrentp(p, hp2);
  10190. if taicpu(hp2).opcode = A_SETcc then
  10191. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/SETcc -> SETcc',p)
  10192. else
  10193. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/Jcc -> Jcc',p);
  10194. end
  10195. else
  10196. if taicpu(hp2).opcode = A_SETcc then
  10197. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/SETcc -> SETcc/SETcc',p)
  10198. else
  10199. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/Jcc -> SETcc/Jcc',p);
  10200. Result := True;
  10201. end
  10202. else if
  10203. { Make sure the instructions are adjacent }
  10204. (
  10205. not (cs_opt_level3 in current_settings.optimizerswitches) or
  10206. GetNextInstruction(p, hp1)
  10207. ) and
  10208. MatchInstruction(hp1, A_MOV, [S_B]) and
  10209. { Writing to memory is allowed }
  10210. MatchOperand(taicpu(p).oper[0]^, taicpu(hp1).oper[0]^.reg) then
  10211. begin
  10212. {
  10213. Watch out for sequences such as:
  10214. set(c)b %regb
  10215. movb %regb,(ref)
  10216. movb $0,1(ref)
  10217. movb $0,2(ref)
  10218. movb $0,3(ref)
  10219. Much more efficient to turn it into:
  10220. movl $0,%regl
  10221. set(c)b %regb
  10222. movl %regl,(ref)
  10223. Or:
  10224. set(c)b %regb
  10225. movzbl %regb,%regl
  10226. movl %regl,(ref)
  10227. }
  10228. if (taicpu(hp1).oper[1]^.typ = top_ref) and
  10229. GetNextInstruction(hp1, hp2) and
  10230. MatchInstruction(hp2, A_MOV, [S_B]) and
  10231. (taicpu(hp2).oper[1]^.typ = top_ref) and
  10232. CheckMemoryWrite(taicpu(hp1), taicpu(hp2)) then
  10233. begin
  10234. { Don't do anything else except set Result to True }
  10235. end
  10236. else
  10237. begin
  10238. if taicpu(p).oper[0]^.typ = top_reg then
  10239. begin
  10240. TransferUsedRegs(TmpUsedRegs);
  10241. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  10242. end;
  10243. { If it's not a register, it's a memory address }
  10244. if (taicpu(p).oper[0]^.typ <> top_reg) or RegUsedAfterInstruction(taicpu(p).oper[0]^.reg, hp1, TmpUsedRegs) then
  10245. begin
  10246. { Even if the register is still in use, we can minimise the
  10247. pipeline stall by changing the MOV into another SETcc. }
  10248. taicpu(hp1).opcode := A_SETcc;
  10249. taicpu(hp1).condition := taicpu(p).condition;
  10250. if taicpu(hp1).oper[1]^.typ = top_ref then
  10251. begin
  10252. { Swapping the operand pointers like this is probably a
  10253. bit naughty, but it is far faster than using loadoper
  10254. to transfer the reference from oper[1] to oper[0] if
  10255. you take into account the extra procedure calls and
  10256. the memory allocation and deallocation required }
  10257. OperPtr := taicpu(hp1).oper[1];
  10258. taicpu(hp1).oper[1] := taicpu(hp1).oper[0];
  10259. taicpu(hp1).oper[0] := OperPtr;
  10260. end
  10261. else
  10262. taicpu(hp1).oper[0]^.reg := taicpu(hp1).oper[1]^.reg;
  10263. taicpu(hp1).clearop(1);
  10264. taicpu(hp1).ops := 1;
  10265. DebugMsg(SPeepholeOptimization + 'SETcc/Mov -> SETcc/SETcc',p);
  10266. end
  10267. else
  10268. begin
  10269. if taicpu(hp1).oper[1]^.typ = top_reg then
  10270. AllocRegBetween(taicpu(hp1).oper[1]^.reg,p,hp1,UsedRegs);
  10271. taicpu(p).loadoper(0, taicpu(hp1).oper[1]^);
  10272. RemoveInstruction(hp1);
  10273. DebugMsg(SPeepholeOptimization + 'SETcc/Mov -> SETcc',p);
  10274. end
  10275. end;
  10276. Result := True;
  10277. end;
  10278. end;
  10279. end;
  10280. function TX86AsmOptimizer.OptPass2Jmp(var p : tai) : boolean;
  10281. var
  10282. hp1: tai;
  10283. Count: Integer;
  10284. OrigLabel: TAsmLabel;
  10285. begin
  10286. result := False;
  10287. { Sometimes, the optimisations below can permit this }
  10288. RemoveDeadCodeAfterJump(p);
  10289. if (taicpu(p).oper[0]^.typ=top_ref) and (taicpu(p).oper[0]^.ref^.refaddr=addr_full) and (taicpu(p).oper[0]^.ref^.base=NR_NO) and
  10290. (taicpu(p).oper[0]^.ref^.index=NR_NO) and (taicpu(p).oper[0]^.ref^.symbol is tasmlabel) then
  10291. begin
  10292. OrigLabel := TAsmLabel(taicpu(p).oper[0]^.ref^.symbol);
  10293. { Also a side-effect of optimisations }
  10294. if CollapseZeroDistJump(p, OrigLabel) then
  10295. begin
  10296. Result := True;
  10297. Exit;
  10298. end;
  10299. hp1 := GetLabelWithSym(OrigLabel);
  10300. if (taicpu(p).condition=C_None) and assigned(hp1) and SkipLabels(hp1,hp1) and (hp1.typ = ait_instruction) then
  10301. begin
  10302. if taicpu(hp1).opcode = A_RET then
  10303. begin
  10304. {
  10305. change
  10306. jmp .L1
  10307. ...
  10308. .L1:
  10309. ret
  10310. into
  10311. ret
  10312. }
  10313. begin
  10314. ConvertJumpToRET(p, hp1);
  10315. result:=true;
  10316. end;
  10317. end
  10318. else if (cs_opt_level3 in current_settings.optimizerswitches) and
  10319. not (cs_opt_size in current_settings.optimizerswitches) and
  10320. CheckJumpMovTransferOpt(p, hp1, 0, Count) then
  10321. begin
  10322. Result := True;
  10323. Exit;
  10324. end;
  10325. end;
  10326. end;
  10327. end;
  10328. class function TX86AsmOptimizer.CanBeCMOV(p, cond_p: tai) : boolean;
  10329. begin
  10330. Result := assigned(p) and
  10331. MatchInstruction(p,A_MOV,[S_W,S_L,S_Q]) and
  10332. (taicpu(p).oper[1]^.typ = top_reg) and
  10333. (
  10334. (taicpu(p).oper[0]^.typ = top_reg) or
  10335. { allow references, but only pure symbols or got rel. addressing with RIP as based,
  10336. it is not expected that this can cause a seg. violation }
  10337. (
  10338. (taicpu(p).oper[0]^.typ = top_ref) and
  10339. { TODO: Can we detect which references become constants at this
  10340. stage so we don't have to do a blanket ban? }
  10341. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) and
  10342. (
  10343. IsRefSafe(taicpu(p).oper[0]^.ref) or
  10344. (
  10345. { If the reference also appears in the condition, then we know it's safe, otherwise
  10346. any kind of access violation would have occurred already }
  10347. Assigned(cond_p) and
  10348. { Make sure the sizes match too so we're reading and writing the same number of bytes }
  10349. (cond_p.typ = ait_instruction) and
  10350. (taicpu(cond_p).opsize = taicpu(p).opsize) and
  10351. { Just consider 2-operand comparison instructions for now to be safe }
  10352. (taicpu(cond_p).ops = 2) and
  10353. (
  10354. ((taicpu(cond_p).oper[1]^.typ = top_ref) and RefsEqual(taicpu(cond_p).oper[1]^.ref^, taicpu(p).oper[0]^.ref^)) or
  10355. (
  10356. (taicpu(cond_p).oper[0]^.typ = top_ref) and
  10357. { Don't risk identical registers but different offsets, as we may have constructs
  10358. such as buffer streams with things like length fields that indicate whether
  10359. any more data follows. And there are probably some contrived examples where
  10360. writing to offsets behind the one being read also lead to access violations }
  10361. RefsEqual(taicpu(cond_p).oper[0]^.ref^, taicpu(p).oper[0]^.ref^) and
  10362. (
  10363. { Check that we're not modifying a register that appears in the reference }
  10364. (InsProp[taicpu(cond_p).opcode].Ch * [Ch_Mop2, Ch_RWop2, Ch_Wop2] = []) or
  10365. (taicpu(cond_p).oper[1]^.typ <> top_reg) or
  10366. not RegInRef(taicpu(cond_p).oper[1]^.reg, taicpu(cond_p).oper[0]^.ref^)
  10367. )
  10368. )
  10369. )
  10370. )
  10371. )
  10372. )
  10373. );
  10374. end;
  10375. class procedure TX86AsmOptimizer.UpdateIntRegsNoDealloc(var AUsedRegs: TAllUsedRegs; p: Tai);
  10376. begin
  10377. { Update integer registers, ignoring deallocations }
  10378. repeat
  10379. while assigned(p) and
  10380. ((p.typ in (SkipInstr - [ait_RegAlloc])) or
  10381. (p.typ = ait_label) or
  10382. ((p.typ = ait_marker) and
  10383. (tai_Marker(p).Kind in [mark_AsmBlockEnd,mark_NoLineInfoStart,mark_NoLineInfoEnd]))) do
  10384. p := tai(p.next);
  10385. while assigned(p) and
  10386. (p.typ=ait_RegAlloc) Do
  10387. begin
  10388. if (getregtype(tai_regalloc(p).reg) = R_INTREGISTER) then
  10389. begin
  10390. case tai_regalloc(p).ratype of
  10391. ra_alloc :
  10392. IncludeRegInUsedRegs(tai_regalloc(p).reg, AUsedRegs);
  10393. else
  10394. ;
  10395. end;
  10396. end;
  10397. p := tai(p.next);
  10398. end;
  10399. until not(assigned(p)) or
  10400. (not(p.typ in SkipInstr) and
  10401. not((p.typ = ait_label) and
  10402. labelCanBeSkipped(tai_label(p))));
  10403. end;
  10404. function TX86AsmOptimizer.OptPass2Jcc(var p : tai) : boolean;
  10405. var
  10406. hp1,hp2: tai;
  10407. carryadd_opcode : TAsmOp;
  10408. symbol: TAsmSymbol;
  10409. increg, tmpreg: TRegister;
  10410. {$ifndef i8086}
  10411. { Code and variables specific to CMOV optimisations }
  10412. hp3,hp4,hp5,
  10413. hp_stop, hp_lblxxx, hp_lblyyy, hpmov1,hpmov2, hp_prev, hp_flagalloc, hp_prev2, hp_new, hp_jump: tai;
  10414. l, c, w, x : Longint;
  10415. condition, second_condition : TAsmCond;
  10416. FoundMatchingJump, RegMatch: Boolean;
  10417. RegWrites: array[0..MAX_CMOV_INSTRUCTIONS*2 - 1] of TRegister;
  10418. ConstRegs: array[0..MAX_CMOV_REGISTERS - 1] of TRegister;
  10419. ConstVals: array[0..MAX_CMOV_REGISTERS - 1] of TCGInt;
  10420. { Tries to convert a mov const,%reg instruction into a CMOV by reserving a
  10421. new register to store the constant }
  10422. function TryCMOVConst(p, search_start_p, stop_search_p: tai; var StoredCount: LongInt; var CMOVCount: LongInt): Boolean;
  10423. var
  10424. RegSize: TSubRegister;
  10425. CurrentVal: TCGInt;
  10426. NewReg: TRegister;
  10427. X: ShortInt;
  10428. begin
  10429. Result := False;
  10430. if not MatchOpType(taicpu(p), top_const, top_reg) then
  10431. Exit;
  10432. if StoredCount >= MAX_CMOV_REGISTERS then
  10433. { Arrays are full }
  10434. Exit;
  10435. { Remember that CMOV can't encode 8-bit registers }
  10436. case taicpu(p).opsize of
  10437. S_W:
  10438. RegSize := R_SUBW;
  10439. S_L:
  10440. RegSize := R_SUBD;
  10441. S_Q:
  10442. RegSize := R_SUBQ;
  10443. else
  10444. InternalError(2021100401);
  10445. end;
  10446. { See if the value has already been reserved for another CMOV instruction }
  10447. CurrentVal := taicpu(p).oper[0]^.val;
  10448. for X := 0 to StoredCount - 1 do
  10449. if ConstVals[X] = CurrentVal then
  10450. begin
  10451. ConstRegs[StoredCount] := ConstRegs[X];
  10452. ConstVals[StoredCount] := CurrentVal;
  10453. Result := True;
  10454. Inc(StoredCount);
  10455. { Don't increase CMOVCount this time, since we're re-using a register }
  10456. Exit;
  10457. end;
  10458. NewReg := GetIntRegisterBetween(RegSize, TmpUsedRegs, search_start_p, stop_search_p, True);
  10459. if NewReg = NR_NO then
  10460. { No free registers }
  10461. Exit;
  10462. { Reserve the register so subsequent TryCMOVConst calls don't all end
  10463. up vying for the same register }
  10464. IncludeRegInUsedRegs(NewReg, TmpUsedRegs);
  10465. ConstRegs[StoredCount] := NewReg;
  10466. ConstVals[StoredCount] := CurrentVal;
  10467. Inc(StoredCount);
  10468. { Increment the CMOV count variable from OptPass2JCC, since the extra
  10469. MOV required adds complexity and will cause diminishing returns
  10470. sooner than normal. This is more of an approximate weighting than
  10471. anything else. }
  10472. Inc(CMOVCount);
  10473. Result := True;
  10474. end;
  10475. {$endif i8086}
  10476. begin
  10477. result:=false;
  10478. if GetNextInstruction(p,hp1) then
  10479. begin
  10480. if (hp1.typ=ait_label) then
  10481. begin
  10482. Result := DoSETccLblRETOpt(p, tai_label(hp1));
  10483. Exit;
  10484. end
  10485. else if (hp1.typ<>ait_instruction) then
  10486. Exit;
  10487. symbol := TAsmLabel(taicpu(p).oper[0]^.ref^.symbol);
  10488. if (
  10489. (
  10490. ((Taicpu(hp1).opcode=A_ADD) or (Taicpu(hp1).opcode=A_SUB)) and
  10491. MatchOptype(Taicpu(hp1),top_const,top_reg) and
  10492. (Taicpu(hp1).oper[0]^.val=1)
  10493. ) or
  10494. ((Taicpu(hp1).opcode=A_INC) or (Taicpu(hp1).opcode=A_DEC))
  10495. ) and
  10496. GetNextInstruction(hp1,hp2) and
  10497. SkipAligns(hp2, hp2) and
  10498. (hp2.typ = ait_label) and
  10499. (Tasmlabel(symbol) = Tai_label(hp2).labsym) then
  10500. { jb @@1 cmc
  10501. inc/dec operand --> adc/sbb operand,0
  10502. @@1:
  10503. ... and ...
  10504. jnb @@1
  10505. inc/dec operand --> adc/sbb operand,0
  10506. @@1: }
  10507. begin
  10508. if Taicpu(p).condition in [C_NAE,C_B,C_C] then
  10509. begin
  10510. case taicpu(hp1).opcode of
  10511. A_INC,
  10512. A_ADD:
  10513. carryadd_opcode:=A_ADC;
  10514. A_DEC,
  10515. A_SUB:
  10516. carryadd_opcode:=A_SBB;
  10517. else
  10518. InternalError(2021011001);
  10519. end;
  10520. Taicpu(p).clearop(0);
  10521. Taicpu(p).ops:=0;
  10522. Taicpu(p).is_jmp:=false;
  10523. Taicpu(p).opcode:=A_CMC;
  10524. Taicpu(p).condition:=C_NONE;
  10525. DebugMsg(SPeepholeOptimization+'JccAdd/Inc/Dec2CmcAdc/Sbb',p);
  10526. Taicpu(hp1).ops:=2;
  10527. if (Taicpu(hp1).opcode=A_ADD) or (Taicpu(hp1).opcode=A_SUB) then
  10528. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[1]^)
  10529. else
  10530. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[0]^);
  10531. Taicpu(hp1).loadconst(0,0);
  10532. Taicpu(hp1).opcode:=carryadd_opcode;
  10533. result:=true;
  10534. exit;
  10535. end
  10536. else if Taicpu(p).condition in [C_AE,C_NB,C_NC] then
  10537. begin
  10538. case taicpu(hp1).opcode of
  10539. A_INC,
  10540. A_ADD:
  10541. carryadd_opcode:=A_ADC;
  10542. A_DEC,
  10543. A_SUB:
  10544. carryadd_opcode:=A_SBB;
  10545. else
  10546. InternalError(2021011002);
  10547. end;
  10548. Taicpu(hp1).ops:=2;
  10549. DebugMsg(SPeepholeOptimization+'JccAdd/Inc/Dec2Adc/Sbb',p);
  10550. if (Taicpu(hp1).opcode=A_ADD) or (Taicpu(hp1).opcode=A_SUB) then
  10551. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[1]^)
  10552. else
  10553. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[0]^);
  10554. Taicpu(hp1).loadconst(0,0);
  10555. Taicpu(hp1).opcode:=carryadd_opcode;
  10556. RemoveCurrentP(p, hp1);
  10557. result:=true;
  10558. exit;
  10559. end
  10560. {
  10561. jcc @@1 setcc tmpreg
  10562. inc/dec/add/sub operand -> (movzx tmpreg)
  10563. @@1: add/sub tmpreg,operand
  10564. While this increases code size slightly, it makes the code much faster if the
  10565. jump is unpredictable
  10566. }
  10567. else if not(cs_opt_size in current_settings.optimizerswitches) then
  10568. begin
  10569. { search for an available register which is volatile }
  10570. increg := GetIntRegisterBetween(R_SUBL, UsedRegs, p, hp1);
  10571. if increg <> NR_NO then
  10572. begin
  10573. { We don't need to check if tmpreg is in hp1 or not, because
  10574. it will be marked as in use at p (if not, this is
  10575. indictive of a compiler bug). }
  10576. TAsmLabel(symbol).decrefs;
  10577. Taicpu(p).clearop(0);
  10578. Taicpu(p).ops:=1;
  10579. Taicpu(p).is_jmp:=false;
  10580. Taicpu(p).opcode:=A_SETcc;
  10581. DebugMsg(SPeepholeOptimization+'JccAdd2SetccAdd',p);
  10582. Taicpu(p).condition:=inverse_cond(Taicpu(p).condition);
  10583. Taicpu(p).loadreg(0,increg);
  10584. if getsubreg(Taicpu(hp1).oper[1]^.reg)<>R_SUBL then
  10585. begin
  10586. case getsubreg(Taicpu(hp1).oper[1]^.reg) of
  10587. R_SUBW:
  10588. begin
  10589. tmpreg := newreg(R_INTREGISTER,getsupreg(increg),R_SUBW);
  10590. hp2:=Taicpu.op_reg_reg(A_MOVZX,S_BW,increg,tmpreg);
  10591. end;
  10592. R_SUBD:
  10593. begin
  10594. tmpreg := newreg(R_INTREGISTER,getsupreg(increg),R_SUBD);
  10595. hp2:=Taicpu.op_reg_reg(A_MOVZX,S_BL,increg,tmpreg);
  10596. end;
  10597. {$ifdef x86_64}
  10598. R_SUBQ:
  10599. begin
  10600. { MOVZX doesn't have a 64-bit variant, because
  10601. the 32-bit version implicitly zeroes the
  10602. upper 32-bits of the destination register }
  10603. tmpreg := newreg(R_INTREGISTER,getsupreg(increg),R_SUBD);
  10604. hp2:=Taicpu.op_reg_reg(A_MOVZX,S_BL,increg,tmpreg);
  10605. setsubreg(tmpreg, R_SUBQ);
  10606. end;
  10607. {$endif x86_64}
  10608. else
  10609. Internalerror(2020030601);
  10610. end;
  10611. taicpu(hp2).fileinfo:=taicpu(hp1).fileinfo;
  10612. asml.InsertAfter(hp2,p);
  10613. end
  10614. else
  10615. tmpreg := increg;
  10616. if (Taicpu(hp1).opcode=A_INC) or (Taicpu(hp1).opcode=A_DEC) then
  10617. begin
  10618. Taicpu(hp1).ops:=2;
  10619. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[0]^)
  10620. end;
  10621. Taicpu(hp1).loadreg(0,tmpreg);
  10622. AllocRegBetween(tmpreg,p,hp1,UsedRegs);
  10623. Result := True;
  10624. { p is no longer a Jcc instruction, so exit }
  10625. Exit;
  10626. end;
  10627. end;
  10628. end;
  10629. { Detect the following:
  10630. jmp<cond> @Lbl1
  10631. jmp @Lbl2
  10632. ...
  10633. @Lbl1:
  10634. ret
  10635. Change to:
  10636. jmp<inv_cond> @Lbl2
  10637. ret
  10638. }
  10639. if MatchInstruction(hp1,A_JMP,[]) and (taicpu(hp1).oper[0]^.ref^.refaddr=addr_full) then
  10640. begin
  10641. hp2:=getlabelwithsym(TAsmLabel(symbol));
  10642. if Assigned(hp2) and SkipLabels(hp2,hp2) and
  10643. MatchInstruction(hp2,A_RET,[S_NO]) then
  10644. begin
  10645. taicpu(p).condition := inverse_cond(taicpu(p).condition);
  10646. { Change label address to that of the unconditional jump }
  10647. taicpu(p).loadoper(0, taicpu(hp1).oper[0]^);
  10648. TAsmLabel(symbol).DecRefs;
  10649. taicpu(hp1).opcode := A_RET;
  10650. taicpu(hp1).is_jmp := false;
  10651. taicpu(hp1).ops := taicpu(hp2).ops;
  10652. DebugMsg(SPeepholeOptimization+'JccJmpRet2J!ccRet',p);
  10653. case taicpu(hp2).ops of
  10654. 0:
  10655. taicpu(hp1).clearop(0);
  10656. 1:
  10657. taicpu(hp1).loadconst(0,taicpu(hp2).oper[0]^.val);
  10658. else
  10659. internalerror(2016041302);
  10660. end;
  10661. end;
  10662. {$ifndef i8086}
  10663. end
  10664. {
  10665. convert
  10666. j<c> .L1
  10667. mov 1,reg
  10668. jmp .L2
  10669. .L1
  10670. mov 0,reg
  10671. .L2
  10672. into
  10673. mov 0,reg
  10674. set<not(c)> reg
  10675. take care of alignment and that the mov 0,reg is not converted into a xor as this
  10676. would destroy the flag contents
  10677. }
  10678. else if MatchInstruction(hp1,A_MOV,[]) and
  10679. MatchOpType(taicpu(hp1),top_const,top_reg) and
  10680. {$ifdef i386}
  10681. (
  10682. { Under i386, ESI, EDI, EBP and ESP
  10683. don't have an 8-bit representation }
  10684. not (getsupreg(taicpu(hp1).oper[1]^.reg) in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  10685. ) and
  10686. {$endif i386}
  10687. (taicpu(hp1).oper[0]^.val=1) and
  10688. GetNextInstruction(hp1,hp2) and
  10689. MatchInstruction(hp2,A_JMP,[]) and (taicpu(hp2).oper[0]^.ref^.refaddr=addr_full) and
  10690. GetNextInstruction(hp2,hp3) and
  10691. { skip align }
  10692. ((hp3.typ<>ait_align) or GetNextInstruction(hp3,hp3)) and
  10693. (hp3.typ=ait_label) and
  10694. (tasmlabel(taicpu(p).oper[0]^.ref^.symbol)=tai_label(hp3).labsym) and
  10695. (tai_label(hp3).labsym.getrefs=1) and
  10696. GetNextInstruction(hp3,hp4) and
  10697. MatchInstruction(hp4,A_MOV,[]) and
  10698. MatchOpType(taicpu(hp4),top_const,top_reg) and
  10699. (taicpu(hp4).oper[0]^.val=0) and
  10700. MatchOperand(taicpu(hp1).oper[1]^,taicpu(hp4).oper[1]^) and
  10701. GetNextInstruction(hp4,hp5) and
  10702. (hp5.typ=ait_label) and
  10703. (tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol)=tai_label(hp5).labsym) and
  10704. (tai_label(hp5).labsym.getrefs=1) then
  10705. begin
  10706. AllocRegBetween(NR_FLAGS,p,hp4,UsedRegs);
  10707. DebugMsg(SPeepholeOptimization+'JccMovJmpMov2MovSetcc',p);
  10708. { remove last label }
  10709. RemoveInstruction(hp5);
  10710. { remove second label }
  10711. RemoveInstruction(hp3);
  10712. { if align is present remove it }
  10713. if GetNextInstruction(hp2,hp3) and (hp3.typ=ait_align) then
  10714. RemoveInstruction(hp3);
  10715. { remove jmp }
  10716. RemoveInstruction(hp2);
  10717. if taicpu(hp1).opsize=S_B then
  10718. RemoveInstruction(hp1)
  10719. else
  10720. taicpu(hp1).loadconst(0,0);
  10721. taicpu(hp4).opcode:=A_SETcc;
  10722. taicpu(hp4).opsize:=S_B;
  10723. taicpu(hp4).condition:=inverse_cond(taicpu(p).condition);
  10724. taicpu(hp4).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(hp4).oper[1]^.reg),R_SUBL));
  10725. taicpu(hp4).opercnt:=1;
  10726. taicpu(hp4).ops:=1;
  10727. taicpu(hp4).freeop(1);
  10728. RemoveCurrentP(p);
  10729. Result:=true;
  10730. exit;
  10731. end
  10732. else if (CPUX86_HAS_CMOV in cpu_capabilities[current_settings.optimizecputype]) and
  10733. MatchInstruction(hp1,A_MOV,[S_W,S_L{$ifdef x86_64},S_Q{$endif x86_64}]) then
  10734. begin
  10735. { check for
  10736. jCC xxx
  10737. <several movs>
  10738. xxx:
  10739. Also spot:
  10740. Jcc xxx
  10741. <several movs>
  10742. jmp xxx
  10743. Change to:
  10744. <several cmovs with inverted condition>
  10745. jmp xxx (only for the 2nd case)
  10746. }
  10747. hp2 := p;
  10748. hp_lblxxx := hp1;
  10749. hp_flagalloc := nil;
  10750. hp_stop := nil;
  10751. FoundMatchingJump := False;
  10752. { Remember the first instruction in the first block of MOVs }
  10753. hpmov1 := hp1;
  10754. TransferUsedRegs(TmpUsedRegs);
  10755. while assigned(hp_lblxxx) and
  10756. { stop on labels }
  10757. (hp_lblxxx.typ <> ait_label) do
  10758. begin
  10759. { Keep track of all integer registers that are used }
  10760. UpdateIntRegsNoDealloc(TmpUsedRegs, tai(hp2.Next));
  10761. if hp_lblxxx.typ = ait_instruction then
  10762. begin
  10763. if (taicpu(hp_lblxxx).opcode = A_JMP) and
  10764. IsJumpToLabel(taicpu(hp_lblxxx)) then
  10765. begin
  10766. hp_stop := hp_lblxxx;
  10767. if (TAsmLabel(taicpu(hp_lblxxx).oper[0]^.ref^.symbol) = symbol) then
  10768. begin
  10769. { We found Jcc xxx; <several movs>; Jmp xxx }
  10770. FoundMatchingJump := True;
  10771. Break;
  10772. end;
  10773. { If it's not the jump we're looking for, it's
  10774. possibly the "if..else" variant }
  10775. end
  10776. { Check to see if we have a valid MOV instruction instead }
  10777. else if (taicpu(hp_lblxxx).opcode <> A_MOV) or
  10778. not (taicpu(hp_lblxxx).opsize in [S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  10779. Break
  10780. else
  10781. { This will be a valid MOV }
  10782. hp_stop := hp_lblxxx;
  10783. end;
  10784. hp2 := hp_lblxxx;
  10785. GetNextInstruction(hp_lblxxx, hp_lblxxx);
  10786. end;
  10787. { Just make sure the last MOV is included if there's no jump }
  10788. if (hp_lblxxx.typ = ait_label) and MatchInstruction(hp_stop, A_MOV, []) then
  10789. hp_stop := hp_lblxxx;
  10790. { Note, the logic behind using hp_stop over hp_lblxxx in the
  10791. range for TryCMOVConst is so GetIntRegisterBetween doesn't
  10792. fail when it reaches a JMP instruction in the "jcc xxx; movs;
  10793. jmp yyy; xxx:; movs; yyy:" variation }
  10794. if assigned(hp_lblxxx) and
  10795. (
  10796. { If we found JMP xxx, we don't actually need a label
  10797. (hp_lblxxx is the JMP instruction instead) }
  10798. FoundMatchingJump or
  10799. { Make sure we actually have the right label }
  10800. FindLabel(TAsmLabel(symbol), hp_lblxxx)
  10801. ) then
  10802. begin
  10803. { Use TmpUsedRegs to track registers that we reserve }
  10804. { When allocating temporary registers, try to look one
  10805. instruction back, as defining them before a CMP or TEST
  10806. instruction will be faster, and also avoid picking a
  10807. register that was only just deallocated }
  10808. if GetLastInstruction(p, hp_prev) and
  10809. MatchInstruction(hp_prev, [A_CMP, A_TEST, A_BSR, A_BSF, A_COMISS, A_COMISD, A_UCOMISS, A_UCOMISD, A_VCOMISS, A_VCOMISD, A_VUCOMISS, A_VUCOMISD], []) then
  10810. begin
  10811. { Mark all the registers in the comparison as 'in use', even if they've just been deallocated }
  10812. for l := 0 to 1 do
  10813. with taicpu(hp_prev).oper[l]^ do
  10814. case typ of
  10815. top_reg:
  10816. if getregtype(reg) = R_INTREGISTER then
  10817. IncludeRegInUsedRegs(reg, TmpUsedRegs);
  10818. top_ref:
  10819. begin
  10820. if
  10821. {$ifdef x86_64}
  10822. (ref^.base <> NR_RIP) and
  10823. {$endif x86_64}
  10824. (ref^.base <> NR_NO) then
  10825. IncludeRegInUsedRegs(ref^.base, TmpUsedRegs);
  10826. if (ref^.index <> NR_NO) then
  10827. IncludeRegInUsedRegs(ref^.index, TmpUsedRegs);
  10828. end
  10829. else
  10830. ;
  10831. end;
  10832. { When inserting instructions before hp_prev, try to insert
  10833. them before the allocation of the FLAGS register }
  10834. if not SetAndTest(FindRegAllocBackward(NR_DEFAULTFLAGS, tai(hp_prev.Previous)), hp_flagalloc) then
  10835. { If not found, set it equal to hp_prev so it's something sensible }
  10836. hp_flagalloc := hp_prev;
  10837. hp_prev2 := nil;
  10838. { When dealing with a comparison against zero, take
  10839. note of the instruction before it to see if we can
  10840. move instructions further back in order to benefit
  10841. PostPeepholeOptTestOr.
  10842. }
  10843. if (
  10844. (
  10845. (taicpu(hp_prev).opcode = A_CMP) and
  10846. MatchOperand(taicpu(hp_prev).oper[0]^, 0)
  10847. ) or
  10848. (
  10849. (taicpu(hp_prev).opcode = A_TEST) and
  10850. (
  10851. OpsEqual(taicpu(hp_prev).oper[0]^, taicpu(hp_prev).oper[1]^) or
  10852. MatchOperand(taicpu(hp_prev).oper[0]^, -1)
  10853. )
  10854. )
  10855. ) and
  10856. GetLastInstruction(hp_prev, hp_prev2) then
  10857. begin
  10858. if (hp_prev2.typ = ait_instruction) and
  10859. { These instructions set the zero flag if the result is zero }
  10860. MatchInstruction(hp_prev2, [A_ADD, A_SUB, A_OR, A_XOR, A_AND, A_POPCNT, A_LZCNT], []) then
  10861. begin
  10862. { Also mark all the registers in this previous instruction
  10863. as 'in use', even if they've just been deallocated }
  10864. for l := 0 to 1 do
  10865. with taicpu(hp_prev2).oper[l]^ do
  10866. case typ of
  10867. top_reg:
  10868. if getregtype(reg) = R_INTREGISTER then
  10869. IncludeRegInUsedRegs(reg, TmpUsedRegs);
  10870. top_ref:
  10871. begin
  10872. if
  10873. {$ifdef x86_64}
  10874. (ref^.base <> NR_RIP) and
  10875. {$endif x86_64}
  10876. (ref^.base <> NR_NO) then
  10877. IncludeRegInUsedRegs(ref^.base, TmpUsedRegs);
  10878. if (ref^.index <> NR_NO) then
  10879. IncludeRegInUsedRegs(ref^.index, TmpUsedRegs);
  10880. end
  10881. else
  10882. ;
  10883. end;
  10884. end
  10885. else
  10886. { Unsuitable instruction }
  10887. hp_prev2 := nil;
  10888. end;
  10889. end
  10890. else
  10891. begin
  10892. hp_prev := p;
  10893. { When inserting instructions before hp_prev, try to insert
  10894. them before the allocation of the FLAGS register }
  10895. if not SetAndTest(FindRegAllocBackward(NR_DEFAULTFLAGS, tai(p.Previous)), hp_flagalloc) then
  10896. { If not found, set it equal to p so it's something sensible }
  10897. hp_flagalloc := p;
  10898. hp_prev2 := nil;
  10899. end;
  10900. l := 0;
  10901. c := 0;
  10902. { Initialise RegWrites, ConstRegs and ConstVals }
  10903. FillChar(RegWrites[0], MAX_CMOV_INSTRUCTIONS * 2 * SizeOf(TRegister), 0);
  10904. FillChar(ConstRegs[0], MAX_CMOV_REGISTERS * SizeOf(TRegister), 0);
  10905. FillChar(ConstVals[0], MAX_CMOV_REGISTERS * SizeOf(TCGInt), 0);
  10906. while assigned(hp1) and
  10907. { Stop on the label we found }
  10908. (hp1 <> hp_lblxxx) do
  10909. begin
  10910. case hp1.typ of
  10911. ait_instruction:
  10912. if MatchInstruction(hp1, A_MOV, [S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  10913. begin
  10914. if CanBeCMOV(hp1, hp_prev) then
  10915. Inc(l)
  10916. else if not (cs_opt_size in current_settings.optimizerswitches) and
  10917. { CMOV with constants grows the code size }
  10918. TryCMOVConst(hp1, hp_prev, hp_stop, c, l) then
  10919. begin
  10920. { Register was reserved by TryCMOVConst and
  10921. stored on ConstRegs[c] }
  10922. end
  10923. else
  10924. Break;
  10925. end
  10926. else
  10927. Break;
  10928. else
  10929. ;
  10930. end;
  10931. GetNextInstruction(hp1,hp1);
  10932. end;
  10933. if (hp1 = hp_lblxxx) then
  10934. begin
  10935. if (l <= MAX_CMOV_INSTRUCTIONS) and (l > 0) then
  10936. begin
  10937. { Repurpose TmpUsedRegs to mark registers that we've defined }
  10938. TmpUsedRegs[R_INTREGISTER].Clear;
  10939. x := 0;
  10940. AllocRegBetween(NR_DEFAULTFLAGS, p, hp_lblxxx, UsedRegs);
  10941. condition := inverse_cond(taicpu(p).condition);
  10942. UpdateUsedRegs(tai(p.next));
  10943. hp1 := hpmov1;
  10944. repeat
  10945. if not Assigned(hp1) then
  10946. InternalError(2018062900);
  10947. if (hp1.typ = ait_instruction) then
  10948. begin
  10949. { Extra safeguard }
  10950. if (taicpu(hp1).opcode <> A_MOV) then
  10951. InternalError(2018062901);
  10952. if taicpu(hp1).oper[0]^.typ = top_const then
  10953. begin
  10954. if x >= MAX_CMOV_REGISTERS then
  10955. InternalError(2021100410);
  10956. { If it's in TmpUsedRegs, then this register
  10957. is being used more than once and hence has
  10958. already had its value defined (it gets
  10959. added to UsedRegs through AllocRegBetween
  10960. below) }
  10961. if not TmpUsedRegs[R_INTREGISTER].IsUsed(ConstRegs[x]) then
  10962. begin
  10963. hp_new := taicpu.op_const_reg(A_MOV, taicpu(hp1).opsize, taicpu(hp1).oper[0]^.val, ConstRegs[x]);
  10964. taicpu(hp_new).fileinfo := taicpu(hp_prev).fileinfo;
  10965. asml.InsertBefore(hp_new, hp_flagalloc);
  10966. if Assigned(hp_prev2) then
  10967. TrySwapMovOp(hp_prev2, hp_new);
  10968. IncludeRegInUsedRegs(ConstRegs[x], TmpUsedRegs);
  10969. end
  10970. else
  10971. { We just need an instruction between hp_prev and hp1
  10972. where we know the register is marked as in use }
  10973. hp_new := hpmov1;
  10974. AllocRegBetween(ConstRegs[x], hp_new, hp1, UsedRegs);
  10975. taicpu(hp1).loadreg(0, ConstRegs[x]);
  10976. Inc(x);
  10977. end;
  10978. taicpu(hp1).opcode := A_CMOVcc;
  10979. taicpu(hp1).condition := condition;
  10980. end;
  10981. UpdateUsedRegs(tai(hp1.next));
  10982. GetNextInstruction(hp1, hp1);
  10983. until (hp1 = hp_lblxxx);
  10984. hp2 := hp_lblxxx;
  10985. repeat
  10986. if not Assigned(hp2) then
  10987. InternalError(2018062910);
  10988. case hp2.typ of
  10989. ait_label:
  10990. { What we expected - break out of the loop (it won't be a dead label at the top of
  10991. a cluster because that was optimised at an earlier stage) }
  10992. Break;
  10993. ait_align:
  10994. { Go to the next entry until a label is found (may be multiple aligns before it) }
  10995. begin
  10996. hp2 := tai(hp2.Next);
  10997. Continue;
  10998. end;
  10999. ait_instruction:
  11000. begin
  11001. if taicpu(hp2).opcode<>A_JMP then
  11002. InternalError(2018062912);
  11003. { This is the Jcc @Lbl; <several movs>; JMP @Lbl variant }
  11004. Break;
  11005. end
  11006. else
  11007. begin
  11008. { Might be a comment or temporary allocation entry }
  11009. if not (hp2.typ in SkipInstr) then
  11010. InternalError(2018062911);
  11011. hp2 := tai(hp2.Next);
  11012. Continue;
  11013. end;
  11014. end;
  11015. until False;
  11016. { Now we can safely decrement the reference count }
  11017. tasmlabel(symbol).decrefs;
  11018. DebugMsg(SPeepholeOptimization+'JccMov2CMov',p);
  11019. { Remove the original jump }
  11020. RemoveInstruction(p); { Note, the choice to not use RemoveCurrentp is deliberate }
  11021. if hp2.typ=ait_instruction then
  11022. begin
  11023. p := hp2;
  11024. Result := True;
  11025. end
  11026. else
  11027. begin
  11028. UpdateUsedRegs(tai(hp2.next));
  11029. Result := GetNextInstruction(hp2, p); { Instruction after the label }
  11030. { Remove the label if this is its final reference }
  11031. if (tasmlabel(symbol).getrefs=0) then
  11032. begin
  11033. { Make sure the aligns get stripped too }
  11034. hp1 := tai(hp_lblxxx.Previous);
  11035. while Assigned(hp1) and (hp1.typ = ait_align) do
  11036. begin
  11037. hp_lblxxx := hp1;
  11038. hp1 := tai(hp_lblxxx.Previous);
  11039. end;
  11040. StripLabelFast(hp_lblxxx);
  11041. end;
  11042. end;
  11043. Exit;
  11044. end;
  11045. end
  11046. else if assigned(hp_lblxxx) and
  11047. { check further for
  11048. jCC xxx
  11049. <several movs 1>
  11050. jmp yyy
  11051. xxx:
  11052. <several movs 2>
  11053. yyy:
  11054. }
  11055. (l <= MAX_CMOV_INSTRUCTIONS - 1) and
  11056. { hp1 should be pointing to jmp yyy }
  11057. MatchInstruction(hp1, A_JMP, []) and
  11058. { real label and jump, no further references to the
  11059. label are allowed }
  11060. (TAsmLabel(symbol).getrefs=1) and
  11061. FindLabel(TAsmLabel(symbol), hp_lblxxx) then
  11062. begin
  11063. hp_jump := hp1;
  11064. { Don't set c to zero }
  11065. l := 0;
  11066. w := 0;
  11067. GetNextInstruction(hp_lblxxx, hpmov2);
  11068. hp2 := hp_lblxxx;
  11069. hp_lblyyy := hpmov2;
  11070. while assigned(hp_lblyyy) and
  11071. { stop on labels }
  11072. (hp_lblyyy.typ <> ait_label) do
  11073. begin
  11074. { Keep track of all integer registers that are used }
  11075. UpdateIntRegsNoDealloc(TmpUsedRegs, tai(hp2.Next));
  11076. if not MatchInstruction(hp_lblyyy, A_MOV, [S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  11077. Break;
  11078. hp2 := hp_lblyyy;
  11079. GetNextInstruction(hp_lblyyy, hp_lblyyy);
  11080. end;
  11081. { Analyse the second batch of MOVs to see if the setup is valid }
  11082. hp1 := hpmov2;
  11083. while assigned(hp1) and
  11084. (hp1 <> hp_lblyyy) do
  11085. begin
  11086. case hp1.typ of
  11087. ait_instruction:
  11088. if MatchInstruction(hp1, A_MOV, [S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  11089. begin
  11090. if CanBeCMOV(hp1, hp_prev) then
  11091. Inc(l)
  11092. else if not (cs_opt_size in current_settings.optimizerswitches)
  11093. { CMOV with constants grows the code size }
  11094. and TryCMOVConst(hp1, hpmov2, hp_lblyyy, c, l) then
  11095. begin
  11096. { Register was reserved by TryCMOVConst and
  11097. stored on ConstRegs[c] }
  11098. end
  11099. else
  11100. Break;
  11101. end
  11102. else
  11103. Break;
  11104. else
  11105. ;
  11106. end;
  11107. GetNextInstruction(hp1,hp1);
  11108. end;
  11109. { Repurpose TmpUsedRegs to mark registers that we've defined }
  11110. TmpUsedRegs[R_INTREGISTER].Clear;
  11111. if (l <= MAX_CMOV_INSTRUCTIONS - 1) and
  11112. (hp1 = hp_lblyyy) and
  11113. FindLabel(TAsmLabel(taicpu(hp_jump).oper[0]^.ref^.symbol), hp_lblyyy) then
  11114. begin
  11115. AllocRegBetween(NR_DEFAULTFLAGS, p, hp_lblyyy, UsedRegs);
  11116. second_condition := taicpu(p).condition;
  11117. condition := inverse_cond(taicpu(p).condition);
  11118. UpdateUsedRegs(tai(p.next));
  11119. { Scan through the first set of MOVs to update UsedRegs,
  11120. but don't process them yet }
  11121. hp1 := hpmov1;
  11122. repeat
  11123. if not Assigned(hp1) then
  11124. InternalError(2018062901);
  11125. UpdateUsedRegs(tai(hp1.next));
  11126. GetNextInstruction(hp1, hp1);
  11127. until (hp1 = hp_lblxxx);
  11128. UpdateUsedRegs(tai(hp_lblxxx.next));
  11129. { Process the second set of MOVs first,
  11130. because if a destination register is
  11131. shared between the first and second MOV
  11132. sets, it is more efficient to turn the
  11133. first one into a MOV instruction and place
  11134. it before the CMP if possible, but we
  11135. won't know which registers are shared
  11136. until we've processed at least one list,
  11137. so we might as well make it the second
  11138. one since that won't be modified again. }
  11139. hp1 := hpmov2;
  11140. repeat
  11141. if not Assigned(hp1) then
  11142. InternalError(2018062902);
  11143. if (hp1.typ = ait_instruction) then
  11144. begin
  11145. { Extra safeguard }
  11146. if (taicpu(hp1).opcode <> A_MOV) then
  11147. InternalError(2018062903);
  11148. if taicpu(hp1).oper[0]^.typ = top_const then
  11149. begin
  11150. RegMatch := False;
  11151. for x := 0 to c - 1 do
  11152. if (ConstVals[x] = taicpu(hp1).oper[0]^.val) then
  11153. begin
  11154. RegMatch := True;
  11155. { If it's in TmpUsedRegs, then this register
  11156. is being used more than once and hence has
  11157. already had its value defined (it gets
  11158. added to UsedRegs through AllocRegBetween
  11159. below) }
  11160. if not TmpUsedRegs[R_INTREGISTER].IsUsed(ConstRegs[x]) then
  11161. begin
  11162. hp_new := taicpu.op_const_reg(A_MOV, taicpu(hp1).opsize, taicpu(hp1).oper[0]^.val, ConstRegs[x]);
  11163. asml.InsertBefore(hp_new, hp_flagalloc);
  11164. if Assigned(hp_prev2) then
  11165. TrySwapMovOp(hp_prev2, hp_new);
  11166. IncludeRegInUsedRegs(ConstRegs[x], TmpUsedRegs);
  11167. end
  11168. else
  11169. { We just need an instruction between hp_prev and hp1
  11170. where we know the register is marked as in use }
  11171. hp_new := hpmov2;
  11172. AllocRegBetween(ConstRegs[x], hp_new, hp1, UsedRegs);
  11173. taicpu(hp1).loadreg(0, ConstRegs[x]);
  11174. Break;
  11175. end;
  11176. if not RegMatch then
  11177. InternalError(2021100411);
  11178. end;
  11179. taicpu(hp1).opcode := A_CMOVcc;
  11180. taicpu(hp1).condition := second_condition;
  11181. { Store these writes to search for
  11182. duplicates later on }
  11183. RegWrites[w] := taicpu(hp1).oper[1]^.reg;
  11184. Inc(w);
  11185. end;
  11186. UpdateUsedRegs(tai(hp1.next));
  11187. GetNextInstruction(hp1, hp1);
  11188. until (hp1 = hp_lblyyy);
  11189. { Now do the first set of MOVs }
  11190. hp1 := hpmov1;
  11191. repeat
  11192. if not Assigned(hp1) then
  11193. InternalError(2018062904);
  11194. if (hp1.typ = ait_instruction) then
  11195. begin
  11196. RegMatch := False;
  11197. { Extra safeguard }
  11198. if (taicpu(hp1).opcode <> A_MOV) then
  11199. InternalError(2018062905);
  11200. { Search through the RegWrites list to see
  11201. if there are any opposing CMOV pairs that
  11202. write to the same register }
  11203. for x := 0 to w - 1 do
  11204. if RegWrites[x] = taicpu(hp1).oper[1]^.reg then
  11205. begin
  11206. { We have a match. Move this instruction
  11207. right to the top }
  11208. hp2 := hp1;
  11209. { Move ahead in preparation }
  11210. GetNextInstruction(hp1, hp1);
  11211. asml.Remove(hp2);
  11212. asml.InsertAfter(hp2, hp_prev);
  11213. { Note we can't use the trick of inserting before hp_prev
  11214. and then calling TrySwapMovOp with hp_prev2, like with
  11215. the MOV imm,reg optimisations, because hp2 may share a
  11216. register with the comparison }
  11217. if (hp_prev <> p) then
  11218. TrySwapMovCmp(hp_prev, hp2);
  11219. RegMatch := True;
  11220. Break;
  11221. end;
  11222. if RegMatch then
  11223. Continue;
  11224. if taicpu(hp1).oper[0]^.typ = top_const then
  11225. begin
  11226. RegMatch := False;
  11227. for x := 0 to c - 1 do
  11228. if (ConstVals[x] = taicpu(hp1).oper[0]^.val) then
  11229. begin
  11230. RegMatch := True;
  11231. { If it's in TmpUsedRegs, then this register
  11232. is being used more than once and hence has
  11233. already had its value defined (it gets
  11234. added to UsedRegs through AllocRegBetween
  11235. below) }
  11236. if not TmpUsedRegs[R_INTREGISTER].IsUsed(ConstRegs[x]) then
  11237. begin
  11238. hp_new := taicpu.op_const_reg(A_MOV, taicpu(hp1).opsize, taicpu(hp1).oper[0]^.val, ConstRegs[x]);
  11239. asml.InsertBefore(hp_new, hp_flagalloc);
  11240. if Assigned(hp_prev2) then
  11241. TrySwapMovOp(hp_prev2, hp_new);
  11242. IncludeRegInUsedRegs(ConstRegs[x], TmpUsedRegs);
  11243. end
  11244. else
  11245. { We just need an instruction between hp_prev and hp1
  11246. where we know the register is marked as in use }
  11247. hp_new := hpmov1;
  11248. AllocRegBetween(ConstRegs[x], hp_new, hp1, UsedRegs);
  11249. taicpu(hp1).loadreg(0, ConstRegs[x]);
  11250. Break;
  11251. end;
  11252. if not RegMatch then
  11253. InternalError(2021100412);
  11254. end;
  11255. taicpu(hp1).opcode := A_CMOVcc;
  11256. taicpu(hp1).condition := condition;
  11257. end;
  11258. GetNextInstruction(hp1, hp1);
  11259. until (hp1 = hp_jump); { Stop at the jump, not lbl xxx }
  11260. UpdateUsedRegs(tai(hp_jump.next));
  11261. UpdateUsedRegs(tai(hp_lblyyy.next));
  11262. { Get first instruction after label }
  11263. hp1 := p;
  11264. GetNextInstruction(hp_lblyyy, p);
  11265. { Don't dereference yet, as doing so will cause
  11266. GetNextInstruction to skip the label and
  11267. optional align marker. [Kit] }
  11268. DebugMsg(SPeepholeOptimization+'JccMovJmpMov2CMovCMov',hp1);
  11269. { remove Jcc }
  11270. RemoveInstruction(hp1);
  11271. { Now we can safely decrement it }
  11272. tasmlabel(symbol).decrefs;
  11273. { Remove label xxx (it will have a ref of zero due to the initial check) }
  11274. { Make sure the aligns get stripped too }
  11275. hp1 := tai(hp_lblxxx.Previous);
  11276. while Assigned(hp1) and (hp1.typ = ait_align) do
  11277. begin
  11278. hp_lblxxx := hp1;
  11279. hp1 := tai(hp_lblxxx.Previous);
  11280. end;
  11281. StripLabelFast(hp_lblxxx);
  11282. { remove jmp }
  11283. symbol := taicpu(hp_jump).oper[0]^.ref^.symbol;
  11284. RemoveInstruction(hp_jump);
  11285. { As before, now we can safely decrement it }
  11286. TAsmLabel(symbol).decrefs;
  11287. { Remove label yyy (and the optional alignment) if its reference falls to zero }
  11288. if TAsmLabel(symbol).getrefs = 0 then
  11289. begin
  11290. { Make sure the aligns get stripped too }
  11291. hp1 := tai(hp_lblyyy.Previous);
  11292. while Assigned(hp1) and (hp1.typ = ait_align) do
  11293. begin
  11294. hp_lblyyy := hp1;
  11295. hp1 := tai(hp_lblyyy.Previous);
  11296. end;
  11297. StripLabelFast(hp_lblyyy);
  11298. end;
  11299. if Assigned(p) then
  11300. result := True;
  11301. exit;
  11302. end;
  11303. end;
  11304. end;
  11305. {$endif i8086}
  11306. end;
  11307. end;
  11308. end;
  11309. function TX86AsmOptimizer.OptPass1Movx(var p : tai) : boolean;
  11310. var
  11311. hp1,hp2,hp3: tai;
  11312. reg_and_hp1_is_instr, RegUsed, AndTest: Boolean;
  11313. NewSize: TOpSize;
  11314. NewRegSize: TSubRegister;
  11315. Limit: TCgInt;
  11316. SwapOper: POper;
  11317. begin
  11318. result:=false;
  11319. reg_and_hp1_is_instr:=(taicpu(p).oper[1]^.typ = top_reg) and
  11320. GetNextInstruction(p,hp1) and
  11321. (hp1.typ = ait_instruction);
  11322. if reg_and_hp1_is_instr and
  11323. (
  11324. (taicpu(hp1).opcode <> A_LEA) or
  11325. { If the LEA instruction can be converted into an arithmetic instruction,
  11326. it may be possible to then fold it. }
  11327. (
  11328. { If the flags register is in use, don't change the instruction
  11329. to an ADD otherwise this will scramble the flags. [Kit] }
  11330. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  11331. ConvertLEA(taicpu(hp1))
  11332. )
  11333. ) and
  11334. IsFoldableArithOp(taicpu(hp1),taicpu(p).oper[1]^.reg) and
  11335. GetNextInstruction(hp1,hp2) and
  11336. MatchInstruction(hp2,A_MOV,[]) and
  11337. (taicpu(hp2).oper[0]^.typ = top_reg) and
  11338. OpsEqual(taicpu(hp2).oper[1]^,taicpu(p).oper[0]^) and
  11339. ((taicpu(p).opsize in [S_BW,S_BL]) and (taicpu(hp2).opsize=S_B) or
  11340. (taicpu(p).opsize in [S_WL]) and (taicpu(hp2).opsize=S_W)) and
  11341. {$ifdef i386}
  11342. { not all registers have byte size sub registers on i386 }
  11343. ((taicpu(hp2).opsize<>S_B) or (getsupreg(taicpu(hp1).oper[0]^.reg) in [RS_EAX, RS_EBX, RS_ECX, RS_EDX])) and
  11344. {$endif i386}
  11345. (((taicpu(hp1).ops=2) and
  11346. (getsupreg(taicpu(hp2).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg))) or
  11347. ((taicpu(hp1).ops=1) and
  11348. (getsupreg(taicpu(hp2).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[0]^.reg)))) and
  11349. not(RegUsedAfterInstruction(taicpu(hp2).oper[0]^.reg,hp2,UsedRegs)) then
  11350. begin
  11351. { change movsX/movzX reg/ref, reg2
  11352. add/sub/or/... reg3/$const, reg2
  11353. mov reg2 reg/ref
  11354. to add/sub/or/... reg3/$const, reg/ref }
  11355. { by example:
  11356. movswl %si,%eax movswl %si,%eax p
  11357. decl %eax addl %edx,%eax hp1
  11358. movw %ax,%si movw %ax,%si hp2
  11359. ->
  11360. movswl %si,%eax movswl %si,%eax p
  11361. decw %eax addw %edx,%eax hp1
  11362. movw %ax,%si movw %ax,%si hp2
  11363. }
  11364. taicpu(hp1).changeopsize(taicpu(hp2).opsize);
  11365. {
  11366. ->
  11367. movswl %si,%eax movswl %si,%eax p
  11368. decw %si addw %dx,%si hp1
  11369. movw %ax,%si movw %ax,%si hp2
  11370. }
  11371. case taicpu(hp1).ops of
  11372. 1:
  11373. taicpu(hp1).loadoper(0,taicpu(hp2).oper[1]^);
  11374. 2:
  11375. begin
  11376. taicpu(hp1).loadoper(1,taicpu(hp2).oper[1]^);
  11377. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  11378. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  11379. end;
  11380. else
  11381. internalerror(2008042702);
  11382. end;
  11383. {
  11384. ->
  11385. decw %si addw %dx,%si p
  11386. }
  11387. DebugMsg(SPeepholeOptimization + 'var3',p);
  11388. RemoveCurrentP(p, hp1);
  11389. RemoveInstruction(hp2);
  11390. Result := True;
  11391. Exit;
  11392. end;
  11393. if reg_and_hp1_is_instr and
  11394. (taicpu(hp1).opcode = A_MOV) and
  11395. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  11396. (MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^)
  11397. {$ifdef x86_64}
  11398. { check for implicit extension to 64 bit }
  11399. or
  11400. ((taicpu(p).opsize in [S_BL,S_WL]) and
  11401. (taicpu(hp1).opsize=S_Q) and
  11402. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.reg)
  11403. )
  11404. {$endif x86_64}
  11405. )
  11406. then
  11407. begin
  11408. { change
  11409. movx %reg1,%reg2
  11410. mov %reg2,%reg3
  11411. dealloc %reg2
  11412. into
  11413. movx %reg,%reg3
  11414. }
  11415. TransferUsedRegs(TmpUsedRegs);
  11416. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  11417. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  11418. begin
  11419. DebugMsg(SPeepholeOptimization + 'MovxMov2Movx',p);
  11420. {$ifdef x86_64}
  11421. if (taicpu(p).opsize in [S_BL,S_WL]) and
  11422. (taicpu(hp1).opsize=S_Q) then
  11423. taicpu(p).loadreg(1,newreg(R_INTREGISTER,getsupreg(taicpu(hp1).oper[1]^.reg),R_SUBD))
  11424. else
  11425. {$endif x86_64}
  11426. taicpu(p).loadreg(1,taicpu(hp1).oper[1]^.reg);
  11427. RemoveInstruction(hp1);
  11428. Result := True;
  11429. Exit;
  11430. end;
  11431. end;
  11432. if reg_and_hp1_is_instr and
  11433. ((taicpu(hp1).opcode=A_MOV) or
  11434. (taicpu(hp1).opcode=A_ADD) or
  11435. (taicpu(hp1).opcode=A_SUB) or
  11436. (taicpu(hp1).opcode=A_CMP) or
  11437. (taicpu(hp1).opcode=A_OR) or
  11438. (taicpu(hp1).opcode=A_XOR) or
  11439. (taicpu(hp1).opcode=A_AND)
  11440. ) and
  11441. (taicpu(hp1).oper[1]^.typ = top_reg) then
  11442. begin
  11443. AndTest := (taicpu(hp1).opcode=A_AND) and
  11444. GetNextInstruction(hp1, hp2) and
  11445. (hp2.typ = ait_instruction) and
  11446. (
  11447. (
  11448. (taicpu(hp2).opcode=A_TEST) and
  11449. (
  11450. MatchOperand(taicpu(hp2).oper[0]^, taicpu(hp1).oper[1]^.reg) or
  11451. MatchOperand(taicpu(hp2).oper[0]^, -1) or
  11452. (
  11453. { If the AND and TEST instructions share a constant, this is also valid }
  11454. (taicpu(hp1).oper[0]^.typ = top_const) and
  11455. MatchOperand(taicpu(hp2).oper[0]^, taicpu(hp1).oper[0]^.val)
  11456. )
  11457. ) and
  11458. MatchOperand(taicpu(hp2).oper[1]^, taicpu(hp1).oper[1]^.reg)
  11459. ) or
  11460. (
  11461. (taicpu(hp2).opcode=A_CMP) and
  11462. MatchOperand(taicpu(hp2).oper[0]^, 0) and
  11463. MatchOperand(taicpu(hp2).oper[1]^, taicpu(hp1).oper[1]^.reg)
  11464. )
  11465. );
  11466. { change
  11467. movx (oper),%reg2
  11468. and $x,%reg2
  11469. test %reg2,%reg2
  11470. dealloc %reg2
  11471. into
  11472. op %reg1,%reg3
  11473. if the second op accesses only the bits stored in reg1
  11474. }
  11475. if ((taicpu(p).oper[0]^.typ=top_reg) or
  11476. ((taicpu(p).oper[0]^.typ=top_ref) and (taicpu(p).oper[0]^.ref^.refaddr<>addr_full))) and
  11477. (taicpu(hp1).oper[0]^.typ = top_const) and
  11478. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  11479. AndTest then
  11480. begin
  11481. { Check if the AND constant is in range }
  11482. case taicpu(p).opsize of
  11483. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  11484. begin
  11485. NewSize := S_B;
  11486. Limit := $FF;
  11487. end;
  11488. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  11489. begin
  11490. NewSize := S_W;
  11491. Limit := $FFFF;
  11492. end;
  11493. {$ifdef x86_64}
  11494. S_LQ:
  11495. begin
  11496. NewSize := S_L;
  11497. Limit := $FFFFFFFF;
  11498. end;
  11499. {$endif x86_64}
  11500. else
  11501. InternalError(2021120303);
  11502. end;
  11503. if (
  11504. ((taicpu(hp1).oper[0]^.val and Limit) = taicpu(hp1).oper[0]^.val) or
  11505. { Check for negative operands }
  11506. (((not taicpu(hp1).oper[0]^.val) and Limit) = (not taicpu(hp1).oper[0]^.val))
  11507. ) and
  11508. GetNextInstruction(hp2,hp3) and
  11509. MatchInstruction(hp3,A_Jcc,A_Setcc,A_CMOVcc,[]) and
  11510. (taicpu(hp3).condition in [C_E,C_NE]) then
  11511. begin
  11512. TransferUsedRegs(TmpUsedRegs);
  11513. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  11514. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  11515. if not(RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp2, TmpUsedRegs)) then
  11516. begin
  11517. DebugMsg(SPeepholeOptimization + 'MovxAndTest2Test done',p);
  11518. taicpu(hp1).loadoper(1, taicpu(p).oper[0]^);
  11519. taicpu(hp1).opcode := A_TEST;
  11520. taicpu(hp1).opsize := NewSize;
  11521. RemoveInstruction(hp2);
  11522. RemoveCurrentP(p, hp1);
  11523. Result:=true;
  11524. exit;
  11525. end;
  11526. end;
  11527. end;
  11528. if (taicpu(hp1).oper[0]^.typ = top_reg) and
  11529. (((taicpu(p).opsize in [S_BW,S_BL,S_WL{$ifdef x86_64},S_BQ,S_WQ,S_LQ{$endif x86_64}]) and
  11530. (taicpu(hp1).opsize=S_B)) or
  11531. ((taicpu(p).opsize in [S_WL{$ifdef x86_64},S_WQ,S_LQ{$endif x86_64}]) and
  11532. (taicpu(hp1).opsize=S_W))
  11533. {$ifdef x86_64}
  11534. or ((taicpu(p).opsize=S_LQ) and
  11535. (taicpu(hp1).opsize=S_L))
  11536. {$endif x86_64}
  11537. ) and
  11538. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.reg) then
  11539. begin
  11540. { change
  11541. movx %reg1,%reg2
  11542. op %reg2,%reg3
  11543. dealloc %reg2
  11544. into
  11545. op %reg1,%reg3
  11546. if the second op accesses only the bits stored in reg1
  11547. }
  11548. TransferUsedRegs(TmpUsedRegs);
  11549. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  11550. if AndTest then
  11551. begin
  11552. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  11553. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs);
  11554. end
  11555. else
  11556. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs);
  11557. if not RegUsed then
  11558. begin
  11559. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 1',p);
  11560. if taicpu(p).oper[0]^.typ=top_reg then
  11561. begin
  11562. case taicpu(hp1).opsize of
  11563. S_B:
  11564. taicpu(hp1).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[0]^.reg),R_SUBL));
  11565. S_W:
  11566. taicpu(hp1).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[0]^.reg),R_SUBW));
  11567. S_L:
  11568. taicpu(hp1).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[0]^.reg),R_SUBD));
  11569. else
  11570. Internalerror(2020102301);
  11571. end;
  11572. AllocRegBetween(taicpu(hp1).oper[0]^.reg,p,hp1,UsedRegs);
  11573. end
  11574. else
  11575. taicpu(hp1).loadref(0,taicpu(p).oper[0]^.ref^);
  11576. RemoveCurrentP(p);
  11577. if AndTest then
  11578. RemoveInstruction(hp2);
  11579. result:=true;
  11580. exit;
  11581. end;
  11582. end
  11583. else if (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg) and
  11584. (
  11585. { Bitwise operations only }
  11586. (taicpu(hp1).opcode=A_AND) or
  11587. (taicpu(hp1).opcode=A_TEST) or
  11588. (
  11589. (taicpu(hp1).oper[0]^.typ = top_const) and
  11590. (
  11591. (taicpu(hp1).opcode=A_OR) or
  11592. (taicpu(hp1).opcode=A_XOR)
  11593. )
  11594. )
  11595. ) and
  11596. (
  11597. (taicpu(hp1).oper[0]^.typ = top_const) or
  11598. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) or
  11599. not RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^)
  11600. ) then
  11601. begin
  11602. { change
  11603. movx %reg2,%reg2
  11604. op const,%reg2
  11605. into
  11606. op const,%reg2 (smaller version)
  11607. movx %reg2,%reg2
  11608. also change
  11609. movx %reg1,%reg2
  11610. and/test (oper),%reg2
  11611. dealloc %reg2
  11612. into
  11613. and/test (oper),%reg1
  11614. }
  11615. case taicpu(p).opsize of
  11616. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  11617. begin
  11618. NewSize := S_B;
  11619. NewRegSize := R_SUBL;
  11620. Limit := $FF;
  11621. end;
  11622. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  11623. begin
  11624. NewSize := S_W;
  11625. NewRegSize := R_SUBW;
  11626. Limit := $FFFF;
  11627. end;
  11628. {$ifdef x86_64}
  11629. S_LQ:
  11630. begin
  11631. NewSize := S_L;
  11632. NewRegSize := R_SUBD;
  11633. Limit := $FFFFFFFF;
  11634. end;
  11635. {$endif x86_64}
  11636. else
  11637. Internalerror(2021120302);
  11638. end;
  11639. TransferUsedRegs(TmpUsedRegs);
  11640. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  11641. if AndTest then
  11642. begin
  11643. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  11644. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs);
  11645. end
  11646. else
  11647. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs);
  11648. if
  11649. (
  11650. (taicpu(p).opcode = A_MOVZX) and
  11651. (
  11652. (taicpu(hp1).opcode=A_AND) or
  11653. (taicpu(hp1).opcode=A_TEST)
  11654. ) and
  11655. not (
  11656. { If both are references, then the final instruction will have
  11657. both operands as references, which is not allowed }
  11658. (taicpu(p).oper[0]^.typ = top_ref) and
  11659. (taicpu(hp1).oper[0]^.typ = top_ref)
  11660. ) and
  11661. not RegUsed
  11662. ) or
  11663. (
  11664. (
  11665. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) or
  11666. not RegUsed
  11667. ) and
  11668. (taicpu(p).oper[0]^.typ = top_reg) and
  11669. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  11670. (taicpu(hp1).oper[0]^.typ = top_const) and
  11671. ((taicpu(hp1).oper[0]^.val and Limit) = taicpu(hp1).oper[0]^.val)
  11672. ) then
  11673. begin
  11674. {$if defined(i386) or defined(i8086)}
  11675. { If the target size is 8-bit, make sure we can actually encode it }
  11676. if (NewRegSize = R_SUBL) and (taicpu(hp1).oper[0]^.typ = top_reg) and not (GetSupReg(taicpu(hp1).oper[0]^.reg) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX]) then
  11677. Exit;
  11678. {$endif i386 or i8086}
  11679. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 2',p);
  11680. taicpu(hp1).opsize := NewSize;
  11681. taicpu(hp1).loadoper(1, taicpu(p).oper[0]^);
  11682. if AndTest then
  11683. begin
  11684. RemoveInstruction(hp2);
  11685. if not RegUsed then
  11686. begin
  11687. taicpu(hp1).opcode := A_TEST;
  11688. if (taicpu(hp1).oper[0]^.typ = top_ref) then
  11689. begin
  11690. { Make sure the reference is the second operand }
  11691. SwapOper := taicpu(hp1).oper[0];
  11692. taicpu(hp1).oper[0] := taicpu(hp1).oper[1];
  11693. taicpu(hp1).oper[1] := SwapOper;
  11694. end;
  11695. end;
  11696. end;
  11697. case taicpu(hp1).oper[0]^.typ of
  11698. top_reg:
  11699. setsubreg(taicpu(hp1).oper[0]^.reg, NewRegSize);
  11700. top_const:
  11701. { For the AND/TEST case }
  11702. taicpu(hp1).oper[0]^.val := taicpu(hp1).oper[0]^.val and Limit;
  11703. else
  11704. ;
  11705. end;
  11706. if RegUsed then
  11707. begin
  11708. AsmL.Remove(p);
  11709. AsmL.InsertAfter(p, hp1);
  11710. p := hp1;
  11711. end
  11712. else
  11713. RemoveCurrentP(p, hp1);
  11714. result:=true;
  11715. exit;
  11716. end;
  11717. end;
  11718. end;
  11719. if reg_and_hp1_is_instr and
  11720. (taicpu(p).oper[0]^.typ = top_reg) and
  11721. (
  11722. (taicpu(hp1).opcode = A_SHL) or (taicpu(hp1).opcode = A_SAL)
  11723. ) and
  11724. (taicpu(hp1).oper[0]^.typ = top_const) and
  11725. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  11726. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  11727. { Minimum shift value allowed is the bit difference between the sizes }
  11728. (taicpu(hp1).oper[0]^.val >=
  11729. { Multiply by 8 because tcgsize2size returns bytes, not bits }
  11730. 8 * (
  11731. tcgsize2size[reg_cgsize(taicpu(p).oper[1]^.reg)] -
  11732. tcgsize2size[reg_cgsize(taicpu(p).oper[0]^.reg)]
  11733. )
  11734. ) then
  11735. begin
  11736. { For:
  11737. movsx/movzx %reg1,%reg1 (same register, just different sizes)
  11738. shl/sal ##, %reg1
  11739. Remove the movsx/movzx instruction if the shift overwrites the
  11740. extended bits of the register (e.g. movslq %eax,%rax; shlq $32,%rax
  11741. }
  11742. DebugMsg(SPeepholeOptimization + 'MovxShl2Shl',p);
  11743. RemoveCurrentP(p, hp1);
  11744. Result := True;
  11745. Exit;
  11746. end
  11747. else if reg_and_hp1_is_instr and
  11748. (taicpu(p).oper[0]^.typ = top_reg) and
  11749. (
  11750. ((taicpu(hp1).opcode = A_SHR) and (taicpu(p).opcode = A_MOVZX)) or
  11751. ((taicpu(hp1).opcode = A_SAR) and (taicpu(p).opcode <> A_MOVZX))
  11752. ) and
  11753. (taicpu(hp1).oper[0]^.typ = top_const) and
  11754. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  11755. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  11756. { Minimum shift value allowed is the bit size of the smallest register - 1 }
  11757. (taicpu(hp1).oper[0]^.val <
  11758. { Multiply by 8 because tcgsize2size returns bytes, not bits }
  11759. 8 * (
  11760. tcgsize2size[reg_cgsize(taicpu(p).oper[0]^.reg)]
  11761. )
  11762. ) then
  11763. begin
  11764. { For:
  11765. movsx %reg1,%reg1 movzx %reg1,%reg1 (same register, just different sizes)
  11766. sar ##, %reg1 shr ##, %reg1
  11767. Move the shift to before the movx instruction if the shift value
  11768. is not too large.
  11769. }
  11770. asml.Remove(hp1);
  11771. asml.InsertBefore(hp1, p);
  11772. taicpu(hp1).oper[1]^.reg := taicpu(p).oper[0]^.reg;
  11773. case taicpu(p).opsize of
  11774. s_BW, S_BL{$ifdef x86_64}, S_BQ{$endif}:
  11775. taicpu(hp1).opsize := S_B;
  11776. S_WL{$ifdef x86_64}, S_WQ{$endif}:
  11777. taicpu(hp1).opsize := S_W;
  11778. {$ifdef x86_64}
  11779. S_LQ:
  11780. taicpu(hp1).opsize := S_L;
  11781. {$endif}
  11782. else
  11783. InternalError(2020112401);
  11784. end;
  11785. if (taicpu(hp1).opcode = A_SHR) then
  11786. DebugMsg(SPeepholeOptimization + 'MovzShr2ShrMovz', hp1)
  11787. else
  11788. DebugMsg(SPeepholeOptimization + 'MovsSar2SarMovs', hp1);
  11789. Result := True;
  11790. end;
  11791. if reg_and_hp1_is_instr and
  11792. (taicpu(p).oper[0]^.typ = top_reg) and
  11793. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  11794. (
  11795. (taicpu(hp1).opcode = taicpu(p).opcode)
  11796. or ((taicpu(p).opcode = A_MOVZX) and ((taicpu(hp1).opcode = A_MOVSX){$ifdef x86_64} or (taicpu(hp1).opcode = A_MOVSXD){$endif x86_64}))
  11797. {$ifdef x86_64}
  11798. or ((taicpu(p).opcode = A_MOVSX) and (taicpu(hp1).opcode = A_MOVSXD))
  11799. {$endif x86_64}
  11800. ) then
  11801. begin
  11802. if MatchOpType(taicpu(hp1), top_reg, top_reg) and
  11803. (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[0]^.reg) and
  11804. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg) then
  11805. begin
  11806. {
  11807. For example:
  11808. movzbw %al,%ax
  11809. movzwl %ax,%eax
  11810. Compress into:
  11811. movzbl %al,%eax
  11812. }
  11813. RegUsed := False;
  11814. case taicpu(p).opsize of
  11815. S_BW:
  11816. case taicpu(hp1).opsize of
  11817. S_WL:
  11818. begin
  11819. taicpu(p).opsize := S_BL;
  11820. RegUsed := True;
  11821. end;
  11822. {$ifdef x86_64}
  11823. S_WQ:
  11824. begin
  11825. if taicpu(p).opcode = A_MOVZX then
  11826. begin
  11827. taicpu(p).opsize := S_BL;
  11828. { 64-bit zero extension is implicit, so change to the 32-bit register }
  11829. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  11830. end
  11831. else
  11832. taicpu(p).opsize := S_BQ;
  11833. RegUsed := True;
  11834. end;
  11835. {$endif x86_64}
  11836. else
  11837. ;
  11838. end;
  11839. {$ifdef x86_64}
  11840. S_BL:
  11841. case taicpu(hp1).opsize of
  11842. S_LQ:
  11843. begin
  11844. if taicpu(p).opcode = A_MOVZX then
  11845. begin
  11846. taicpu(p).opsize := S_BL;
  11847. { 64-bit zero extension is implicit, so change to the 32-bit register }
  11848. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  11849. end
  11850. else
  11851. taicpu(p).opsize := S_BQ;
  11852. RegUsed := True;
  11853. end;
  11854. else
  11855. ;
  11856. end;
  11857. S_WL:
  11858. case taicpu(hp1).opsize of
  11859. S_LQ:
  11860. begin
  11861. if taicpu(p).opcode = A_MOVZX then
  11862. begin
  11863. taicpu(p).opsize := S_WL;
  11864. { 64-bit zero extension is implicit, so change to the 32-bit register }
  11865. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  11866. end
  11867. else
  11868. taicpu(p).opsize := S_WQ;
  11869. RegUsed := True;
  11870. end;
  11871. else
  11872. ;
  11873. end;
  11874. {$endif x86_64}
  11875. else
  11876. ;
  11877. end;
  11878. if RegUsed then
  11879. begin
  11880. DebugMsg(SPeepholeOptimization + 'MovxMovx2Movx', p);
  11881. taicpu(p).oper[1]^.reg := taicpu(hp1).oper[1]^.reg;
  11882. RemoveInstruction(hp1);
  11883. Result := True;
  11884. Exit;
  11885. end;
  11886. end;
  11887. if (taicpu(hp1).opsize = taicpu(p).opsize) and
  11888. not RegInInstruction(taicpu(p).oper[1]^.reg, hp1) and
  11889. GetNextInstruction(hp1, hp2) and
  11890. MatchInstruction(hp2, [A_AND, A_OR, A_XOR, A_TEST], []) and
  11891. (
  11892. ((taicpu(hp2).opsize = S_W) and (taicpu(p).opsize = S_BW)) or
  11893. ((taicpu(hp2).opsize = S_L) and (taicpu(p).opsize in [S_BL, S_WL]))
  11894. {$ifdef x86_64}
  11895. or ((taicpu(hp2).opsize = S_Q) and (taicpu(p).opsize in [S_BL, S_BQ, S_WL, S_WQ, S_LQ]))
  11896. {$endif x86_64}
  11897. ) and
  11898. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  11899. (
  11900. (
  11901. (taicpu(hp2).oper[0]^.reg = taicpu(hp1).oper[1]^.reg) and
  11902. (taicpu(hp2).oper[1]^.reg = taicpu(p).oper[1]^.reg)
  11903. ) or
  11904. (
  11905. { Only allow the operands in reverse order for TEST instructions }
  11906. (taicpu(hp2).opcode = A_TEST) and
  11907. (taicpu(hp2).oper[0]^.reg = taicpu(p).oper[1]^.reg) and
  11908. (taicpu(hp2).oper[1]^.reg = taicpu(hp1).oper[1]^.reg)
  11909. )
  11910. ) then
  11911. begin
  11912. {
  11913. For example:
  11914. movzbl %al,%eax
  11915. movzbl (ref),%edx
  11916. andl %edx,%eax
  11917. (%edx deallocated)
  11918. Change to:
  11919. andb (ref),%al
  11920. movzbl %al,%eax
  11921. Rules are:
  11922. - First two instructions have the same opcode and opsize
  11923. - First instruction's operands are the same super-register
  11924. - Second instruction operates on a different register
  11925. - Third instruction is AND, OR, XOR or TEST
  11926. - Third instruction's operands are the destination registers of the first two instructions
  11927. - Third instruction writes to the destination register of the first instruction (except with TEST)
  11928. - Second instruction's destination register is deallocated afterwards
  11929. }
  11930. TransferUsedRegs(TmpUsedRegs);
  11931. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  11932. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  11933. if not RegUsedAfterInstruction(taicpu(hp1).oper[1]^.reg, hp2, TmpUsedRegs) then
  11934. begin
  11935. case taicpu(p).opsize of
  11936. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  11937. NewSize := S_B;
  11938. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  11939. NewSize := S_W;
  11940. {$ifdef x86_64}
  11941. S_LQ:
  11942. NewSize := S_L;
  11943. {$endif x86_64}
  11944. else
  11945. InternalError(2021120301);
  11946. end;
  11947. taicpu(hp2).loadoper(0, taicpu(hp1).oper[0]^);
  11948. taicpu(hp2).loadreg(1, taicpu(p).oper[0]^.reg);
  11949. taicpu(hp2).opsize := NewSize;
  11950. RemoveInstruction(hp1);
  11951. { With TEST, it's best to keep the MOVX instruction at the top }
  11952. if (taicpu(hp2).opcode <> A_TEST) then
  11953. begin
  11954. DebugMsg(SPeepholeOptimization + 'MovxMovxTest2MovxTest', p);
  11955. asml.Remove(p);
  11956. { If the third instruction uses the flags, the MOVX instruction won't modify then }
  11957. asml.InsertAfter(p, hp2);
  11958. p := hp2;
  11959. end
  11960. else
  11961. DebugMsg(SPeepholeOptimization + 'MovxMovxOp2OpMovx', p);
  11962. Result := True;
  11963. Exit;
  11964. end;
  11965. end;
  11966. end;
  11967. if taicpu(p).opcode=A_MOVZX then
  11968. begin
  11969. { removes superfluous And's after movzx's }
  11970. if reg_and_hp1_is_instr and
  11971. (taicpu(hp1).opcode = A_AND) and
  11972. MatchOpType(taicpu(hp1),top_const,top_reg) and
  11973. ((taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg)
  11974. {$ifdef x86_64}
  11975. { check for implicit extension to 64 bit }
  11976. or
  11977. ((taicpu(p).opsize in [S_BL,S_WL]) and
  11978. (taicpu(hp1).opsize=S_Q) and
  11979. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^.reg)
  11980. )
  11981. {$endif x86_64}
  11982. )
  11983. then
  11984. begin
  11985. case taicpu(p).opsize Of
  11986. S_BL, S_BW{$ifdef x86_64}, S_BQ{$endif x86_64}:
  11987. if (taicpu(hp1).oper[0]^.val = $ff) then
  11988. begin
  11989. DebugMsg(SPeepholeOptimization + 'MovzAnd2Movz1',p);
  11990. RemoveInstruction(hp1);
  11991. Result:=true;
  11992. exit;
  11993. end;
  11994. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  11995. if (taicpu(hp1).oper[0]^.val = $ffff) then
  11996. begin
  11997. DebugMsg(SPeepholeOptimization + 'MovzAnd2Movz2',p);
  11998. RemoveInstruction(hp1);
  11999. Result:=true;
  12000. exit;
  12001. end;
  12002. {$ifdef x86_64}
  12003. S_LQ:
  12004. if (taicpu(hp1).oper[0]^.val = $ffffffff) then
  12005. begin
  12006. DebugMsg(SPeepholeOptimization + 'MovzAnd2Movz3',p);
  12007. RemoveInstruction(hp1);
  12008. Result:=true;
  12009. exit;
  12010. end;
  12011. {$endif x86_64}
  12012. else
  12013. ;
  12014. end;
  12015. { we cannot get rid of the and, but can we get rid of the movz ?}
  12016. if SuperRegistersEqual(taicpu(p).oper[0]^.reg,taicpu(p).oper[1]^.reg) then
  12017. begin
  12018. case taicpu(p).opsize Of
  12019. S_BL, S_BW{$ifdef x86_64}, S_BQ{$endif x86_64}:
  12020. if (taicpu(hp1).oper[0]^.val and $ff)=taicpu(hp1).oper[0]^.val then
  12021. begin
  12022. DebugMsg(SPeepholeOptimization + 'MovzAnd2And1',p);
  12023. RemoveCurrentP(p,hp1);
  12024. Result:=true;
  12025. exit;
  12026. end;
  12027. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  12028. if (taicpu(hp1).oper[0]^.val and $ffff)=taicpu(hp1).oper[0]^.val then
  12029. begin
  12030. DebugMsg(SPeepholeOptimization + 'MovzAnd2And2',p);
  12031. RemoveCurrentP(p,hp1);
  12032. Result:=true;
  12033. exit;
  12034. end;
  12035. {$ifdef x86_64}
  12036. S_LQ:
  12037. if (taicpu(hp1).oper[0]^.val and $ffffffff)=taicpu(hp1).oper[0]^.val then
  12038. begin
  12039. DebugMsg(SPeepholeOptimization + 'MovzAnd2And3',p);
  12040. RemoveCurrentP(p,hp1);
  12041. Result:=true;
  12042. exit;
  12043. end;
  12044. {$endif x86_64}
  12045. else
  12046. ;
  12047. end;
  12048. end;
  12049. end;
  12050. { changes some movzx constructs to faster synonyms (all examples
  12051. are given with eax/ax, but are also valid for other registers)}
  12052. if MatchOpType(taicpu(p),top_reg,top_reg) then
  12053. begin
  12054. case taicpu(p).opsize of
  12055. { Technically, movzbw %al,%ax cannot be encoded in 32/64-bit mode
  12056. (the machine code is equivalent to movzbl %al,%eax), but the
  12057. code generator still generates that assembler instruction and
  12058. it is silently converted. This should probably be checked.
  12059. [Kit] }
  12060. S_BW:
  12061. begin
  12062. if (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) and
  12063. (
  12064. not IsMOVZXAcceptable
  12065. { and $0xff,%ax has a smaller encoding but risks a partial write penalty }
  12066. or (
  12067. (cs_opt_size in current_settings.optimizerswitches) and
  12068. (taicpu(p).oper[1]^.reg = NR_AX)
  12069. )
  12070. ) then
  12071. {Change "movzbw %al, %ax" to "andw $0x0ffh, %ax"}
  12072. begin
  12073. DebugMsg(SPeepholeOptimization + 'var7',p);
  12074. taicpu(p).opcode := A_AND;
  12075. taicpu(p).changeopsize(S_W);
  12076. taicpu(p).loadConst(0,$ff);
  12077. Result := True;
  12078. end
  12079. else if not IsMOVZXAcceptable and
  12080. GetNextInstruction(p, hp1) and
  12081. (tai(hp1).typ = ait_instruction) and
  12082. (taicpu(hp1).opcode = A_AND) and
  12083. MatchOpType(taicpu(hp1),top_const,top_reg) and
  12084. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  12085. { Change "movzbw %reg1, %reg2; andw $const, %reg2"
  12086. to "movw %reg1, reg2; andw $(const1 and $ff), %reg2"}
  12087. begin
  12088. DebugMsg(SPeepholeOptimization + 'var8',p);
  12089. taicpu(p).opcode := A_MOV;
  12090. taicpu(p).changeopsize(S_W);
  12091. setsubreg(taicpu(p).oper[0]^.reg,R_SUBW);
  12092. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  12093. Result := True;
  12094. end;
  12095. end;
  12096. {$ifndef i8086} { movzbl %al,%eax cannot be encoded in 16-bit mode (the machine code is equivalent to movzbw %al,%ax }
  12097. S_BL:
  12098. if not IsMOVZXAcceptable then
  12099. begin
  12100. if (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) then
  12101. { Change "movzbl %al, %eax" to "andl $0x0ffh, %eax" }
  12102. begin
  12103. DebugMsg(SPeepholeOptimization + 'var9',p);
  12104. taicpu(p).opcode := A_AND;
  12105. taicpu(p).changeopsize(S_L);
  12106. taicpu(p).loadConst(0,$ff);
  12107. Result := True;
  12108. end
  12109. else if GetNextInstruction(p, hp1) and
  12110. (tai(hp1).typ = ait_instruction) and
  12111. (taicpu(hp1).opcode = A_AND) and
  12112. MatchOpType(taicpu(hp1),top_const,top_reg) and
  12113. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  12114. { Change "movzbl %reg1, %reg2; andl $const, %reg2"
  12115. to "movl %reg1, reg2; andl $(const1 and $ff), %reg2"}
  12116. begin
  12117. DebugMsg(SPeepholeOptimization + 'var10',p);
  12118. taicpu(p).opcode := A_MOV;
  12119. taicpu(p).changeopsize(S_L);
  12120. { do not use R_SUBWHOLE
  12121. as movl %rdx,%eax
  12122. is invalid in assembler PM }
  12123. setsubreg(taicpu(p).oper[0]^.reg, R_SUBD);
  12124. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  12125. Result := True;
  12126. end;
  12127. end;
  12128. {$endif i8086}
  12129. S_WL:
  12130. if not IsMOVZXAcceptable then
  12131. begin
  12132. if (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) then
  12133. { Change "movzwl %ax, %eax" to "andl $0x0ffffh, %eax" }
  12134. begin
  12135. DebugMsg(SPeepholeOptimization + 'var11',p);
  12136. taicpu(p).opcode := A_AND;
  12137. taicpu(p).changeopsize(S_L);
  12138. taicpu(p).loadConst(0,$ffff);
  12139. Result := True;
  12140. end
  12141. else if GetNextInstruction(p, hp1) and
  12142. (tai(hp1).typ = ait_instruction) and
  12143. (taicpu(hp1).opcode = A_AND) and
  12144. (taicpu(hp1).oper[0]^.typ = top_const) and
  12145. (taicpu(hp1).oper[1]^.typ = top_reg) and
  12146. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  12147. { Change "movzwl %reg1, %reg2; andl $const, %reg2"
  12148. to "movl %reg1, reg2; andl $(const1 and $ffff), %reg2"}
  12149. begin
  12150. DebugMsg(SPeepholeOptimization + 'var12',p);
  12151. taicpu(p).opcode := A_MOV;
  12152. taicpu(p).changeopsize(S_L);
  12153. { do not use R_SUBWHOLE
  12154. as movl %rdx,%eax
  12155. is invalid in assembler PM }
  12156. setsubreg(taicpu(p).oper[0]^.reg, R_SUBD);
  12157. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ffff);
  12158. Result := True;
  12159. end;
  12160. end;
  12161. else
  12162. InternalError(2017050705);
  12163. end;
  12164. end
  12165. else if not IsMOVZXAcceptable and (taicpu(p).oper[0]^.typ = top_ref) then
  12166. begin
  12167. if GetNextInstruction(p, hp1) and
  12168. (tai(hp1).typ = ait_instruction) and
  12169. (taicpu(hp1).opcode = A_AND) and
  12170. MatchOpType(taicpu(hp1),top_const,top_reg) and
  12171. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  12172. begin
  12173. //taicpu(p).opcode := A_MOV;
  12174. case taicpu(p).opsize Of
  12175. S_BL:
  12176. begin
  12177. DebugMsg(SPeepholeOptimization + 'var13',p);
  12178. taicpu(hp1).changeopsize(S_L);
  12179. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  12180. end;
  12181. S_WL:
  12182. begin
  12183. DebugMsg(SPeepholeOptimization + 'var14',p);
  12184. taicpu(hp1).changeopsize(S_L);
  12185. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ffff);
  12186. end;
  12187. S_BW:
  12188. begin
  12189. DebugMsg(SPeepholeOptimization + 'var15',p);
  12190. taicpu(hp1).changeopsize(S_W);
  12191. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  12192. end;
  12193. else
  12194. Internalerror(2017050704)
  12195. end;
  12196. Result := True;
  12197. end;
  12198. end;
  12199. end;
  12200. end;
  12201. function TX86AsmOptimizer.OptPass1AND(var p : tai) : boolean;
  12202. var
  12203. hp1, hp2 : tai;
  12204. MaskLength : Cardinal;
  12205. MaskedBits : TCgInt;
  12206. ActiveReg : TRegister;
  12207. begin
  12208. Result:=false;
  12209. { There are no optimisations for reference targets }
  12210. if (taicpu(p).oper[1]^.typ <> top_reg) then
  12211. Exit;
  12212. while GetNextInstruction(p, hp1) and
  12213. (hp1.typ = ait_instruction) do
  12214. begin
  12215. if (taicpu(p).oper[0]^.typ = top_const) then
  12216. begin
  12217. case taicpu(hp1).opcode of
  12218. A_AND:
  12219. if MatchOpType(taicpu(hp1),top_const,top_reg) and
  12220. (getsupreg(taicpu(p).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) and
  12221. { the second register must contain the first one, so compare their subreg types }
  12222. (getsubreg(taicpu(p).oper[1]^.reg)<=getsubreg(taicpu(hp1).oper[1]^.reg)) and
  12223. (abs(taicpu(p).oper[0]^.val and taicpu(hp1).oper[0]^.val)<$80000000) then
  12224. { change
  12225. and const1, reg
  12226. and const2, reg
  12227. to
  12228. and (const1 and const2), reg
  12229. }
  12230. begin
  12231. taicpu(hp1).loadConst(0, taicpu(p).oper[0]^.val and taicpu(hp1).oper[0]^.val);
  12232. DebugMsg(SPeepholeOptimization + 'AndAnd2And done',hp1);
  12233. RemoveCurrentP(p, hp1);
  12234. Result:=true;
  12235. exit;
  12236. end;
  12237. A_CMP:
  12238. if (PopCnt(DWord(taicpu(p).oper[0]^.val)) = 1) and { Only 1 bit set }
  12239. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^.val) and
  12240. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  12241. { Just check that the condition on the next instruction is compatible }
  12242. GetNextInstruction(hp1, hp2) and
  12243. (hp2.typ = ait_instruction) and
  12244. (taicpu(hp2).condition in [C_Z, C_E, C_NZ, C_NE])
  12245. then
  12246. { change
  12247. and 2^n, reg
  12248. cmp 2^n, reg
  12249. j(c) / set(c) / cmov(c) (c is equal or not equal)
  12250. to
  12251. and 2^n, reg
  12252. test reg, reg
  12253. j(~c) / set(~c) / cmov(~c)
  12254. }
  12255. begin
  12256. { Keep TEST instruction in, rather than remove it, because
  12257. it may trigger other optimisations such as MovAndTest2Test }
  12258. taicpu(hp1).loadreg(0, taicpu(hp1).oper[1]^.reg);
  12259. taicpu(hp1).opcode := A_TEST;
  12260. DebugMsg(SPeepholeOptimization + 'AND/CMP/J(c) -> AND/J(~c) with power of 2 constant', p);
  12261. taicpu(hp2).condition := inverse_cond(taicpu(hp2).condition);
  12262. Result := True;
  12263. Exit;
  12264. end;
  12265. A_MOVZX:
  12266. if MatchOpType(taicpu(hp1),top_reg,top_reg) and
  12267. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^.reg) and
  12268. (getsupreg(taicpu(hp1).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg)) and
  12269. (
  12270. (
  12271. (taicpu(p).opsize=S_W) and
  12272. (taicpu(hp1).opsize=S_BW)
  12273. ) or
  12274. (
  12275. (taicpu(p).opsize=S_L) and
  12276. (taicpu(hp1).opsize in [S_WL,S_BL{$ifdef x86_64},S_BQ,S_WQ{$endif x86_64}])
  12277. )
  12278. {$ifdef x86_64}
  12279. or
  12280. (
  12281. (taicpu(p).opsize=S_Q) and
  12282. (taicpu(hp1).opsize in [S_BQ,S_WQ,S_BL,S_WL])
  12283. )
  12284. {$endif x86_64}
  12285. ) then
  12286. begin
  12287. if (((taicpu(hp1).opsize) in [S_BW,S_BL{$ifdef x86_64},S_BQ{$endif x86_64}]) and
  12288. ((taicpu(p).oper[0]^.val and $ff)=taicpu(p).oper[0]^.val)
  12289. ) or
  12290. (((taicpu(hp1).opsize) in [S_WL{$ifdef x86_64},S_WQ{$endif x86_64}]) and
  12291. ((taicpu(p).oper[0]^.val and $ffff)=taicpu(p).oper[0]^.val))
  12292. then
  12293. begin
  12294. { Unlike MOVSX, MOVZX doesn't actually have a version that zero-extends a
  12295. 32-bit register to a 64-bit register, or even a version called MOVZXD, so
  12296. code that tests for the presence of AND 0xffffffff followed by MOVZX is
  12297. wasted, and is indictive of a compiler bug if it were triggered. [Kit]
  12298. NOTE: To zero-extend from 32 bits to 64 bits, simply use the standard MOV.
  12299. }
  12300. DebugMsg(SPeepholeOptimization + 'AndMovzToAnd done',p);
  12301. RemoveInstruction(hp1);
  12302. { See if there are other optimisations possible }
  12303. Continue;
  12304. end;
  12305. end;
  12306. A_SHL:
  12307. if MatchOpType(taicpu(hp1),top_const,top_reg) and
  12308. (getsupreg(taicpu(p).oper[1]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg)) then
  12309. begin
  12310. {$ifopt R+}
  12311. {$define RANGE_WAS_ON}
  12312. {$R-}
  12313. {$endif}
  12314. { get length of potential and mask }
  12315. MaskLength:=SizeOf(taicpu(p).oper[0]^.val)*8-BsrQWord(taicpu(p).oper[0]^.val)-1;
  12316. { really a mask? }
  12317. {$ifdef RANGE_WAS_ON}
  12318. {$R+}
  12319. {$endif}
  12320. if (((QWord(1) shl MaskLength)-1)=taicpu(p).oper[0]^.val) and
  12321. { unmasked part shifted out? }
  12322. ((MaskLength+taicpu(hp1).oper[0]^.val)>=topsize2memsize[taicpu(hp1).opsize]) then
  12323. begin
  12324. DebugMsg(SPeepholeOptimization + 'AndShlToShl done',p);
  12325. RemoveCurrentP(p, hp1);
  12326. Result:=true;
  12327. exit;
  12328. end;
  12329. end;
  12330. A_SHR:
  12331. if MatchOpType(taicpu(hp1),top_const,top_reg) and
  12332. (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg) and
  12333. (taicpu(hp1).oper[0]^.val <= 63) then
  12334. begin
  12335. { Does SHR combined with the AND cover all the bits?
  12336. e.g. for "andb $252,%reg; shrb $2,%reg" - the "and" can be removed }
  12337. MaskedBits := taicpu(p).oper[0]^.val or ((TCgInt(1) shl taicpu(hp1).oper[0]^.val) - 1);
  12338. if ((taicpu(p).opsize = S_B) and ((MaskedBits and $FF) = $FF)) or
  12339. ((taicpu(p).opsize = S_W) and ((MaskedBits and $FFFF) = $FFFF)) or
  12340. ((taicpu(p).opsize = S_L) and ((MaskedBits and $FFFFFFFF) = $FFFFFFFF)) then
  12341. begin
  12342. DebugMsg(SPeepholeOptimization + 'AndShrToShr done', p);
  12343. RemoveCurrentP(p, hp1);
  12344. Result := True;
  12345. Exit;
  12346. end;
  12347. end;
  12348. A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  12349. if (taicpu(hp1).oper[0]^.typ = top_reg) and
  12350. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg) then
  12351. begin
  12352. if SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) and
  12353. (
  12354. (
  12355. (taicpu(hp1).opsize in [S_BW,S_BL{$ifdef x86_64},S_BQ{$endif x86_64}]) and
  12356. ((taicpu(p).oper[0]^.val and $7F) = taicpu(p).oper[0]^.val)
  12357. ) or (
  12358. (taicpu(hp1).opsize in [S_WL{$ifdef x86_64},S_WQ{$endif x86_64}]) and
  12359. ((taicpu(p).oper[0]^.val and $7FFF) = taicpu(p).oper[0]^.val)
  12360. {$ifdef x86_64}
  12361. ) or (
  12362. (taicpu(hp1).opsize = S_LQ) and
  12363. ((taicpu(p).oper[0]^.val and $7fffffff) = taicpu(p).oper[0]^.val)
  12364. {$endif x86_64}
  12365. )
  12366. ) then
  12367. begin
  12368. if (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg){$ifdef x86_64} or (taicpu(hp1).opsize = S_LQ){$endif x86_64} then
  12369. begin
  12370. DebugMsg(SPeepholeOptimization + 'AndMovsxToAnd',p);
  12371. RemoveInstruction(hp1);
  12372. { See if there are other optimisations possible }
  12373. Continue;
  12374. end;
  12375. { The super-registers are the same though.
  12376. Note that this change by itself doesn't improve
  12377. code speed, but it opens up other optimisations. }
  12378. {$ifdef x86_64}
  12379. { Convert 64-bit register to 32-bit }
  12380. case taicpu(hp1).opsize of
  12381. S_BQ:
  12382. begin
  12383. taicpu(hp1).opsize := S_BL;
  12384. taicpu(hp1).oper[1]^.reg := newreg(R_INTREGISTER, getsupreg(taicpu(hp1).oper[1]^.reg), R_SUBD);
  12385. end;
  12386. S_WQ:
  12387. begin
  12388. taicpu(hp1).opsize := S_WL;
  12389. taicpu(hp1).oper[1]^.reg := newreg(R_INTREGISTER, getsupreg(taicpu(hp1).oper[1]^.reg), R_SUBD);
  12390. end
  12391. else
  12392. ;
  12393. end;
  12394. {$endif x86_64}
  12395. DebugMsg(SPeepholeOptimization + 'AndMovsxToAndMovzx', hp1);
  12396. taicpu(hp1).opcode := A_MOVZX;
  12397. { See if there are other optimisations possible }
  12398. Continue;
  12399. end;
  12400. end;
  12401. else
  12402. ;
  12403. end;
  12404. end
  12405. else if MatchOperand(taicpu(p).oper[0]^, taicpu(p).oper[1]^.reg) and
  12406. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  12407. begin
  12408. {$ifdef x86_64}
  12409. if (taicpu(p).opsize = S_Q) then
  12410. begin
  12411. { Never necessary }
  12412. DebugMsg(SPeepholeOptimization + 'Andq2Nop', p);
  12413. RemoveCurrentP(p, hp1);
  12414. Result := True;
  12415. Exit;
  12416. end;
  12417. {$endif x86_64}
  12418. { Forward check to determine necessity of and %reg,%reg }
  12419. TransferUsedRegs(TmpUsedRegs);
  12420. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  12421. { Saves on a bunch of dereferences }
  12422. ActiveReg := taicpu(p).oper[1]^.reg;
  12423. case taicpu(hp1).opcode of
  12424. A_MOV, A_MOVZX, A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  12425. if (
  12426. (taicpu(hp1).oper[0]^.typ <> top_ref) or
  12427. not RegInRef(ActiveReg, taicpu(hp1).oper[0]^.ref^)
  12428. ) and
  12429. (
  12430. (taicpu(hp1).opcode <> A_MOV) or
  12431. (taicpu(hp1).oper[1]^.typ <> top_ref) or
  12432. not RegInRef(ActiveReg, taicpu(hp1).oper[1]^.ref^)
  12433. ) and
  12434. not (
  12435. { If mov %reg,%reg is present, remove that instruction instead in OptPass1MOV }
  12436. (taicpu(hp1).opcode = A_MOV) and
  12437. MatchOperand(taicpu(hp1).oper[0]^, ActiveReg) and
  12438. MatchOperand(taicpu(hp1).oper[1]^, ActiveReg)
  12439. ) and
  12440. (
  12441. (
  12442. (taicpu(hp1).oper[0]^.typ = top_reg) and
  12443. (taicpu(hp1).oper[0]^.reg = ActiveReg) and
  12444. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg)
  12445. ) or
  12446. (
  12447. {$ifdef x86_64}
  12448. (
  12449. { If we read from the register, make sure it's not dependent on the upper 32 bits }
  12450. (taicpu(hp1).oper[0]^.typ <> top_reg) or
  12451. not SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, ActiveReg) or
  12452. (GetSubReg(taicpu(hp1).oper[0]^.reg) <> R_SUBQ)
  12453. ) and
  12454. {$endif x86_64}
  12455. not RegUsedAfterInstruction(ActiveReg, hp1, TmpUsedRegs)
  12456. )
  12457. ) then
  12458. begin
  12459. DebugMsg(SPeepholeOptimization + 'AndMovx2Movx', p);
  12460. RemoveCurrentP(p, hp1);
  12461. Result := True;
  12462. Exit;
  12463. end;
  12464. A_ADD,
  12465. A_AND,
  12466. A_BSF,
  12467. A_BSR,
  12468. A_BTC,
  12469. A_BTR,
  12470. A_BTS,
  12471. A_OR,
  12472. A_SUB,
  12473. A_XOR:
  12474. { Register is written to, so this will clear the upper 32 bits (2-operand instructions) }
  12475. if (
  12476. (taicpu(hp1).oper[0]^.typ <> top_ref) or
  12477. not RegInRef(ActiveReg, taicpu(hp1).oper[0]^.ref^)
  12478. ) and
  12479. MatchOperand(taicpu(hp1).oper[1]^, ActiveReg) then
  12480. begin
  12481. DebugMsg(SPeepholeOptimization + 'AndOp2Op 2', p);
  12482. RemoveCurrentP(p, hp1);
  12483. Result := True;
  12484. Exit;
  12485. end;
  12486. A_CMP,
  12487. A_TEST:
  12488. if (
  12489. (taicpu(hp1).oper[0]^.typ <> top_ref) or
  12490. not RegInRef(ActiveReg, taicpu(hp1).oper[0]^.ref^)
  12491. ) and
  12492. MatchOperand(taicpu(hp1).oper[1]^, ActiveReg) and
  12493. not RegUsedAfterInstruction(ActiveReg, hp1, TmpUsedRegs) then
  12494. begin
  12495. DebugMsg(SPeepholeOptimization + 'AND; CMP/TEST -> CMP/TEST', p);
  12496. RemoveCurrentP(p, hp1);
  12497. Result := True;
  12498. Exit;
  12499. end;
  12500. A_BSWAP,
  12501. A_NEG,
  12502. A_NOT:
  12503. { Register is written to, so this will clear the upper 32 bits (1-operand instructions) }
  12504. if MatchOperand(taicpu(hp1).oper[0]^, ActiveReg) then
  12505. begin
  12506. DebugMsg(SPeepholeOptimization + 'AndOp2Op 1', p);
  12507. RemoveCurrentP(p, hp1);
  12508. Result := True;
  12509. Exit;
  12510. end;
  12511. else
  12512. ;
  12513. end;
  12514. end;
  12515. if (taicpu(hp1).is_jmp) and
  12516. (taicpu(hp1).opcode<>A_JMP) and
  12517. not(RegInUsedRegs(taicpu(p).oper[1]^.reg,UsedRegs)) then
  12518. begin
  12519. { change
  12520. and x, reg
  12521. jxx
  12522. to
  12523. test x, reg
  12524. jxx
  12525. if reg is deallocated before the
  12526. jump, but only if it's a conditional jump (PFV)
  12527. }
  12528. DebugMsg(SPeepholeOptimization + 'AndJcc2TestJcc', p);
  12529. taicpu(p).opcode := A_TEST;
  12530. Exit;
  12531. end;
  12532. Break;
  12533. end;
  12534. { Lone AND tests }
  12535. if (taicpu(p).oper[0]^.typ = top_const) then
  12536. begin
  12537. {
  12538. - Convert and $0xFF,reg to and reg,reg if reg is 8-bit
  12539. - Convert and $0xFFFF,reg to and reg,reg if reg is 16-bit
  12540. - Convert and $0xFFFFFFFF,reg to and reg,reg if reg is 32-bit
  12541. }
  12542. if ((taicpu(p).oper[0]^.val = $FF) and (taicpu(p).opsize = S_B)) or
  12543. ((taicpu(p).oper[0]^.val = $FFFF) and (taicpu(p).opsize = S_W)) or
  12544. ((taicpu(p).oper[0]^.val = $FFFFFFFF) and (taicpu(p).opsize = S_L)) then
  12545. begin
  12546. taicpu(p).loadreg(0, taicpu(p).oper[1]^.reg);
  12547. if taicpu(p).opsize = S_L then
  12548. begin
  12549. Include(OptsToCheck,aoc_MovAnd2Mov_3);
  12550. Result := True;
  12551. end;
  12552. end;
  12553. end;
  12554. { Backward check to determine necessity of and %reg,%reg }
  12555. if (taicpu(p).oper[0]^.typ = top_reg) and
  12556. (taicpu(p).oper[0]^.reg = taicpu(p).oper[1]^.reg) and
  12557. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  12558. GetLastInstruction(p, hp2) and
  12559. RegModifiedByInstruction(taicpu(p).oper[1]^.reg, hp2) and
  12560. { Check size of adjacent instruction to determine if the AND is
  12561. effectively a null operation }
  12562. (
  12563. (taicpu(p).opsize = taicpu(hp2).opsize) or
  12564. { Note: Don't include S_Q }
  12565. ((taicpu(p).opsize = S_L) and (taicpu(hp2).opsize in [S_BL, S_WL])) or
  12566. ((taicpu(p).opsize = S_W) and (taicpu(hp2).opsize in [S_BW, S_BL, S_WL, S_L])) or
  12567. ((taicpu(p).opsize = S_B) and (taicpu(hp2).opsize in [S_BW, S_BL, S_WL, S_W, S_L]))
  12568. ) then
  12569. begin
  12570. DebugMsg(SPeepholeOptimization + 'And2Nop', p);
  12571. { If GetNextInstruction returned False, hp1 will be nil }
  12572. RemoveCurrentP(p, hp1);
  12573. Result := True;
  12574. Exit;
  12575. end;
  12576. end;
  12577. function TX86AsmOptimizer.OptPass2ADD(var p : tai) : boolean;
  12578. var
  12579. hp1, hp2: tai;
  12580. NewRef: TReference;
  12581. Distance: Cardinal;
  12582. TempTracking: TAllUsedRegs;
  12583. { This entire nested function is used in an if-statement below, but we
  12584. want to avoid all the used reg transfers and GetNextInstruction calls
  12585. until we really have to check }
  12586. function MemRegisterNotUsedLater: Boolean; inline;
  12587. var
  12588. hp2: tai;
  12589. begin
  12590. TransferUsedRegs(TmpUsedRegs);
  12591. hp2 := p;
  12592. repeat
  12593. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  12594. until not (cs_opt_level3 in current_settings.optimizerswitches) or not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  12595. Result := not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs);
  12596. end;
  12597. begin
  12598. Result := False;
  12599. if (taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif}]) and
  12600. (taicpu(p).oper[1]^.typ = top_reg) then
  12601. begin
  12602. Distance := GetNextInstructionUsingRegCount(p, hp1, taicpu(p).oper[1]^.reg);
  12603. if (Distance = 0) or (Distance > 3) { Likely too far to make a meaningful difference } or
  12604. (hp1.typ <> ait_instruction) or
  12605. not
  12606. (
  12607. (cs_opt_level3 in current_settings.optimizerswitches) or
  12608. { GetNextInstructionUsingRegCount just returns the next valid instruction under -O2 and under }
  12609. RegInInstruction(taicpu(p).oper[1]^.reg, hp1)
  12610. ) then
  12611. Exit;
  12612. { Some of the MOV optimisations are much more in-depth. For example, if we have:
  12613. addq $x, %rax
  12614. movq %rax, %rdx
  12615. sarq $63, %rdx
  12616. (%rax still in use)
  12617. ...letting OptPass2ADD run its course (and without -Os) will produce:
  12618. leaq $x(%rax),%rdx
  12619. addq $x, %rax
  12620. sarq $63, %rdx
  12621. ...which is okay since it breaks the dependency chain between
  12622. addq and movq, but if OptPass2MOV is called first:
  12623. addq $x, %rax
  12624. cqto
  12625. ...which is better in all ways, taking only 2 cycles to execute
  12626. and much smaller in code size.
  12627. }
  12628. { The extra register tracking is quite strenuous }
  12629. if (cs_opt_level2 in current_settings.optimizerswitches) and
  12630. MatchInstruction(hp1, A_MOV, []) then
  12631. begin
  12632. { Update the register tracking to the MOV instruction }
  12633. CopyUsedRegs(TempTracking);
  12634. hp2 := p;
  12635. repeat
  12636. UpdateUsedRegs(tai(hp2.Next));
  12637. until not (cs_opt_level3 in current_settings.optimizerswitches) or not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  12638. { if hp1 <> hp2 after the call, then hp1 got removed, so let
  12639. OptPass2ADD get called again }
  12640. if OptPass2MOV(hp1) and (hp1 <> hp2) then
  12641. begin
  12642. { Reset the tracking to the current instruction }
  12643. RestoreUsedRegs(TempTracking);
  12644. ReleaseUsedRegs(TempTracking);
  12645. Result := True;
  12646. Exit;
  12647. end;
  12648. { Reset the tracking to the current instruction }
  12649. RestoreUsedRegs(TempTracking);
  12650. ReleaseUsedRegs(TempTracking);
  12651. { If OptPass2MOV returned True, we don't need to set Result to
  12652. True if hp1 didn't change because the ADD instruction didn't
  12653. get modified and we'll be evaluating hp1 again when the
  12654. peephole optimizer reaches it }
  12655. end;
  12656. { Change:
  12657. add %reg2,%reg1
  12658. (%reg2 not modified in between)
  12659. mov/s/z #(%reg1),%reg1 (%reg1 superregisters must be the same)
  12660. To:
  12661. mov/s/z #(%reg1,%reg2),%reg1
  12662. }
  12663. if (taicpu(p).oper[0]^.typ = top_reg) and
  12664. MatchInstruction(hp1, [A_MOV, A_MOVZX, A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif}], []) and
  12665. MatchOpType(taicpu(hp1), top_ref, top_reg) and
  12666. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 1) and
  12667. (
  12668. (
  12669. (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) and
  12670. (taicpu(hp1).oper[0]^.ref^.index = NR_NO) and
  12671. { r/esp cannot be an index }
  12672. (taicpu(p).oper[0]^.reg<>NR_STACK_POINTER_REG)
  12673. ) or (
  12674. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  12675. (taicpu(hp1).oper[0]^.ref^.base = NR_NO)
  12676. )
  12677. ) and (
  12678. Reg1WriteOverwritesReg2Entirely(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) or
  12679. (
  12680. { If the super registers ARE equal, then this MOV/S/Z does a partial write }
  12681. not SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) and
  12682. MemRegisterNotUsedLater
  12683. )
  12684. ) then
  12685. begin
  12686. if (
  12687. { Instructions are guaranteed to be adjacent on -O2 and under }
  12688. (cs_opt_level3 in current_settings.optimizerswitches) and
  12689. RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp1)
  12690. ) then
  12691. begin
  12692. { If the other register is used in between, move the MOV
  12693. instruction to right after the ADD instruction so a
  12694. saving can still be made }
  12695. Asml.Remove(hp1);
  12696. Asml.InsertAfter(hp1, p);
  12697. taicpu(hp1).oper[0]^.ref^.base := taicpu(p).oper[1]^.reg;
  12698. taicpu(hp1).oper[0]^.ref^.index := taicpu(p).oper[0]^.reg;
  12699. DebugMsg(SPeepholeOptimization + 'AddMov2Mov done (instruction moved)', p);
  12700. RemoveCurrentp(p, hp1);
  12701. end
  12702. else
  12703. begin
  12704. AllocRegBetween(taicpu(p).oper[0]^.reg, p, hp1, UsedRegs);
  12705. taicpu(hp1).oper[0]^.ref^.base := taicpu(p).oper[1]^.reg;
  12706. taicpu(hp1).oper[0]^.ref^.index := taicpu(p).oper[0]^.reg;
  12707. DebugMsg(SPeepholeOptimization + 'AddMov2Mov done', p);
  12708. if (cs_opt_level3 in current_settings.optimizerswitches) then
  12709. { hp1 may not be the immediate next instruction under -O3 }
  12710. RemoveCurrentp(p)
  12711. else
  12712. RemoveCurrentp(p, hp1);
  12713. end;
  12714. Result := True;
  12715. Exit;
  12716. end;
  12717. { Change:
  12718. addl/q $x,%reg1
  12719. movl/q %reg1,%reg2
  12720. To:
  12721. leal/q $x(%reg1),%reg2
  12722. addl/q $x,%reg1 (can be removed if %reg1 or the flags are not used afterwards)
  12723. Breaks the dependency chain.
  12724. }
  12725. if (taicpu(p).oper[0]^.typ = top_const) and
  12726. MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  12727. (taicpu(hp1).oper[1]^.typ = top_reg) and
  12728. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) and
  12729. (
  12730. { Instructions are guaranteed to be adjacent on -O2 and under }
  12731. not (cs_opt_level3 in current_settings.optimizerswitches) or
  12732. not RegUsedBetween(taicpu(hp1).oper[1]^.reg, p, hp1)
  12733. ) then
  12734. begin
  12735. TransferUsedRegs(TmpUsedRegs);
  12736. hp2 := p;
  12737. repeat
  12738. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  12739. until not (cs_opt_level3 in current_settings.optimizerswitches) or not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  12740. if (
  12741. { Don't do AddMov2LeaAdd under -Os, but do allow AddMov2Lea }
  12742. not (cs_opt_size in current_settings.optimizerswitches) or
  12743. (
  12744. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) and
  12745. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)
  12746. )
  12747. ) then
  12748. begin
  12749. { Change the MOV instruction to a LEA instruction, and update the
  12750. first operand }
  12751. reference_reset(NewRef, 1, []);
  12752. NewRef.base := taicpu(p).oper[1]^.reg;
  12753. NewRef.scalefactor := 1;
  12754. NewRef.offset := asizeint(taicpu(p).oper[0]^.val);
  12755. taicpu(hp1).opcode := A_LEA;
  12756. taicpu(hp1).loadref(0, NewRef);
  12757. if RegUsedAfterInstruction(NewRef.base, hp1, TmpUsedRegs) or
  12758. RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs) then
  12759. begin
  12760. hp2 := tai(hp1.Next); { for the benefit of AllocRegBetween }
  12761. { Move what is now the LEA instruction to before the ADD instruction }
  12762. Asml.Remove(hp1);
  12763. Asml.InsertBefore(hp1, p);
  12764. AllocRegBetween(taicpu(hp1).oper[1]^.reg, hp1, hp2, UsedRegs);
  12765. DebugMsg(SPeepholeOptimization + 'AddMov2LeaAdd', p);
  12766. p := hp1;
  12767. end
  12768. else
  12769. begin
  12770. { Since %reg1 or the flags aren't used afterwards, we can delete p completely }
  12771. DebugMsg(SPeepholeOptimization + 'AddMov2Lea', hp1);
  12772. if (cs_opt_level3 in current_settings.optimizerswitches) then
  12773. { hp1 may not be the immediate next instruction under -O3 }
  12774. RemoveCurrentp(p)
  12775. else
  12776. RemoveCurrentp(p, hp1);
  12777. end;
  12778. Result := True;
  12779. end;
  12780. end;
  12781. end;
  12782. end;
  12783. function TX86AsmOptimizer.OptPass2Lea(var p : tai) : Boolean;
  12784. var
  12785. SubReg: TSubRegister;
  12786. begin
  12787. Result:=false;
  12788. SubReg := getsubreg(taicpu(p).oper[1]^.reg);
  12789. if not (RegInUsedRegs(NR_DEFAULTFLAGS,UsedRegs)) then
  12790. with taicpu(p).oper[0]^.ref^ do
  12791. if (offset = 0) and not Assigned(symbol) and not Assigned(relsymbol) and (index <> NR_NO) then
  12792. begin
  12793. if (scalefactor <= 1) and SuperRegistersEqual(base, taicpu(p).oper[1]^.reg) then
  12794. begin
  12795. taicpu(p).loadreg(0, newreg(R_INTREGISTER, getsupreg(index), SubReg));
  12796. taicpu(p).opcode := A_ADD;
  12797. DebugMsg(SPeepholeOptimization + 'Lea2AddBase done',p);
  12798. Result := True;
  12799. end
  12800. else if SuperRegistersEqual(index, taicpu(p).oper[1]^.reg) then
  12801. begin
  12802. if (base <> NR_NO) then
  12803. begin
  12804. if (scalefactor <= 1) then
  12805. begin
  12806. taicpu(p).loadreg(0, newreg(R_INTREGISTER, getsupreg(base), SubReg));
  12807. taicpu(p).opcode := A_ADD;
  12808. DebugMsg(SPeepholeOptimization + 'Lea2AddIndex done',p);
  12809. Result := True;
  12810. end;
  12811. end
  12812. else
  12813. { Convert lea (%reg,2^x),%reg to shl x,%reg }
  12814. if (scalefactor in [2, 4, 8]) then
  12815. begin
  12816. { BsrByte is, in essence, the base-2 logarithm of the scale factor }
  12817. taicpu(p).loadconst(0, BsrByte(scalefactor));
  12818. taicpu(p).opcode := A_SHL;
  12819. DebugMsg(SPeepholeOptimization + 'Lea2Shl done',p);
  12820. Result := True;
  12821. end;
  12822. end;
  12823. end;
  12824. end;
  12825. function TX86AsmOptimizer.OptPass2SUB(var p: tai): Boolean;
  12826. var
  12827. hp1, hp2: tai;
  12828. NewRef: TReference;
  12829. Distance: Cardinal;
  12830. TempTracking: TAllUsedRegs;
  12831. begin
  12832. Result := False;
  12833. if (taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif}]) and
  12834. MatchOpType(taicpu(p),top_const,top_reg) then
  12835. begin
  12836. Distance := GetNextInstructionUsingRegCount(p, hp1, taicpu(p).oper[1]^.reg);
  12837. if (Distance = 0) or (Distance > 3) { Likely too far to make a meaningful difference } or
  12838. (hp1.typ <> ait_instruction) or
  12839. not
  12840. (
  12841. (cs_opt_level3 in current_settings.optimizerswitches) or
  12842. { GetNextInstructionUsingRegCount just returns the next valid instruction under -O2 and under }
  12843. RegInInstruction(taicpu(p).oper[1]^.reg, hp1)
  12844. ) then
  12845. Exit;
  12846. { Some of the MOV optimisations are much more in-depth. For example, if we have:
  12847. subq $x, %rax
  12848. movq %rax, %rdx
  12849. sarq $63, %rdx
  12850. (%rax still in use)
  12851. ...letting OptPass2SUB run its course (and without -Os) will produce:
  12852. leaq $-x(%rax),%rdx
  12853. movq $x, %rax
  12854. sarq $63, %rdx
  12855. ...which is okay since it breaks the dependency chain between
  12856. subq and movq, but if OptPass2MOV is called first:
  12857. subq $x, %rax
  12858. cqto
  12859. ...which is better in all ways, taking only 2 cycles to execute
  12860. and much smaller in code size.
  12861. }
  12862. { The extra register tracking is quite strenuous }
  12863. if (cs_opt_level2 in current_settings.optimizerswitches) and
  12864. MatchInstruction(hp1, A_MOV, []) then
  12865. begin
  12866. { Update the register tracking to the MOV instruction }
  12867. CopyUsedRegs(TempTracking);
  12868. hp2 := p;
  12869. repeat
  12870. UpdateUsedRegs(tai(hp2.Next));
  12871. until not (cs_opt_level3 in current_settings.optimizerswitches) or not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  12872. { if hp1 <> hp2 after the call, then hp1 got removed, so let
  12873. OptPass2SUB get called again }
  12874. if OptPass2MOV(hp1) and (hp1 <> hp2) then
  12875. begin
  12876. { Reset the tracking to the current instruction }
  12877. RestoreUsedRegs(TempTracking);
  12878. ReleaseUsedRegs(TempTracking);
  12879. Result := True;
  12880. Exit;
  12881. end;
  12882. { Reset the tracking to the current instruction }
  12883. RestoreUsedRegs(TempTracking);
  12884. ReleaseUsedRegs(TempTracking);
  12885. { If OptPass2MOV returned True, we don't need to set Result to
  12886. True if hp1 didn't change because the SUB instruction didn't
  12887. get modified and we'll be evaluating hp1 again when the
  12888. peephole optimizer reaches it }
  12889. end;
  12890. { Change:
  12891. subl/q $x,%reg1
  12892. movl/q %reg1,%reg2
  12893. To:
  12894. leal/q $-x(%reg1),%reg2
  12895. subl/q $x,%reg1 (can be removed if %reg1 or the flags are not used afterwards)
  12896. Breaks the dependency chain and potentially permits the removal of
  12897. a CMP instruction if one follows.
  12898. }
  12899. if MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  12900. (taicpu(hp1).oper[1]^.typ = top_reg) and
  12901. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) and
  12902. (
  12903. { Instructions are guaranteed to be adjacent on -O2 and under }
  12904. not (cs_opt_level3 in current_settings.optimizerswitches) or
  12905. not RegUsedBetween(taicpu(hp1).oper[1]^.reg, p, hp1)
  12906. ) then
  12907. begin
  12908. TransferUsedRegs(TmpUsedRegs);
  12909. hp2 := p;
  12910. repeat
  12911. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  12912. until not (cs_opt_level3 in current_settings.optimizerswitches) or not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  12913. if (
  12914. { Don't do SubMov2LeaSub under -Os, but do allow SubMov2Lea }
  12915. not (cs_opt_size in current_settings.optimizerswitches) or
  12916. (
  12917. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) and
  12918. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)
  12919. )
  12920. ) then
  12921. begin
  12922. { Change the MOV instruction to a LEA instruction, and update the
  12923. first operand }
  12924. reference_reset(NewRef, 1, []);
  12925. NewRef.base := taicpu(p).oper[1]^.reg;
  12926. NewRef.scalefactor := 1;
  12927. NewRef.offset := -taicpu(p).oper[0]^.val;
  12928. taicpu(hp1).opcode := A_LEA;
  12929. taicpu(hp1).loadref(0, NewRef);
  12930. TransferUsedRegs(TmpUsedRegs);
  12931. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  12932. if RegUsedAfterInstruction(NewRef.base, hp1, TmpUsedRegs) or
  12933. RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs) then
  12934. begin
  12935. hp2 := tai(hp1.Next); { for the benefit of AllocRegBetween }
  12936. { Move what is now the LEA instruction to before the SUB instruction }
  12937. Asml.Remove(hp1);
  12938. Asml.InsertBefore(hp1, p);
  12939. AllocRegBetween(taicpu(hp1).oper[1]^.reg, hp1, hp2, UsedRegs);
  12940. DebugMsg(SPeepholeOptimization + 'SubMov2LeaSub', p);
  12941. p := hp1;
  12942. end
  12943. else
  12944. begin
  12945. { Since %reg1 or the flags aren't used afterwards, we can delete p completely }
  12946. DebugMsg(SPeepholeOptimization + 'SubMov2Lea', hp1);
  12947. if (cs_opt_level3 in current_settings.optimizerswitches) then
  12948. { hp1 may not be the immediate next instruction under -O3 }
  12949. RemoveCurrentp(p)
  12950. else
  12951. RemoveCurrentp(p, hp1);
  12952. end;
  12953. Result := True;
  12954. end;
  12955. end;
  12956. end;
  12957. end;
  12958. function TX86AsmOptimizer.SkipSimpleInstructions(var hp1 : tai) : Boolean;
  12959. begin
  12960. { we can skip all instructions not messing with the stack pointer }
  12961. while assigned(hp1) and {MatchInstruction(hp1,[A_LEA,A_MOV,A_MOVQ,A_MOVSQ,A_MOVSX,A_MOVSXD,A_MOVZX,
  12962. A_AND,A_OR,A_XOR,A_ADD,A_SHR,A_SHL,A_IMUL,A_SETcc,A_SAR,A_SUB,A_TEST,A_CMOVcc,
  12963. A_MOVSS,A_MOVSD,A_MOVAPS,A_MOVUPD,A_MOVAPD,A_MOVUPS,
  12964. A_VMOVSS,A_VMOVSD,A_VMOVAPS,A_VMOVUPD,A_VMOVAPD,A_VMOVUPS],[]) and}
  12965. ({(taicpu(hp1).ops=0) or }
  12966. ({(MatchOpType(taicpu(hp1),top_reg,top_reg) or MatchOpType(taicpu(hp1),top_const,top_reg) or
  12967. (MatchOpType(taicpu(hp1),top_ref,top_reg))
  12968. ) and }
  12969. not(RegInInstruction(NR_STACK_POINTER_REG,hp1)) { and not(RegInInstruction(NR_FRAME_POINTER_REG,hp1))}
  12970. )
  12971. ) do
  12972. GetNextInstruction(hp1,hp1);
  12973. Result:=assigned(hp1);
  12974. end;
  12975. function TX86AsmOptimizer.PostPeepholeOptLea(var p : tai) : Boolean;
  12976. var
  12977. hp1, hp2, hp3, hp4, hp5: tai;
  12978. begin
  12979. Result:=false;
  12980. hp5:=nil;
  12981. { replace
  12982. leal(q) x(<stackpointer>),<stackpointer>
  12983. call procname
  12984. leal(q) -x(<stackpointer>),<stackpointer>
  12985. ret
  12986. by
  12987. jmp procname
  12988. but do it only on level 4 because it destroys stack back traces
  12989. }
  12990. if (cs_opt_level4 in current_settings.optimizerswitches) and
  12991. MatchOpType(taicpu(p),top_ref,top_reg) and
  12992. (taicpu(p).oper[0]^.ref^.base=NR_STACK_POINTER_REG) and
  12993. (taicpu(p).oper[0]^.ref^.index=NR_NO) and
  12994. { the -8 or -24 are not required, but bail out early if possible,
  12995. higher values are unlikely }
  12996. ((taicpu(p).oper[0]^.ref^.offset=-8) or
  12997. (taicpu(p).oper[0]^.ref^.offset=-24)) and
  12998. (taicpu(p).oper[0]^.ref^.symbol=nil) and
  12999. (taicpu(p).oper[0]^.ref^.relsymbol=nil) and
  13000. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG) and
  13001. GetNextInstruction(p, hp1) and
  13002. { Take a copy of hp1 }
  13003. SetAndTest(hp1, hp4) and
  13004. { trick to skip label }
  13005. ((hp1.typ=ait_instruction) or GetNextInstruction(hp1, hp1)) and
  13006. SkipSimpleInstructions(hp1) and
  13007. MatchInstruction(hp1,A_CALL,[S_NO]) and
  13008. GetNextInstruction(hp1, hp2) and
  13009. MatchInstruction(hp2,A_LEA,[taicpu(p).opsize]) and
  13010. MatchOpType(taicpu(hp2),top_ref,top_reg) and
  13011. (taicpu(hp2).oper[0]^.ref^.offset=-taicpu(p).oper[0]^.ref^.offset) and
  13012. (taicpu(hp2).oper[0]^.ref^.base=NR_STACK_POINTER_REG) and
  13013. (taicpu(hp2).oper[0]^.ref^.index=NR_NO) and
  13014. (taicpu(hp2).oper[0]^.ref^.symbol=nil) and
  13015. (taicpu(hp2).oper[0]^.ref^.relsymbol=nil) and
  13016. { Segment register will be NR_NO }
  13017. (taicpu(hp2).oper[1]^.reg=NR_STACK_POINTER_REG) and
  13018. GetNextInstruction(hp2, hp3) and
  13019. { trick to skip label }
  13020. ((hp3.typ=ait_instruction) or GetNextInstruction(hp3, hp3)) and
  13021. (MatchInstruction(hp3,A_RET,[S_NO]) or
  13022. (MatchInstruction(hp3,A_VZEROUPPER,[S_NO]) and
  13023. SetAndTest(hp3,hp5) and
  13024. GetNextInstruction(hp3,hp3) and
  13025. MatchInstruction(hp3,A_RET,[S_NO])
  13026. )
  13027. ) and
  13028. (taicpu(hp3).ops=0) then
  13029. begin
  13030. taicpu(hp1).opcode := A_JMP;
  13031. taicpu(hp1).is_jmp := true;
  13032. DebugMsg(SPeepholeOptimization + 'LeaCallLeaRet2Jmp done',p);
  13033. RemoveCurrentP(p, hp4);
  13034. RemoveInstruction(hp2);
  13035. RemoveInstruction(hp3);
  13036. if Assigned(hp5) then
  13037. begin
  13038. AsmL.Remove(hp5);
  13039. ASmL.InsertBefore(hp5,hp1)
  13040. end;
  13041. Result:=true;
  13042. end;
  13043. end;
  13044. function TX86AsmOptimizer.PostPeepholeOptPush(var p : tai) : Boolean;
  13045. {$ifdef x86_64}
  13046. var
  13047. hp1, hp2, hp3, hp4, hp5: tai;
  13048. {$endif x86_64}
  13049. begin
  13050. Result:=false;
  13051. {$ifdef x86_64}
  13052. hp5:=nil;
  13053. { replace
  13054. push %rax
  13055. call procname
  13056. pop %rcx
  13057. ret
  13058. by
  13059. jmp procname
  13060. but do it only on level 4 because it destroys stack back traces
  13061. It depends on the fact, that the sequence push rax/pop rcx is used for stack alignment as rcx is volatile
  13062. for all supported calling conventions
  13063. }
  13064. if (cs_opt_level4 in current_settings.optimizerswitches) and
  13065. MatchOpType(taicpu(p),top_reg) and
  13066. (taicpu(p).oper[0]^.reg=NR_RAX) and
  13067. GetNextInstruction(p, hp1) and
  13068. { Take a copy of hp1 }
  13069. SetAndTest(hp1, hp4) and
  13070. { trick to skip label }
  13071. ((hp1.typ=ait_instruction) or GetNextInstruction(hp1, hp1)) and
  13072. SkipSimpleInstructions(hp1) and
  13073. MatchInstruction(hp1,A_CALL,[S_NO]) and
  13074. GetNextInstruction(hp1, hp2) and
  13075. MatchInstruction(hp2,A_POP,[taicpu(p).opsize]) and
  13076. MatchOpType(taicpu(hp2),top_reg) and
  13077. (taicpu(hp2).oper[0]^.reg=NR_RCX) and
  13078. GetNextInstruction(hp2, hp3) and
  13079. { trick to skip label }
  13080. ((hp3.typ=ait_instruction) or GetNextInstruction(hp3, hp3)) and
  13081. (MatchInstruction(hp3,A_RET,[S_NO]) or
  13082. (MatchInstruction(hp3,A_VZEROUPPER,[S_NO]) and
  13083. SetAndTest(hp3,hp5) and
  13084. GetNextInstruction(hp3,hp3) and
  13085. MatchInstruction(hp3,A_RET,[S_NO])
  13086. )
  13087. ) and
  13088. (taicpu(hp3).ops=0) then
  13089. begin
  13090. taicpu(hp1).opcode := A_JMP;
  13091. taicpu(hp1).is_jmp := true;
  13092. DebugMsg(SPeepholeOptimization + 'PushCallPushRet2Jmp done',p);
  13093. RemoveCurrentP(p, hp4);
  13094. RemoveInstruction(hp2);
  13095. RemoveInstruction(hp3);
  13096. if Assigned(hp5) then
  13097. begin
  13098. AsmL.Remove(hp5);
  13099. ASmL.InsertBefore(hp5,hp1)
  13100. end;
  13101. Result:=true;
  13102. end;
  13103. {$endif x86_64}
  13104. end;
  13105. function TX86AsmOptimizer.PostPeepholeOptMov(var p : tai) : Boolean;
  13106. var
  13107. Value, RegName: string;
  13108. begin
  13109. Result:=false;
  13110. if (taicpu(p).oper[1]^.typ = top_reg) and (taicpu(p).oper[0]^.typ = top_const) then
  13111. begin
  13112. case taicpu(p).oper[0]^.val of
  13113. 0:
  13114. { Don't make this optimisation if the CPU flags are required, since XOR scrambles them }
  13115. if not (RegInUsedRegs(NR_DEFAULTFLAGS,UsedRegs)) then
  13116. begin
  13117. { change "mov $0,%reg" into "xor %reg,%reg" }
  13118. taicpu(p).opcode := A_XOR;
  13119. taicpu(p).loadReg(0,taicpu(p).oper[1]^.reg);
  13120. Result := True;
  13121. {$ifdef x86_64}
  13122. end
  13123. else if (taicpu(p).opsize = S_Q) then
  13124. begin
  13125. RegName := debug_regname(taicpu(p).oper[1]^.reg); { 64-bit register name }
  13126. { The actual optimization }
  13127. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  13128. taicpu(p).changeopsize(S_L);
  13129. DebugMsg(SPeepholeOptimization + 'movq $0,' + RegName + ' -> movl $0,' + debug_regname(taicpu(p).oper[1]^.reg) + ' (immediate can be represented with just 32 bits)', p);
  13130. Result := True;
  13131. end;
  13132. $1..$FFFFFFFF:
  13133. begin
  13134. { Code size reduction by J. Gareth "Kit" Moreton }
  13135. { change 64-bit register to 32-bit register to reduce code size (upper 32 bits will be set to zero) }
  13136. case taicpu(p).opsize of
  13137. S_Q:
  13138. begin
  13139. RegName := debug_regname(taicpu(p).oper[1]^.reg); { 64-bit register name }
  13140. Value := debug_tostr(taicpu(p).oper[0]^.val);
  13141. { The actual optimization }
  13142. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  13143. taicpu(p).changeopsize(S_L);
  13144. DebugMsg(SPeepholeOptimization + 'movq $' + Value + ',' + RegName + ' -> movl $' + Value + ',' + debug_regname(taicpu(p).oper[1]^.reg) + ' (immediate can be represented with just 32 bits)', p);
  13145. Result := True;
  13146. end;
  13147. else
  13148. { Do nothing };
  13149. end;
  13150. {$endif x86_64}
  13151. end;
  13152. -1:
  13153. { Don't make this optimisation if the CPU flags are required, since OR scrambles them }
  13154. if (cs_opt_size in current_settings.optimizerswitches) and
  13155. (taicpu(p).opsize <> S_B) and
  13156. not (RegInUsedRegs(NR_DEFAULTFLAGS,UsedRegs)) then
  13157. begin
  13158. { change "mov $-1,%reg" into "or $-1,%reg" }
  13159. { NOTES:
  13160. - No size saving is made when changing a Word-sized assignment unless the register is AX (smaller encoding)
  13161. - This operation creates a false dependency on the register, so only do it when optimising for size
  13162. - It is possible to set memory operands using this method, but this creates an even greater false dependency, so don't do this at all
  13163. }
  13164. taicpu(p).opcode := A_OR;
  13165. DebugMsg(SPeepholeOptimization + 'Mov-12Or-1',p);
  13166. Result := True;
  13167. end;
  13168. else
  13169. { Do nothing };
  13170. end;
  13171. end;
  13172. end;
  13173. { Returns true if the given logic instruction can be converted into a BTx instruction (BT not included) }
  13174. class function TX86AsmOptimizer.IsBTXAcceptable(p : tai) : boolean;
  13175. begin
  13176. Result := False;
  13177. if not (CPUX86_HAS_BTX in cpu_capabilities[current_settings.optimizecputype]) then
  13178. Exit;
  13179. { For sizes less than S_L, the byte size is equal or larger with BTx,
  13180. so don't bother optimising }
  13181. if not MatchInstruction(p, A_AND, A_OR, A_XOR, [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  13182. Exit;
  13183. if (taicpu(p).oper[0]^.typ <> top_const) or
  13184. { If the value can fit into an 8-bit signed integer, a smaller
  13185. instruction can be encoded with AND/OR/XOR, so don't optimise if it
  13186. falls within this range }
  13187. (
  13188. (taicpu(p).oper[0]^.val > -128) and
  13189. (taicpu(p).oper[0]^.val <= 127)
  13190. ) then
  13191. Exit;
  13192. { If we're optimising for size, this is acceptable }
  13193. if (cs_opt_size in current_settings.optimizerswitches) then
  13194. Exit(True);
  13195. if (taicpu(p).oper[1]^.typ = top_reg) and
  13196. (CPUX86_HINT_FAST_BTX_REG_IMM in cpu_optimization_hints[current_settings.optimizecputype]) then
  13197. Exit(True);
  13198. if (taicpu(p).oper[1]^.typ <> top_reg) and
  13199. (CPUX86_HINT_FAST_BTX_MEM_IMM in cpu_optimization_hints[current_settings.optimizecputype]) then
  13200. Exit(True);
  13201. end;
  13202. function TX86AsmOptimizer.PostPeepholeOptAnd(var p : tai) : boolean;
  13203. var
  13204. hp1: tai;
  13205. Value: TCGInt;
  13206. begin
  13207. Result := False;
  13208. if MatchOpType(taicpu(p), top_const, top_reg) then
  13209. begin
  13210. { Detect:
  13211. andw x, %ax (0 <= x < $8000)
  13212. ...
  13213. movzwl %ax,%eax
  13214. Change movzwl %ax,%eax to cwtl (shorter encoding for movswl %ax,%eax)
  13215. }
  13216. if (taicpu(p).oper[1]^.reg = NR_AX) and { This is also enough to determine that opsize = S_W }
  13217. ((taicpu(p).oper[0]^.val and $7FFF) = taicpu(p).oper[0]^.val) and
  13218. GetNextInstructionUsingReg(p, hp1, NR_EAX) and
  13219. MatchInstruction(hp1, A_MOVZX, [S_WL]) and
  13220. MatchOperand(taicpu(hp1).oper[0]^, NR_AX) and
  13221. MatchOperand(taicpu(hp1).oper[1]^, NR_EAX) then
  13222. begin
  13223. DebugMsg(SPeepholeOptimization + 'Converted movzwl %ax,%eax to cwtl (via AndMovz2AndCwtl)', hp1);
  13224. taicpu(hp1).opcode := A_CWDE;
  13225. taicpu(hp1).clearop(0);
  13226. taicpu(hp1).clearop(1);
  13227. taicpu(hp1).ops := 0;
  13228. { A change was made, but not with p, so move forward 1 }
  13229. p := tai(p.Next);
  13230. Result := True;
  13231. Exit; { and -> btr won't happen because an opsize of S_W won't be optimised anyway }
  13232. end;
  13233. end;
  13234. { If "not x" is a power of 2 (popcnt = 1), change:
  13235. and $x, %reg/ref
  13236. To:
  13237. btr lb(x), %reg/ref
  13238. }
  13239. if IsBTXAcceptable(p) and
  13240. (
  13241. { Make sure a TEST doesn't follow that plays with the register }
  13242. not GetNextInstruction(p, hp1) or
  13243. not MatchInstruction(hp1, A_TEST, A_CMP, [taicpu(p).opsize]) or
  13244. not MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg)
  13245. ) then
  13246. begin
  13247. {$push}{$R-}{$Q-}
  13248. { Value is a sign-extended 32-bit integer - just correct it
  13249. if it's represented as an unsigned value. Also, IsBTXAcceptable
  13250. checks to see if this operand is an immediate. }
  13251. Value := not taicpu(p).oper[0]^.val;
  13252. {$pop}
  13253. {$ifdef x86_64}
  13254. if taicpu(p).opsize = S_L then
  13255. {$endif x86_64}
  13256. Value := Value and $FFFFFFFF;
  13257. if (PopCnt(QWord(Value)) = 1) then
  13258. begin
  13259. DebugMsg(SPeepholeOptimization + 'Changed AND (not $0x' + hexstr(taicpu(p).oper[0]^.val, 2) + ') to BTR ' + debug_tostr(BsrQWord(Value)) + ' to shrink instruction size (And2Btr)', p);
  13260. taicpu(p).opcode := A_BTR;
  13261. taicpu(p).oper[0]^.val := BsrQWord(Value); { Essentially the base 2 logarithm }
  13262. Result := True;
  13263. Exit;
  13264. end;
  13265. end;
  13266. end;
  13267. function TX86AsmOptimizer.PostPeepholeOptMOVSX(var p : tai) : boolean;
  13268. begin
  13269. Result := False;
  13270. if not MatchOpType(taicpu(p), top_reg, top_reg) then
  13271. Exit;
  13272. { Convert:
  13273. movswl %ax,%eax -> cwtl
  13274. movslq %eax,%rax -> cdqe
  13275. NOTE: Don't convert movswl %al,%ax to cbw, because cbw and cwde
  13276. refer to the same opcode and depends only on the assembler's
  13277. current operand-size attribute. [Kit]
  13278. }
  13279. with taicpu(p) do
  13280. case opsize of
  13281. S_WL:
  13282. if (oper[0]^.reg = NR_AX) and (oper[1]^.reg = NR_EAX) then
  13283. begin
  13284. DebugMsg(SPeepholeOptimization + 'Converted movswl %ax,%eax to cwtl', p);
  13285. opcode := A_CWDE;
  13286. clearop(0);
  13287. clearop(1);
  13288. ops := 0;
  13289. Result := True;
  13290. end;
  13291. {$ifdef x86_64}
  13292. S_LQ:
  13293. if (oper[0]^.reg = NR_EAX) and (oper[1]^.reg = NR_RAX) then
  13294. begin
  13295. DebugMsg(SPeepholeOptimization + 'Converted movslq %eax,%rax to cltq', p);
  13296. opcode := A_CDQE;
  13297. clearop(0);
  13298. clearop(1);
  13299. ops := 0;
  13300. Result := True;
  13301. end;
  13302. {$endif x86_64}
  13303. else
  13304. ;
  13305. end;
  13306. end;
  13307. function TX86AsmOptimizer.PostPeepholeOptShr(var p : tai) : boolean;
  13308. var
  13309. hp1, hp2: tai;
  13310. IdentityMask, Shift: TCGInt;
  13311. LimitSize: Topsize;
  13312. DoNotMerge: Boolean;
  13313. begin
  13314. Result := False;
  13315. { All these optimisations work on "shr const,%reg" }
  13316. if not MatchOpType(taicpu(p), top_const, top_reg) then
  13317. Exit;
  13318. DoNotMerge := False;
  13319. Shift := taicpu(p).oper[0]^.val;
  13320. LimitSize := taicpu(p).opsize;
  13321. hp1 := p;
  13322. repeat
  13323. if not GetNextInstructionUsingReg(hp1, hp1, taicpu(p).oper[1]^.reg) or (hp1.typ <> ait_instruction) then
  13324. Break;
  13325. { Detect:
  13326. shr x, %reg
  13327. and y, %reg
  13328. If and y, %reg doesn't actually change the value of %reg (e.g. with
  13329. "shrl $24,%reg; andl $255,%reg", remove the AND instruction.
  13330. }
  13331. case taicpu(hp1).opcode of
  13332. A_AND:
  13333. if (taicpu(hp1).opsize = taicpu(p).opsize) and
  13334. MatchOpType(taicpu(hp1), top_const, top_reg) and
  13335. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  13336. begin
  13337. { Make sure the FLAGS register isn't in use }
  13338. TransferUsedRegs(TmpUsedRegs);
  13339. hp2 := p;
  13340. repeat
  13341. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  13342. until not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  13343. if not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs) then
  13344. begin
  13345. { Generate the identity mask }
  13346. case taicpu(p).opsize of
  13347. S_B:
  13348. IdentityMask := $FF shr Shift;
  13349. S_W:
  13350. IdentityMask := $FFFF shr Shift;
  13351. S_L:
  13352. IdentityMask := $FFFFFFFF shr Shift;
  13353. {$ifdef x86_64}
  13354. S_Q:
  13355. { We need to force the operands to be unsigned 64-bit
  13356. integers otherwise the wrong value is generated }
  13357. IdentityMask := TCGInt(QWord($FFFFFFFFFFFFFFFF) shr QWord(Shift));
  13358. {$endif x86_64}
  13359. else
  13360. InternalError(2022081501);
  13361. end;
  13362. if (taicpu(hp1).oper[0]^.val and IdentityMask) = IdentityMask then
  13363. begin
  13364. DebugMsg(SPeepholeOptimization + 'Removed AND instruction since previous SHR makes this an identity operation (ShrAnd2Shr)', hp1);
  13365. { All the possible 1 bits are covered, so we can remove the AND }
  13366. hp2 := tai(hp1.Previous);
  13367. RemoveInstruction(hp1);
  13368. { p wasn't actually changed, so don't set Result to True,
  13369. but a change was nonetheless made elsewhere }
  13370. Include(OptsToCheck, aoc_ForceNewIteration);
  13371. { Do another pass in case other AND or MOVZX instructions
  13372. follow }
  13373. hp1 := hp2;
  13374. Continue;
  13375. end;
  13376. end;
  13377. end;
  13378. A_TEST, A_CMP, A_Jcc:
  13379. { Skip over conditional jumps and relevant comparisons }
  13380. Continue;
  13381. A_MOVZX:
  13382. if MatchOpType(taicpu(hp1), top_reg, top_reg) and
  13383. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(p).oper[1]^.reg) then
  13384. begin
  13385. { Since the original register is being read as is, subsequent
  13386. SHRs must not be merged at this point }
  13387. DoNotMerge := True;
  13388. if IsShrMovZFoldable(taicpu(p).opsize, taicpu(hp1).opsize, Shift) then
  13389. begin
  13390. if SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg) then
  13391. begin
  13392. DebugMsg(SPeepholeOptimization + 'Removed MOVZX instruction since previous SHR makes it unnecessary (ShrMovz2Shr)', hp1);
  13393. { All the possible 1 bits are covered, so we can remove the AND }
  13394. hp2 := tai(hp1.Previous);
  13395. RemoveInstruction(hp1);
  13396. hp1 := hp2;
  13397. end
  13398. else { Different register target }
  13399. begin
  13400. DebugMsg(SPeepholeOptimization + 'Converted MOVZX instruction to MOV since previous SHR makes zero-extension unnecessary (ShrMovz2ShrMov 2)', hp1);
  13401. taicpu(hp1).opcode := A_MOV;
  13402. setsubreg(taicpu(hp1).oper[0]^.reg, getsubreg(taicpu(hp1).oper[1]^.reg));
  13403. case taicpu(hp1).opsize of
  13404. S_BW:
  13405. taicpu(hp1).opsize := S_W;
  13406. S_BL, S_WL:
  13407. taicpu(hp1).opsize := S_L;
  13408. else
  13409. InternalError(2022081503);
  13410. end;
  13411. end;
  13412. end
  13413. else if (Shift > 0) and
  13414. (taicpu(p).opsize = S_W) and
  13415. (taicpu(hp1).opsize = S_WL) and
  13416. (taicpu(hp1).oper[0]^.reg = NR_AX) and
  13417. (taicpu(hp1).oper[1]^.reg = NR_EAX) then
  13418. begin
  13419. { Detect:
  13420. shr x, %ax (x > 0)
  13421. ...
  13422. movzwl %ax,%eax
  13423. Change movzwl %ax,%eax to cwtl (shorter encoding for movswl %ax,%eax)
  13424. }
  13425. DebugMsg(SPeepholeOptimization + 'Converted movzwl %ax,%eax to cwtl (via ShrMovz2ShrCwtl)', hp1);
  13426. taicpu(hp1).opcode := A_CWDE;
  13427. taicpu(hp1).clearop(0);
  13428. taicpu(hp1).clearop(1);
  13429. taicpu(hp1).ops := 0;
  13430. end;
  13431. { Move onto the next instruction }
  13432. Continue;
  13433. end;
  13434. A_SHL, A_SAL, A_SHR:
  13435. if (taicpu(hp1).opsize <= LimitSize) and
  13436. MatchOpType(taicpu(hp1), top_const, top_reg) and
  13437. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[1]^.reg) then
  13438. begin
  13439. { Make sure the sizes don't exceed the register size limit
  13440. (measured by the shift value falling below the limit) }
  13441. if taicpu(hp1).opsize < LimitSize then
  13442. LimitSize := taicpu(hp1).opsize;
  13443. if taicpu(hp1).opcode = A_SHR then
  13444. Inc(Shift, taicpu(hp1).oper[0]^.val)
  13445. else
  13446. begin
  13447. Dec(Shift, taicpu(hp1).oper[0]^.val);
  13448. DoNotMerge := True;
  13449. end;
  13450. if Shift < topsize2memsize[taicpu(p).opsize] - topsize2memsize[LimitSize] then
  13451. Break;
  13452. { Since we've established that the combined shift is within
  13453. limits, we can actually combine the adjacent SHR
  13454. instructions even if they're different sizes }
  13455. if not DoNotMerge and (taicpu(hp1).opcode = A_SHR) then
  13456. begin
  13457. hp2 := tai(hp1.Previous);
  13458. DebugMsg(SPeepholeOptimization + 'ShrShr2Shr 2', p);
  13459. Inc(taicpu(p).oper[0]^.val, taicpu(hp1).oper[0]^.val);
  13460. RemoveInstruction(hp1);
  13461. hp1 := hp2;
  13462. end;
  13463. { Move onto the next instruction }
  13464. Continue;
  13465. end;
  13466. else
  13467. ;
  13468. end;
  13469. Break;
  13470. until False;
  13471. { Detect the following (looking backwards):
  13472. shr %cl,%reg
  13473. shr x, %reg
  13474. Swap the two SHR instructions to minimise a pipeline stall.
  13475. }
  13476. if GetLastInstruction(p, hp1) and
  13477. MatchInstruction(hp1, A_SHR, [taicpu(p).opsize]) and
  13478. MatchOpType(taicpu(hp1), top_reg, top_reg) and
  13479. { First operand will be %cl }
  13480. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  13481. { Just to be sure }
  13482. (getsupreg(taicpu(hp1).oper[1]^.reg) <> RS_ECX) then
  13483. begin
  13484. DebugMsg(SPeepholeOptimization + 'Swapped variable and constant SHR instructions to minimise pipeline stall (ShrShr2ShrShr)', hp1);
  13485. { Moving the entries this way ensures the register tracking remains correct }
  13486. Asml.Remove(p);
  13487. Asml.InsertBefore(p, hp1);
  13488. p := hp1;
  13489. { Don't set Result to True because the current instruction is now
  13490. "shr %cl,%reg" and there's nothing more we can do with it }
  13491. end;
  13492. end;
  13493. function TX86AsmOptimizer.PostPeepholeOptADDSUB(var p : tai) : boolean;
  13494. var
  13495. hp1, hp2: tai;
  13496. Opposite, SecondOpposite: TAsmOp;
  13497. NewCond: TAsmCond;
  13498. begin
  13499. Result := False;
  13500. { Change:
  13501. add/sub 128,(dest)
  13502. To:
  13503. sub/add -128,(dest)
  13504. This generaally takes fewer bytes to encode because -128 can be stored
  13505. in a signed byte, whereas +128 cannot.
  13506. }
  13507. if (taicpu(p).opsize <> S_B) and MatchOperand(taicpu(p).oper[0]^, 128) then
  13508. begin
  13509. if taicpu(p).opcode = A_ADD then
  13510. Opposite := A_SUB
  13511. else
  13512. Opposite := A_ADD;
  13513. { Be careful if the flags are in use, because the CF flag inverts
  13514. when changing from ADD to SUB and vice versa }
  13515. if RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  13516. GetNextInstruction(p, hp1) then
  13517. begin
  13518. TransferUsedRegs(TmpUsedRegs);
  13519. TmpUsedRegs[R_SPECIALREGISTER].Update(tai(p.Next), True);
  13520. hp2 := hp1;
  13521. { Scan ahead to check if everything's safe }
  13522. while Assigned(hp1) and RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) do
  13523. begin
  13524. if (hp1.typ <> ait_instruction) then
  13525. { Probably unsafe since the flags are still in use }
  13526. Exit;
  13527. if MatchInstruction(hp1, A_CALL, A_JMP, A_RET, []) then
  13528. { Stop searching at an unconditional jump }
  13529. Break;
  13530. if not
  13531. (
  13532. MatchInstruction(hp1, A_ADC, A_SBB, []) and
  13533. (taicpu(hp1).oper[0]^.typ = top_const) { We need to be able to invert a constant }
  13534. ) and
  13535. (taicpu(hp1).condition = C_None) and RegInInstruction(NR_DEFAULTFLAGS, hp1) then
  13536. { Instruction depends on FLAGS (and is not ADC or SBB); break out }
  13537. Exit;
  13538. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  13539. TmpUsedRegs[R_SPECIALREGISTER].Update(tai(hp1.Next), True);
  13540. { Move to the next instruction }
  13541. GetNextInstruction(hp1, hp1);
  13542. end;
  13543. while Assigned(hp2) and (hp2 <> hp1) do
  13544. begin
  13545. NewCond := C_None;
  13546. case taicpu(hp2).condition of
  13547. C_A, C_NBE:
  13548. NewCond := C_BE;
  13549. C_B, C_C, C_NAE:
  13550. NewCond := C_AE;
  13551. C_AE, C_NB, C_NC:
  13552. NewCond := C_B;
  13553. C_BE, C_NA:
  13554. NewCond := C_A;
  13555. else
  13556. { No change needed };
  13557. end;
  13558. if NewCond <> C_None then
  13559. begin
  13560. DebugMsg(SPeepholeOptimization + 'Condition changed from ' + cond2str[taicpu(hp2).condition] + ' to ' + cond2str[NewCond] +
  13561. ' to accommodate ' + debug_op2str(taicpu(p).opcode) + ' -> ' + debug_op2str(opposite) + ' above', hp2);
  13562. taicpu(hp2).condition := NewCond;
  13563. end
  13564. else
  13565. if MatchInstruction(hp2, A_ADC, A_SBB, []) then
  13566. begin
  13567. { Because of the flipping of the carry bit, to ensure
  13568. the operation remains equivalent, ADC becomes SBB
  13569. and vice versa, and the constant is not-inverted.
  13570. If multiple ADCs or SBBs appear in a row, each one
  13571. changed causes the carry bit to invert, so they all
  13572. need to be flipped }
  13573. if taicpu(hp2).opcode = A_ADC then
  13574. SecondOpposite := A_SBB
  13575. else
  13576. SecondOpposite := A_ADC;
  13577. if taicpu(hp2).oper[0]^.typ <> top_const then
  13578. { Should have broken out of this optimisation already }
  13579. InternalError(2021112901);
  13580. DebugMsg(SPeepholeOptimization + debug_op2str(taicpu(hp2).opcode) + debug_opsize2str(taicpu(hp2).opsize) + ' $' + debug_tostr(taicpu(hp2).oper[0]^.val) + ',' + debug_operstr(taicpu(hp2).oper[1]^) + ' -> ' +
  13581. debug_op2str(SecondOpposite) + debug_opsize2str(taicpu(hp2).opsize) + ' $' + debug_tostr(not taicpu(hp2).oper[0]^.val) + ',' + debug_operstr(taicpu(hp2).oper[1]^) + ' to accommodate inverted carry bit', hp2);
  13582. { Bit-invert the constant (effectively equivalent to "-1 - val") }
  13583. taicpu(hp2).opcode := SecondOpposite;
  13584. taicpu(hp2).oper[0]^.val := not taicpu(hp2).oper[0]^.val;
  13585. end;
  13586. { Move to the next instruction }
  13587. GetNextInstruction(hp2, hp2);
  13588. end;
  13589. if (hp2 <> hp1) then
  13590. InternalError(2021111501);
  13591. end;
  13592. DebugMsg(SPeepholeOptimization + debug_op2str(taicpu(p).opcode) + debug_opsize2str(taicpu(p).opsize) + ' $128,' + debug_operstr(taicpu(p).oper[1]^) + ' changed to ' +
  13593. debug_op2str(opposite) + debug_opsize2str(taicpu(p).opsize) + ' $-128,' + debug_operstr(taicpu(p).oper[1]^) + ' to reduce instruction size', p);
  13594. taicpu(p).opcode := Opposite;
  13595. taicpu(p).oper[0]^.val := -128;
  13596. { No further optimisations can be made on this instruction, so move
  13597. onto the next one to save time }
  13598. p := tai(p.Next);
  13599. UpdateUsedRegs(p);
  13600. Result := True;
  13601. Exit;
  13602. end;
  13603. { Detect:
  13604. add/sub %reg2,(dest)
  13605. add/sub x, (dest)
  13606. (dest can be a register or a reference)
  13607. Swap the instructions to minimise a pipeline stall. This reverses the
  13608. "Add swap" and "Sub swap" optimisations done in pass 1 if no new
  13609. optimisations could be made.
  13610. }
  13611. if (taicpu(p).oper[0]^.typ = top_reg) and
  13612. not RegInOp(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^) and
  13613. (
  13614. (
  13615. (taicpu(p).oper[1]^.typ = top_reg) and
  13616. { We can try searching further ahead if we're writing to a register }
  13617. GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[1]^.reg)
  13618. ) or
  13619. (
  13620. (taicpu(p).oper[1]^.typ = top_ref) and
  13621. GetNextInstruction(p, hp1)
  13622. )
  13623. ) and
  13624. MatchInstruction(hp1, A_ADD, A_SUB, [taicpu(p).opsize]) and
  13625. (taicpu(hp1).oper[0]^.typ = top_const) and
  13626. MatchOperand(taicpu(p).oper[1]^, taicpu(hp1).oper[1]^) then
  13627. begin
  13628. { Make doubly sure the flags aren't in use because the order of additions may affect them }
  13629. TransferUsedRegs(TmpUsedRegs);
  13630. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  13631. hp2 := p;
  13632. while not (cs_opt_level3 in current_settings.optimizerswitches) and
  13633. GetNextInstruction(hp2, hp2) and (hp2 <> hp1) do
  13634. UpdateUsedRegs(TmpUsedRegs, tai(hp2.next));
  13635. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  13636. begin
  13637. asml.remove(hp1);
  13638. asml.InsertBefore(hp1, p);
  13639. DebugMsg(SPeepholeOptimization + 'Add/Sub swap 2 done', hp1);
  13640. Result := True;
  13641. end;
  13642. end;
  13643. end;
  13644. function TX86AsmOptimizer.PostPeepholeOptCmp(var p : tai) : Boolean;
  13645. begin
  13646. Result:=false;
  13647. { change "cmp $0, %reg" to "test %reg, %reg" }
  13648. if MatchOpType(taicpu(p),top_const,top_reg) and
  13649. (taicpu(p).oper[0]^.val = 0) then
  13650. begin
  13651. taicpu(p).opcode := A_TEST;
  13652. taicpu(p).loadreg(0,taicpu(p).oper[1]^.reg);
  13653. DebugMsg(SPeepholeOptimization + 'Cmp2Test', p);
  13654. Result:=true;
  13655. end;
  13656. end;
  13657. function TX86AsmOptimizer.PostPeepholeOptTestOr(var p : tai) : Boolean;
  13658. var
  13659. IsTestConstX, IsValid : Boolean;
  13660. hp1,hp2 : tai;
  13661. begin
  13662. Result:=false;
  13663. { If x is a power of 2 (popcnt = 1), change:
  13664. or $x, %reg/ref
  13665. To:
  13666. bts lb(x), %reg/ref
  13667. }
  13668. if (taicpu(p).opcode = A_OR) and
  13669. IsBTXAcceptable(p) and
  13670. { IsBTXAcceptable checks to see if oper[0] is an immediate }
  13671. (PopCnt(QWord(taicpu(p).oper[0]^.val)) = 1) and
  13672. (
  13673. { Don't optimise if a test instruction follows }
  13674. not GetNextInstruction(p, hp1) or
  13675. not MatchInstruction(hp1, A_TEST, [taicpu(p).opsize])
  13676. ) then
  13677. begin
  13678. DebugMsg(SPeepholeOptimization + 'Changed OR $0x' + hexstr(taicpu(p).oper[0]^.val, 2) + ' to BTS ' + debug_tostr(BsrQWord(taicpu(p).oper[0]^.val)) + ' to shrink instruction size (Or2Bts)', p);
  13679. taicpu(p).opcode := A_BTS;
  13680. taicpu(p).oper[0]^.val := BsrQWord(taicpu(p).oper[0]^.val); { Essentially the base 2 logarithm }
  13681. Result := True;
  13682. Exit;
  13683. end;
  13684. { If x is a power of 2 (popcnt = 1), change:
  13685. test $x, %reg/ref
  13686. je / sete / cmove (or jne / setne)
  13687. To:
  13688. bt lb(x), %reg/ref
  13689. jnc / setnc / cmovnc (or jc / setc / cmovnc)
  13690. }
  13691. if (taicpu(p).opcode = A_TEST) and
  13692. (CPUX86_HAS_BTX in cpu_capabilities[current_settings.optimizecputype]) and
  13693. (taicpu(p).oper[0]^.typ = top_const) and
  13694. (
  13695. (cs_opt_size in current_settings.optimizerswitches) or
  13696. (
  13697. (taicpu(p).oper[1]^.typ = top_reg) and
  13698. (CPUX86_HINT_FAST_BT_REG_IMM in cpu_optimization_hints[current_settings.optimizecputype])
  13699. ) or
  13700. (
  13701. (taicpu(p).oper[1]^.typ <> top_reg) and
  13702. (CPUX86_HINT_FAST_BT_MEM_IMM in cpu_optimization_hints[current_settings.optimizecputype])
  13703. )
  13704. ) and
  13705. (PopCnt(QWord(taicpu(p).oper[0]^.val)) = 1) and
  13706. { For sizes less than S_L, the byte size is equal or larger with BT,
  13707. so don't bother optimising }
  13708. (taicpu(p).opsize >= S_L) then
  13709. begin
  13710. IsValid := True;
  13711. { Check the next set of instructions, watching the FLAGS register
  13712. and the conditions used }
  13713. TransferUsedRegs(TmpUsedRegs);
  13714. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  13715. hp1 := p;
  13716. hp2 := nil;
  13717. while GetNextInstruction(hp1, hp1) do
  13718. begin
  13719. if not Assigned(hp2) then
  13720. { The first instruction after TEST }
  13721. hp2 := hp1;
  13722. if (hp1.typ <> ait_instruction) then
  13723. begin
  13724. { If the flags are no longer in use, everything is fine }
  13725. if RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  13726. IsValid := False;
  13727. Break;
  13728. end;
  13729. case taicpu(hp1).condition of
  13730. C_None:
  13731. begin
  13732. if RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  13733. { Something is not quite normal, so play safe and don't change }
  13734. IsValid := False;
  13735. Break;
  13736. end;
  13737. C_E, C_Z, C_NE, C_NZ:
  13738. { This is fine };
  13739. else
  13740. begin
  13741. { Unsupported condition }
  13742. IsValid := False;
  13743. Break;
  13744. end;
  13745. end;
  13746. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  13747. end;
  13748. if IsValid then
  13749. begin
  13750. while hp2 <> hp1 do
  13751. begin
  13752. case taicpu(hp2).condition of
  13753. C_Z, C_E:
  13754. taicpu(hp2).condition := C_NC;
  13755. C_NZ, C_NE:
  13756. taicpu(hp2).condition := C_C;
  13757. else
  13758. { Should not get this by this point }
  13759. InternalError(2022110701);
  13760. end;
  13761. GetNextInstruction(hp2, hp2);
  13762. end;
  13763. DebugMsg(SPeepholeOptimization + 'Changed TEST $0x' + hexstr(taicpu(p).oper[0]^.val, 2) + ' to BT ' + debug_tostr(BsrQWord(taicpu(p).oper[0]^.val)) + ' to shrink instruction size (Test2Bt)', p);
  13764. taicpu(p).opcode := A_BT;
  13765. taicpu(p).oper[0]^.val := BsrQWord(taicpu(p).oper[0]^.val); { Essentially the base 2 logarithm }
  13766. Result := True;
  13767. Exit;
  13768. end;
  13769. end;
  13770. { removes the line marked with (x) from the sequence
  13771. and/or/xor/add/sub/... $x, %y
  13772. test/or %y, %y | test $-1, %y (x)
  13773. j(n)z _Label
  13774. as the first instruction already adjusts the ZF
  13775. %y operand may also be a reference }
  13776. IsTestConstX:=(taicpu(p).opcode=A_TEST) and
  13777. MatchOperand(taicpu(p).oper[0]^,-1);
  13778. if (OpsEqual(taicpu(p).oper[0]^,taicpu(p).oper[1]^) or IsTestConstX) and
  13779. GetLastInstruction(p, hp1) and
  13780. (tai(hp1).typ = ait_instruction) and
  13781. GetNextInstruction(p,hp2) and
  13782. MatchInstruction(hp2,A_SETcc,A_Jcc,A_CMOVcc,[]) then
  13783. case taicpu(hp1).opcode Of
  13784. A_ADD, A_SUB, A_OR, A_XOR, A_AND,
  13785. { These two instructions set the zero flag if the result is zero }
  13786. A_POPCNT, A_LZCNT:
  13787. begin
  13788. if (
  13789. { With POPCNT, an input of zero will set the zero flag
  13790. because the population count of zero is zero }
  13791. (taicpu(hp1).opcode = A_POPCNT) and
  13792. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) and
  13793. (
  13794. OpsEqual(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^) or
  13795. { Faster than going through the second half of the 'or'
  13796. condition below }
  13797. OpsEqual(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^)
  13798. )
  13799. ) or (
  13800. OpsEqual(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^) and
  13801. { does not work in case of overflow for G(E)/L(E)/C_O/C_NO }
  13802. { and in case of carry for A(E)/B(E)/C/NC }
  13803. (
  13804. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) or
  13805. (
  13806. (taicpu(hp1).opcode <> A_ADD) and
  13807. (taicpu(hp1).opcode <> A_SUB) and
  13808. (taicpu(hp1).opcode <> A_LZCNT)
  13809. )
  13810. )
  13811. ) then
  13812. begin
  13813. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (2-op) done', hp1);
  13814. RemoveCurrentP(p, hp2);
  13815. Result:=true;
  13816. Exit;
  13817. end;
  13818. end;
  13819. A_SHL, A_SAL, A_SHR, A_SAR:
  13820. begin
  13821. if OpsEqual(taicpu(hp1).oper[1]^,taicpu(p).oper[1]^) and
  13822. { SHL/SAL/SHR/SAR with a value of 0 do not change the flags }
  13823. { therefore, it's only safe to do this optimization for }
  13824. { shifts by a (nonzero) constant }
  13825. (taicpu(hp1).oper[0]^.typ = top_const) and
  13826. (taicpu(hp1).oper[0]^.val <> 0) and
  13827. { does not work in case of overflow for G(E)/L(E)/C_O/C_NO }
  13828. { and in case of carry for A(E)/B(E)/C/NC }
  13829. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) then
  13830. begin
  13831. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (shift) done', hp1);
  13832. RemoveCurrentP(p, hp2);
  13833. Result:=true;
  13834. Exit;
  13835. end;
  13836. end;
  13837. A_DEC, A_INC, A_NEG:
  13838. begin
  13839. if OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) and
  13840. { does not work in case of overflow for G(E)/L(E)/C_O/C_NO }
  13841. { and in case of carry for A(E)/B(E)/C/NC }
  13842. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) then
  13843. begin
  13844. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (1-op) done', hp1);
  13845. RemoveCurrentP(p, hp2);
  13846. Result:=true;
  13847. Exit;
  13848. end;
  13849. end;
  13850. A_ANDN, A_BZHI:
  13851. begin
  13852. if OpsEqual(taicpu(hp1).oper[2]^,taicpu(p).oper[1]^) and
  13853. { Only the zero and sign flags are consistent with what the result is }
  13854. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE,C_S,C_NS]) then
  13855. begin
  13856. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (ANDN/BZHI) done', hp1);
  13857. RemoveCurrentP(p, hp2);
  13858. Result:=true;
  13859. Exit;
  13860. end;
  13861. end;
  13862. A_BEXTR:
  13863. begin
  13864. if OpsEqual(taicpu(hp1).oper[2]^,taicpu(p).oper[1]^) and
  13865. { Only the zero flag is set }
  13866. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) then
  13867. begin
  13868. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (BEXTR) done', hp1);
  13869. RemoveCurrentP(p, hp2);
  13870. Result:=true;
  13871. Exit;
  13872. end;
  13873. end;
  13874. else
  13875. ;
  13876. end; { case }
  13877. { change "test $-1,%reg" into "test %reg,%reg" }
  13878. if IsTestConstX and (taicpu(p).oper[1]^.typ=top_reg) then
  13879. taicpu(p).loadoper(0,taicpu(p).oper[1]^);
  13880. { Change "or %reg,%reg" to "test %reg,%reg" as OR generates a false dependency }
  13881. if MatchInstruction(p, A_OR, []) and
  13882. { Can only match if they're both registers }
  13883. MatchOperand(taicpu(p).oper[0]^, taicpu(p).oper[1]^) then
  13884. begin
  13885. DebugMsg(SPeepholeOptimization + 'or %reg,%reg -> test %reg,%reg to remove false dependency (Or2Test)', p);
  13886. taicpu(p).opcode := A_TEST;
  13887. { No need to set Result to True, as we've done all the optimisations we can }
  13888. end;
  13889. end;
  13890. function TX86AsmOptimizer.PostPeepholeOptCall(var p : tai) : Boolean;
  13891. var
  13892. hp1,hp3 : tai;
  13893. {$ifndef x86_64}
  13894. hp2 : taicpu;
  13895. {$endif x86_64}
  13896. begin
  13897. Result:=false;
  13898. hp3:=nil;
  13899. {$ifndef x86_64}
  13900. { don't do this on modern CPUs, this really hurts them due to
  13901. broken call/ret pairing }
  13902. if (current_settings.optimizecputype < cpu_Pentium2) and
  13903. not(cs_create_pic in current_settings.moduleswitches) and
  13904. GetNextInstruction(p, hp1) and
  13905. MatchInstruction(hp1,A_JMP,[S_NO]) and
  13906. MatchOpType(taicpu(hp1),top_ref) and
  13907. (taicpu(hp1).oper[0]^.ref^.refaddr=addr_full) then
  13908. begin
  13909. hp2 := taicpu.Op_sym(A_PUSH,S_L,taicpu(hp1).oper[0]^.ref^.symbol);
  13910. taicpu(hp2).fileinfo := taicpu(p).fileinfo;
  13911. InsertLLItem(p.previous, p, hp2);
  13912. taicpu(p).opcode := A_JMP;
  13913. taicpu(p).is_jmp := true;
  13914. RemoveInstruction(hp1);
  13915. Result:=true;
  13916. end
  13917. else
  13918. {$endif x86_64}
  13919. { replace
  13920. call procname
  13921. ret
  13922. by
  13923. jmp procname
  13924. but do it only on level 4 because it destroys stack back traces
  13925. else if the subroutine is marked as no return, remove the ret
  13926. }
  13927. if ((cs_opt_level4 in current_settings.optimizerswitches) or
  13928. (po_noreturn in current_procinfo.procdef.procoptions)) and
  13929. GetNextInstruction(p, hp1) and
  13930. (MatchInstruction(hp1,A_RET,[S_NO]) or
  13931. (MatchInstruction(hp1,A_VZEROUPPER,[S_NO]) and
  13932. SetAndTest(hp1,hp3) and
  13933. GetNextInstruction(hp1,hp1) and
  13934. MatchInstruction(hp1,A_RET,[S_NO])
  13935. )
  13936. ) and
  13937. (taicpu(hp1).ops=0) then
  13938. begin
  13939. if (cs_opt_level4 in current_settings.optimizerswitches) and
  13940. { we might destroy stack alignment here if we do not do a call }
  13941. (target_info.stackalign<=sizeof(SizeUInt)) then
  13942. begin
  13943. taicpu(p).opcode := A_JMP;
  13944. taicpu(p).is_jmp := true;
  13945. DebugMsg(SPeepholeOptimization + 'CallRet2Jmp done',p);
  13946. end
  13947. else
  13948. DebugMsg(SPeepholeOptimization + 'CallRet2Call done',p);
  13949. RemoveInstruction(hp1);
  13950. if Assigned(hp3) then
  13951. begin
  13952. AsmL.Remove(hp3);
  13953. AsmL.InsertBefore(hp3,p)
  13954. end;
  13955. Result:=true;
  13956. end;
  13957. end;
  13958. function TX86AsmOptimizer.PostPeepholeOptMovzx(var p : tai) : Boolean;
  13959. function ConstInRange(const Val: TCGInt; const OpSize: TOpSize): Boolean;
  13960. begin
  13961. case OpSize of
  13962. S_B, S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  13963. Result := (Val <= $FF) and (Val >= -128);
  13964. S_W, S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  13965. Result := (Val <= $FFFF) and (Val >= -32768);
  13966. S_L{$ifdef x86_64}, S_LQ{$endif x86_64}:
  13967. Result := (Val <= $FFFFFFFF) and (Val >= -2147483648);
  13968. else
  13969. Result := True;
  13970. end;
  13971. end;
  13972. var
  13973. hp1, hp2 : tai;
  13974. SizeChange: Boolean;
  13975. PreMessage: string;
  13976. begin
  13977. Result := False;
  13978. if (taicpu(p).oper[0]^.typ = top_reg) and
  13979. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  13980. GetNextInstruction(p, hp1) and (hp1.typ = ait_instruction) then
  13981. begin
  13982. { Change (using movzbl %al,%eax as an example):
  13983. movzbl %al, %eax movzbl %al, %eax
  13984. cmpl x, %eax testl %eax,%eax
  13985. To:
  13986. cmpb x, %al testb %al, %al (Move one back to avoid a false dependency)
  13987. movzbl %al, %eax movzbl %al, %eax
  13988. Smaller instruction and minimises pipeline stall as the CPU
  13989. doesn't have to wait for the register to get zero-extended. [Kit]
  13990. Also allow if the smaller of the two registers is being checked,
  13991. as this still removes the false dependency.
  13992. }
  13993. if
  13994. (
  13995. (
  13996. (taicpu(hp1).opcode = A_CMP) and MatchOpType(taicpu(hp1), top_const, top_reg) and
  13997. ConstInRange(taicpu(hp1).oper[0]^.val, taicpu(p).opsize)
  13998. ) or (
  13999. { If MatchOperand returns True, they must both be registers }
  14000. (taicpu(hp1).opcode = A_TEST) and MatchOperand(taicpu(hp1).oper[0]^, taicpu(hp1).oper[1]^)
  14001. )
  14002. ) and
  14003. (reg2opsize(taicpu(hp1).oper[1]^.reg) <= reg2opsize(taicpu(p).oper[1]^.reg)) and
  14004. SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) then
  14005. begin
  14006. PreMessage := debug_op2str(taicpu(hp1).opcode) + debug_opsize2str(taicpu(hp1).opsize) + ' ' + debug_operstr(taicpu(hp1).oper[0]^) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' -> ' + debug_op2str(taicpu(hp1).opcode);
  14007. asml.Remove(hp1);
  14008. asml.InsertBefore(hp1, p);
  14009. { Swap instructions in the case of cmp 0,%reg or test %reg,%reg }
  14010. if (taicpu(hp1).opcode = A_TEST) or (taicpu(hp1).oper[0]^.val = 0) then
  14011. begin
  14012. taicpu(hp1).opcode := A_TEST;
  14013. taicpu(hp1).loadreg(0, taicpu(p).oper[0]^.reg);
  14014. end;
  14015. taicpu(hp1).oper[1]^.reg := taicpu(p).oper[0]^.reg;
  14016. case taicpu(p).opsize of
  14017. S_BW, S_BL:
  14018. begin
  14019. SizeChange := taicpu(hp1).opsize <> S_B;
  14020. taicpu(hp1).changeopsize(S_B);
  14021. end;
  14022. S_WL:
  14023. begin
  14024. SizeChange := taicpu(hp1).opsize <> S_W;
  14025. taicpu(hp1).changeopsize(S_W);
  14026. end
  14027. else
  14028. InternalError(2020112701);
  14029. end;
  14030. UpdateUsedRegs(tai(p.Next));
  14031. { Check if the register is used aferwards - if not, we can
  14032. remove the movzx instruction completely }
  14033. if not RegUsedAfterInstruction(taicpu(hp1).oper[1]^.reg, p, UsedRegs) then
  14034. begin
  14035. { Hp1 is a better position than p for debugging purposes }
  14036. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 4a', hp1);
  14037. RemoveCurrentp(p, hp1);
  14038. Result := True;
  14039. end;
  14040. if SizeChange then
  14041. DebugMsg(SPeepholeOptimization + PreMessage +
  14042. debug_opsize2str(taicpu(hp1).opsize) + ' ' + debug_operstr(taicpu(hp1).oper[0]^) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' (smaller and minimises pipeline stall - MovzxCmp2CmpMovzx)', hp1)
  14043. else
  14044. DebugMsg(SPeepholeOptimization + 'MovzxCmp2CmpMovzx', hp1);
  14045. Exit;
  14046. end;
  14047. { Change (using movzwl %ax,%eax as an example):
  14048. movzwl %ax, %eax
  14049. movb %al, (dest) (Register is smaller than read register in movz)
  14050. To:
  14051. movb %al, (dest) (Move one back to avoid a false dependency)
  14052. movzwl %ax, %eax
  14053. }
  14054. if (taicpu(hp1).opcode = A_MOV) and
  14055. (taicpu(hp1).oper[0]^.typ = top_reg) and
  14056. not RegInOp(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^) and
  14057. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(p).oper[0]^.reg) and
  14058. (reg2opsize(taicpu(hp1).oper[0]^.reg) <= reg2opsize(taicpu(p).oper[0]^.reg)) then
  14059. begin
  14060. DebugMsg(SPeepholeOptimization + 'MovzxMov2MovMovzx', hp1);
  14061. hp2 := tai(hp1.Previous); { Effectively the old position of hp1 }
  14062. asml.Remove(hp1);
  14063. asml.InsertBefore(hp1, p);
  14064. if taicpu(hp1).oper[1]^.typ = top_reg then
  14065. AllocRegBetween(taicpu(hp1).oper[1]^.reg, hp1, hp2, UsedRegs);
  14066. { Check if the register is used aferwards - if not, we can
  14067. remove the movzx instruction completely }
  14068. if not RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg, p, UsedRegs) then
  14069. begin
  14070. { Hp1 is a better position than p for debugging purposes }
  14071. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 4b', hp1);
  14072. RemoveCurrentp(p, hp1);
  14073. Result := True;
  14074. end;
  14075. Exit;
  14076. end;
  14077. end;
  14078. end;
  14079. function TX86AsmOptimizer.PostPeepholeOptXor(var p : tai) : Boolean;
  14080. var
  14081. hp1: tai;
  14082. {$ifdef x86_64}
  14083. PreMessage, RegName: string;
  14084. {$endif x86_64}
  14085. begin
  14086. Result := False;
  14087. { If x is a power of 2 (popcnt = 1), change:
  14088. xor $x, %reg/ref
  14089. To:
  14090. btc lb(x), %reg/ref
  14091. }
  14092. if IsBTXAcceptable(p) and
  14093. { IsBTXAcceptable checks to see if oper[0] is an immediate }
  14094. (PopCnt(QWord(taicpu(p).oper[0]^.val)) = 1) and
  14095. (
  14096. { Don't optimise if a test instruction follows }
  14097. not GetNextInstruction(p, hp1) or
  14098. not MatchInstruction(hp1, A_TEST, [taicpu(p).opsize])
  14099. ) then
  14100. begin
  14101. DebugMsg(SPeepholeOptimization + 'Changed XOR $0x' + hexstr(taicpu(p).oper[0]^.val, 2) + ' to BTC ' + debug_tostr(BsrQWord(taicpu(p).oper[0]^.val)) + ' to shrink instruction size (Xor2Btc)', p);
  14102. taicpu(p).opcode := A_BTC;
  14103. taicpu(p).oper[0]^.val := BsrQWord(taicpu(p).oper[0]^.val); { Essentially the base 2 logarithm }
  14104. Result := True;
  14105. Exit;
  14106. end;
  14107. {$ifdef x86_64}
  14108. { Code size reduction by J. Gareth "Kit" Moreton }
  14109. { change "xorq %reg,%reg" to "xorl %reg,%reg" for %rax, %rcx, %rdx, %rbx, %rsi, %rdi, %rbp and %rsp,
  14110. as this removes the REX prefix }
  14111. if not OpsEqual(taicpu(p).oper[0]^,taicpu(p).oper[1]^) then
  14112. Exit;
  14113. if taicpu(p).oper[0]^.typ <> top_reg then
  14114. { Should be impossible if both operands were equal, since one of XOR's operands must be a register }
  14115. InternalError(2018011500);
  14116. case taicpu(p).opsize of
  14117. S_Q:
  14118. begin
  14119. RegName := debug_regname(taicpu(p).oper[0]^.reg); { 64-bit register name }
  14120. PreMessage := 'xorq ' + RegName + ',' + RegName + ' -> xorl ';
  14121. { The actual optimization }
  14122. setsubreg(taicpu(p).oper[0]^.reg, R_SUBD);
  14123. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  14124. taicpu(p).changeopsize(S_L);
  14125. RegName := debug_regname(taicpu(p).oper[0]^.reg); { 32-bit register name }
  14126. DebugMsg(SPeepholeOptimization + PreMessage + RegName + ',' + RegName + ' (32-bit register recommended when zeroing 64-bit counterpart)', p);
  14127. end;
  14128. else
  14129. ;
  14130. end;
  14131. {$endif x86_64}
  14132. end;
  14133. function TX86AsmOptimizer.PostPeepholeOptVPXOR(var p : tai) : Boolean;
  14134. var
  14135. XReg: TRegister;
  14136. begin
  14137. Result := False;
  14138. { Turn "vpxor %ymmreg2,%ymmreg2,%ymmreg1" to "vpxor %xmmreg2,%xmmreg2,%xmmreg1"
  14139. Smaller encoding and slightly faster on some platforms (also works for
  14140. ZMM-sized registers) }
  14141. if (taicpu(p).opsize in [S_YMM, S_ZMM]) and
  14142. MatchOpType(taicpu(p), top_reg, top_reg, top_reg) then
  14143. begin
  14144. XReg := taicpu(p).oper[0]^.reg;
  14145. if (taicpu(p).oper[1]^.reg = XReg) then
  14146. begin
  14147. taicpu(p).changeopsize(S_XMM);
  14148. setsubreg(taicpu(p).oper[2]^.reg, R_SUBMMX);
  14149. if (cs_opt_size in current_settings.optimizerswitches) then
  14150. begin
  14151. { Change input registers to %xmm0 to reduce size. Note that
  14152. there's a risk of a false dependency doing this, so only
  14153. optimise for size here }
  14154. XReg := NR_XMM0;
  14155. DebugMsg(SPeepholeOptimization + 'Changed zero-setting vpxor from Y/ZMM to XMM and changed input registers to %xmm0 to reduce size', p);
  14156. end
  14157. else
  14158. begin
  14159. setsubreg(XReg, R_SUBMMX);
  14160. DebugMsg(SPeepholeOptimization + 'Changed zero-setting vpxor from Y/ZMM to XMM to reduce size and increase efficiency', p);
  14161. end;
  14162. taicpu(p).oper[0]^.reg := XReg;
  14163. taicpu(p).oper[1]^.reg := XReg;
  14164. Result := True;
  14165. end;
  14166. end;
  14167. end;
  14168. class procedure TX86AsmOptimizer.OptimizeRefs(var p: taicpu);
  14169. var
  14170. OperIdx: Integer;
  14171. begin
  14172. for OperIdx := 0 to p.ops - 1 do
  14173. if p.oper[OperIdx]^.typ = top_ref then
  14174. optimize_ref(p.oper[OperIdx]^.ref^, False);
  14175. end;
  14176. end.