aoptx86.pas 773 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825108261082710828108291083010831108321083310834108351083610837108381083910840108411084210843108441084510846108471084810849108501085110852108531085410855108561085710858108591086010861108621086310864108651086610867108681086910870108711087210873108741087510876108771087810879108801088110882108831088410885108861088710888108891089010891108921089310894108951089610897108981089910900109011090210903109041090510906109071090810909109101091110912109131091410915109161091710918109191092010921109221092310924109251092610927109281092910930109311093210933109341093510936109371093810939109401094110942109431094410945109461094710948109491095010951109521095310954109551095610957109581095910960109611096210963109641096510966109671096810969109701097110972109731097410975109761097710978109791098010981109821098310984109851098610987109881098910990109911099210993109941099510996109971099810999110001100111002110031100411005110061100711008110091101011011110121101311014110151101611017110181101911020110211102211023110241102511026110271102811029110301103111032110331103411035110361103711038110391104011041110421104311044110451104611047110481104911050110511105211053110541105511056110571105811059110601106111062110631106411065110661106711068110691107011071110721107311074110751107611077110781107911080110811108211083110841108511086110871108811089110901109111092110931109411095110961109711098110991110011101111021110311104111051110611107111081110911110111111111211113111141111511116111171111811119111201112111122111231112411125111261112711128111291113011131111321113311134111351113611137111381113911140111411114211143111441114511146111471114811149111501115111152111531115411155111561115711158111591116011161111621116311164111651116611167111681116911170111711117211173111741117511176111771117811179111801118111182111831118411185111861118711188111891119011191111921119311194111951119611197111981119911200112011120211203112041120511206112071120811209112101121111212112131121411215112161121711218112191122011221112221122311224112251122611227112281122911230112311123211233112341123511236112371123811239112401124111242112431124411245112461124711248112491125011251112521125311254112551125611257112581125911260112611126211263112641126511266112671126811269112701127111272112731127411275112761127711278112791128011281112821128311284112851128611287112881128911290112911129211293112941129511296112971129811299113001130111302113031130411305113061130711308113091131011311113121131311314113151131611317113181131911320113211132211323113241132511326113271132811329113301133111332113331133411335113361133711338113391134011341113421134311344113451134611347113481134911350113511135211353113541135511356113571135811359113601136111362113631136411365113661136711368113691137011371113721137311374113751137611377113781137911380113811138211383113841138511386113871138811389113901139111392113931139411395113961139711398113991140011401114021140311404114051140611407114081140911410114111141211413114141141511416114171141811419114201142111422114231142411425114261142711428114291143011431114321143311434114351143611437114381143911440114411144211443114441144511446114471144811449114501145111452114531145411455114561145711458114591146011461114621146311464114651146611467114681146911470114711147211473114741147511476114771147811479114801148111482114831148411485114861148711488114891149011491114921149311494114951149611497114981149911500115011150211503115041150511506115071150811509115101151111512115131151411515115161151711518115191152011521115221152311524115251152611527115281152911530115311153211533115341153511536115371153811539115401154111542115431154411545115461154711548115491155011551115521155311554115551155611557115581155911560115611156211563115641156511566115671156811569115701157111572115731157411575115761157711578115791158011581115821158311584115851158611587115881158911590115911159211593115941159511596115971159811599116001160111602116031160411605116061160711608116091161011611116121161311614116151161611617116181161911620116211162211623116241162511626116271162811629116301163111632116331163411635116361163711638116391164011641116421164311644116451164611647116481164911650116511165211653116541165511656116571165811659116601166111662116631166411665116661166711668116691167011671116721167311674116751167611677116781167911680116811168211683116841168511686116871168811689116901169111692116931169411695116961169711698116991170011701117021170311704117051170611707117081170911710117111171211713117141171511716117171171811719117201172111722117231172411725117261172711728117291173011731117321173311734117351173611737117381173911740117411174211743117441174511746117471174811749117501175111752117531175411755117561175711758117591176011761117621176311764117651176611767117681176911770117711177211773117741177511776117771177811779117801178111782117831178411785117861178711788117891179011791117921179311794117951179611797117981179911800118011180211803118041180511806118071180811809118101181111812118131181411815118161181711818118191182011821118221182311824118251182611827118281182911830118311183211833118341183511836118371183811839118401184111842118431184411845118461184711848118491185011851118521185311854118551185611857118581185911860118611186211863118641186511866118671186811869118701187111872118731187411875118761187711878118791188011881118821188311884118851188611887118881188911890118911189211893118941189511896118971189811899119001190111902119031190411905119061190711908119091191011911119121191311914119151191611917119181191911920119211192211923119241192511926119271192811929119301193111932119331193411935119361193711938119391194011941119421194311944119451194611947119481194911950119511195211953119541195511956119571195811959119601196111962119631196411965119661196711968119691197011971119721197311974119751197611977119781197911980119811198211983119841198511986119871198811989119901199111992119931199411995119961199711998119991200012001120021200312004120051200612007120081200912010120111201212013120141201512016120171201812019120201202112022120231202412025120261202712028120291203012031120321203312034120351203612037120381203912040120411204212043120441204512046120471204812049120501205112052120531205412055120561205712058120591206012061120621206312064120651206612067120681206912070120711207212073120741207512076120771207812079120801208112082120831208412085120861208712088120891209012091120921209312094120951209612097120981209912100121011210212103121041210512106121071210812109121101211112112121131211412115121161211712118121191212012121121221212312124121251212612127121281212912130121311213212133121341213512136121371213812139121401214112142121431214412145121461214712148121491215012151121521215312154121551215612157121581215912160121611216212163121641216512166121671216812169121701217112172121731217412175121761217712178121791218012181121821218312184121851218612187121881218912190121911219212193121941219512196121971219812199122001220112202122031220412205122061220712208122091221012211122121221312214122151221612217122181221912220122211222212223122241222512226122271222812229122301223112232122331223412235122361223712238122391224012241122421224312244122451224612247122481224912250122511225212253122541225512256122571225812259122601226112262122631226412265122661226712268122691227012271122721227312274122751227612277122781227912280122811228212283122841228512286122871228812289122901229112292122931229412295122961229712298122991230012301123021230312304123051230612307123081230912310123111231212313123141231512316123171231812319123201232112322123231232412325123261232712328123291233012331123321233312334123351233612337123381233912340123411234212343123441234512346123471234812349123501235112352123531235412355123561235712358123591236012361123621236312364123651236612367123681236912370123711237212373123741237512376123771237812379123801238112382123831238412385123861238712388123891239012391123921239312394123951239612397123981239912400124011240212403124041240512406124071240812409124101241112412124131241412415124161241712418124191242012421124221242312424124251242612427124281242912430124311243212433124341243512436124371243812439124401244112442124431244412445124461244712448124491245012451124521245312454124551245612457124581245912460124611246212463124641246512466124671246812469124701247112472124731247412475124761247712478124791248012481124821248312484124851248612487124881248912490124911249212493124941249512496124971249812499125001250112502125031250412505125061250712508125091251012511125121251312514125151251612517125181251912520125211252212523125241252512526125271252812529125301253112532125331253412535125361253712538125391254012541125421254312544125451254612547125481254912550125511255212553125541255512556125571255812559125601256112562125631256412565125661256712568125691257012571125721257312574125751257612577125781257912580125811258212583125841258512586125871258812589125901259112592125931259412595125961259712598125991260012601126021260312604126051260612607126081260912610126111261212613126141261512616126171261812619126201262112622126231262412625126261262712628126291263012631126321263312634126351263612637126381263912640126411264212643126441264512646126471264812649126501265112652126531265412655126561265712658126591266012661126621266312664126651266612667126681266912670126711267212673126741267512676126771267812679126801268112682126831268412685126861268712688126891269012691126921269312694126951269612697126981269912700127011270212703127041270512706127071270812709127101271112712127131271412715127161271712718127191272012721127221272312724127251272612727127281272912730127311273212733127341273512736127371273812739127401274112742127431274412745127461274712748127491275012751127521275312754127551275612757127581275912760127611276212763127641276512766127671276812769127701277112772127731277412775127761277712778127791278012781127821278312784127851278612787127881278912790127911279212793127941279512796127971279812799128001280112802128031280412805128061280712808128091281012811128121281312814128151281612817128181281912820128211282212823128241282512826128271282812829128301283112832128331283412835128361283712838128391284012841128421284312844128451284612847128481284912850128511285212853128541285512856128571285812859128601286112862128631286412865128661286712868128691287012871128721287312874128751287612877128781287912880128811288212883128841288512886128871288812889128901289112892128931289412895128961289712898128991290012901129021290312904129051290612907129081290912910129111291212913129141291512916129171291812919129201292112922129231292412925129261292712928129291293012931129321293312934129351293612937129381293912940129411294212943129441294512946129471294812949129501295112952129531295412955129561295712958129591296012961129621296312964129651296612967129681296912970129711297212973129741297512976129771297812979129801298112982129831298412985129861298712988129891299012991129921299312994129951299612997129981299913000130011300213003130041300513006130071300813009130101301113012130131301413015130161301713018130191302013021130221302313024130251302613027130281302913030130311303213033130341303513036130371303813039130401304113042130431304413045130461304713048130491305013051130521305313054130551305613057130581305913060130611306213063130641306513066130671306813069130701307113072130731307413075130761307713078130791308013081130821308313084130851308613087130881308913090130911309213093130941309513096130971309813099131001310113102131031310413105131061310713108131091311013111131121311313114131151311613117131181311913120131211312213123131241312513126131271312813129131301313113132131331313413135131361313713138131391314013141131421314313144131451314613147131481314913150131511315213153131541315513156131571315813159131601316113162131631316413165131661316713168131691317013171131721317313174131751317613177131781317913180131811318213183131841318513186131871318813189131901319113192131931319413195131961319713198131991320013201132021320313204132051320613207132081320913210132111321213213132141321513216132171321813219132201322113222132231322413225132261322713228132291323013231132321323313234132351323613237132381323913240132411324213243132441324513246132471324813249132501325113252132531325413255132561325713258132591326013261132621326313264132651326613267132681326913270132711327213273132741327513276132771327813279132801328113282132831328413285132861328713288132891329013291132921329313294132951329613297132981329913300133011330213303133041330513306133071330813309133101331113312133131331413315133161331713318133191332013321133221332313324133251332613327133281332913330133311333213333133341333513336133371333813339133401334113342133431334413345133461334713348133491335013351133521335313354133551335613357133581335913360133611336213363133641336513366133671336813369133701337113372133731337413375133761337713378133791338013381133821338313384133851338613387133881338913390133911339213393133941339513396133971339813399134001340113402134031340413405134061340713408134091341013411134121341313414134151341613417134181341913420134211342213423134241342513426134271342813429134301343113432134331343413435134361343713438134391344013441134421344313444134451344613447134481344913450134511345213453134541345513456134571345813459134601346113462134631346413465134661346713468134691347013471134721347313474134751347613477134781347913480134811348213483134841348513486134871348813489134901349113492134931349413495134961349713498134991350013501135021350313504135051350613507135081350913510135111351213513135141351513516135171351813519135201352113522135231352413525135261352713528135291353013531135321353313534135351353613537135381353913540135411354213543135441354513546135471354813549135501355113552135531355413555135561355713558135591356013561135621356313564135651356613567135681356913570135711357213573135741357513576135771357813579135801358113582135831358413585135861358713588135891359013591135921359313594135951359613597135981359913600136011360213603136041360513606136071360813609136101361113612136131361413615136161361713618136191362013621136221362313624136251362613627136281362913630136311363213633136341363513636136371363813639136401364113642136431364413645136461364713648136491365013651136521365313654136551365613657136581365913660136611366213663136641366513666136671366813669136701367113672136731367413675136761367713678136791368013681136821368313684136851368613687136881368913690136911369213693136941369513696136971369813699137001370113702137031370413705137061370713708137091371013711137121371313714137151371613717137181371913720137211372213723137241372513726137271372813729137301373113732137331373413735137361373713738137391374013741137421374313744137451374613747137481374913750137511375213753137541375513756137571375813759137601376113762137631376413765137661376713768137691377013771137721377313774137751377613777137781377913780137811378213783137841378513786137871378813789137901379113792137931379413795137961379713798137991380013801138021380313804138051380613807138081380913810138111381213813138141381513816138171381813819138201382113822138231382413825138261382713828138291383013831138321383313834138351383613837138381383913840138411384213843138441384513846138471384813849138501385113852138531385413855138561385713858138591386013861138621386313864138651386613867138681386913870138711387213873138741387513876138771387813879138801388113882138831388413885138861388713888138891389013891138921389313894138951389613897138981389913900139011390213903139041390513906139071390813909139101391113912139131391413915139161391713918139191392013921139221392313924139251392613927139281392913930139311393213933139341393513936139371393813939139401394113942139431394413945139461394713948139491395013951139521395313954139551395613957139581395913960139611396213963139641396513966139671396813969139701397113972139731397413975139761397713978139791398013981139821398313984139851398613987139881398913990139911399213993139941399513996139971399813999140001400114002140031400414005140061400714008140091401014011140121401314014140151401614017140181401914020140211402214023140241402514026140271402814029140301403114032140331403414035140361403714038140391404014041140421404314044140451404614047140481404914050140511405214053140541405514056140571405814059140601406114062140631406414065140661406714068140691407014071140721407314074140751407614077140781407914080140811408214083140841408514086140871408814089140901409114092140931409414095140961409714098140991410014101141021410314104141051410614107141081410914110141111411214113141141411514116141171411814119141201412114122141231412414125141261412714128141291413014131141321413314134141351413614137141381413914140141411414214143141441414514146141471414814149141501415114152141531415414155141561415714158141591416014161141621416314164141651416614167141681416914170141711417214173141741417514176141771417814179141801418114182141831418414185141861418714188141891419014191141921419314194141951419614197141981419914200142011420214203142041420514206142071420814209142101421114212142131421414215142161421714218142191422014221142221422314224142251422614227142281422914230142311423214233142341423514236142371423814239142401424114242142431424414245142461424714248142491425014251142521425314254142551425614257142581425914260142611426214263142641426514266142671426814269142701427114272142731427414275142761427714278142791428014281142821428314284142851428614287142881428914290142911429214293142941429514296142971429814299143001430114302143031430414305143061430714308143091431014311143121431314314143151431614317143181431914320143211432214323143241432514326143271432814329143301433114332143331433414335143361433714338143391434014341143421434314344143451434614347143481434914350143511435214353143541435514356143571435814359143601436114362143631436414365143661436714368143691437014371143721437314374143751437614377143781437914380143811438214383143841438514386143871438814389143901439114392143931439414395143961439714398143991440014401144021440314404144051440614407144081440914410144111441214413144141441514416144171441814419144201442114422144231442414425144261442714428144291443014431144321443314434144351443614437144381443914440144411444214443144441444514446144471444814449144501445114452144531445414455144561445714458144591446014461144621446314464144651446614467144681446914470144711447214473144741447514476144771447814479144801448114482144831448414485144861448714488144891449014491144921449314494144951449614497144981449914500145011450214503145041450514506145071450814509145101451114512145131451414515145161451714518145191452014521145221452314524145251452614527145281452914530145311453214533145341453514536145371453814539145401454114542145431454414545145461454714548145491455014551145521455314554145551455614557145581455914560145611456214563145641456514566145671456814569145701457114572145731457414575145761457714578145791458014581145821458314584145851458614587145881458914590145911459214593145941459514596145971459814599146001460114602146031460414605146061460714608146091461014611146121461314614146151461614617146181461914620146211462214623146241462514626146271462814629146301463114632146331463414635146361463714638146391464014641146421464314644146451464614647146481464914650146511465214653146541465514656146571465814659146601466114662146631466414665146661466714668146691467014671146721467314674146751467614677146781467914680146811468214683146841468514686146871468814689146901469114692146931469414695146961469714698146991470014701147021470314704147051470614707147081470914710147111471214713147141471514716147171471814719147201472114722147231472414725147261472714728147291473014731147321473314734147351473614737147381473914740147411474214743147441474514746147471474814749147501475114752147531475414755147561475714758147591476014761147621476314764147651476614767147681476914770147711477214773147741477514776147771477814779147801478114782147831478414785147861478714788147891479014791147921479314794147951479614797147981479914800148011480214803148041480514806148071480814809148101481114812148131481414815148161481714818148191482014821148221482314824148251482614827148281482914830148311483214833148341483514836148371483814839148401484114842148431484414845148461484714848148491485014851148521485314854148551485614857148581485914860148611486214863148641486514866148671486814869148701487114872148731487414875148761487714878148791488014881148821488314884148851488614887148881488914890148911489214893148941489514896148971489814899149001490114902149031490414905149061490714908149091491014911149121491314914149151491614917149181491914920149211492214923149241492514926149271492814929149301493114932149331493414935149361493714938149391494014941149421494314944149451494614947149481494914950149511495214953149541495514956149571495814959149601496114962149631496414965149661496714968149691497014971149721497314974149751497614977149781497914980149811498214983149841498514986149871498814989149901499114992149931499414995149961499714998149991500015001150021500315004150051500615007150081500915010150111501215013150141501515016150171501815019150201502115022150231502415025150261502715028150291503015031150321503315034150351503615037150381503915040150411504215043150441504515046150471504815049150501505115052150531505415055150561505715058150591506015061150621506315064150651506615067150681506915070150711507215073150741507515076150771507815079150801508115082150831508415085150861508715088150891509015091150921509315094150951509615097150981509915100151011510215103151041510515106151071510815109151101511115112151131511415115151161511715118151191512015121151221512315124151251512615127151281512915130151311513215133151341513515136151371513815139151401514115142151431514415145151461514715148151491515015151151521515315154151551515615157151581515915160151611516215163151641516515166151671516815169151701517115172151731517415175151761517715178151791518015181151821518315184151851518615187151881518915190151911519215193151941519515196151971519815199152001520115202152031520415205152061520715208152091521015211152121521315214152151521615217152181521915220152211522215223152241522515226152271522815229152301523115232152331523415235152361523715238152391524015241152421524315244152451524615247152481524915250152511525215253152541525515256152571525815259152601526115262152631526415265152661526715268152691527015271152721527315274152751527615277152781527915280152811528215283152841528515286152871528815289152901529115292152931529415295152961529715298152991530015301153021530315304153051530615307153081530915310153111531215313153141531515316153171531815319153201532115322153231532415325153261532715328153291533015331153321533315334153351533615337153381533915340153411534215343153441534515346153471534815349153501535115352153531535415355153561535715358153591536015361153621536315364153651536615367153681536915370153711537215373153741537515376153771537815379153801538115382153831538415385153861538715388153891539015391153921539315394153951539615397153981539915400154011540215403154041540515406154071540815409154101541115412154131541415415154161541715418154191542015421154221542315424154251542615427154281542915430154311543215433154341543515436154371543815439154401544115442154431544415445154461544715448154491545015451154521545315454154551545615457154581545915460154611546215463154641546515466154671546815469154701547115472154731547415475154761547715478154791548015481154821548315484154851548615487154881548915490154911549215493154941549515496154971549815499155001550115502155031550415505155061550715508155091551015511155121551315514155151551615517155181551915520155211552215523155241552515526155271552815529155301553115532155331553415535155361553715538155391554015541155421554315544155451554615547155481554915550155511555215553155541555515556155571555815559155601556115562155631556415565155661556715568155691557015571155721557315574155751557615577155781557915580155811558215583155841558515586155871558815589155901559115592155931559415595155961559715598155991560015601156021560315604156051560615607156081560915610156111561215613156141561515616156171561815619156201562115622156231562415625156261562715628156291563015631156321563315634156351563615637156381563915640156411564215643156441564515646156471564815649156501565115652156531565415655156561565715658156591566015661156621566315664156651566615667156681566915670156711567215673156741567515676156771567815679156801568115682156831568415685156861568715688156891569015691156921569315694156951569615697156981569915700157011570215703157041570515706157071570815709157101571115712157131571415715157161571715718157191572015721157221572315724157251572615727157281572915730157311573215733157341573515736157371573815739157401574115742157431574415745157461574715748157491575015751157521575315754157551575615757157581575915760157611576215763157641576515766157671576815769157701577115772157731577415775157761577715778157791578015781157821578315784157851578615787157881578915790157911579215793157941579515796157971579815799158001580115802158031580415805158061580715808158091581015811158121581315814158151581615817158181581915820158211582215823158241582515826158271582815829158301583115832158331583415835158361583715838158391584015841158421584315844158451584615847158481584915850158511585215853158541585515856158571585815859158601586115862158631586415865158661586715868158691587015871158721587315874158751587615877158781587915880158811588215883158841588515886158871588815889158901589115892158931589415895158961589715898158991590015901159021590315904159051590615907159081590915910159111591215913159141591515916159171591815919159201592115922159231592415925159261592715928159291593015931159321593315934159351593615937159381593915940159411594215943159441594515946159471594815949159501595115952159531595415955159561595715958159591596015961159621596315964159651596615967159681596915970159711597215973159741597515976159771597815979159801598115982159831598415985159861598715988159891599015991159921599315994159951599615997159981599916000160011600216003160041600516006160071600816009160101601116012160131601416015160161601716018160191602016021160221602316024160251602616027160281602916030160311603216033160341603516036160371603816039160401604116042160431604416045160461604716048160491605016051160521605316054160551605616057160581605916060160611606216063160641606516066160671606816069160701607116072160731607416075160761607716078160791608016081160821608316084160851608616087160881608916090160911609216093160941609516096160971609816099161001610116102161031610416105161061610716108161091611016111161121611316114161151611616117161181611916120161211612216123161241612516126161271612816129161301613116132161331613416135161361613716138161391614016141161421614316144161451614616147161481614916150161511615216153161541615516156161571615816159161601616116162161631616416165161661616716168161691617016171161721617316174161751617616177161781617916180161811618216183161841618516186161871618816189161901619116192161931619416195161961619716198161991620016201162021620316204162051620616207162081620916210162111621216213162141621516216162171621816219162201622116222162231622416225162261622716228162291623016231162321623316234162351623616237162381623916240162411624216243162441624516246162471624816249162501625116252162531625416255162561625716258162591626016261162621626316264162651626616267162681626916270162711627216273162741627516276162771627816279162801628116282162831628416285162861628716288162891629016291162921629316294162951629616297162981629916300163011630216303163041630516306163071630816309163101631116312163131631416315163161631716318163191632016321163221632316324163251632616327163281632916330163311633216333163341633516336163371633816339163401634116342163431634416345163461634716348163491635016351163521635316354163551635616357163581635916360163611636216363163641636516366163671636816369163701637116372163731637416375163761637716378163791638016381163821638316384163851638616387163881638916390163911639216393163941639516396163971639816399164001640116402164031640416405164061640716408164091641016411164121641316414164151641616417164181641916420164211642216423164241642516426164271642816429164301643116432164331643416435164361643716438164391644016441164421644316444164451644616447164481644916450164511645216453164541645516456164571645816459164601646116462164631646416465164661646716468164691647016471164721647316474164751647616477164781647916480164811648216483164841648516486164871648816489164901649116492164931649416495164961649716498164991650016501165021650316504165051650616507165081650916510165111651216513165141651516516165171651816519165201652116522165231652416525165261652716528165291653016531165321653316534165351653616537165381653916540165411654216543165441654516546165471654816549165501655116552165531655416555165561655716558165591656016561165621656316564165651656616567165681656916570165711657216573165741657516576165771657816579165801658116582165831658416585165861658716588165891659016591165921659316594165951659616597165981659916600166011660216603166041660516606166071660816609166101661116612166131661416615166161661716618166191662016621166221662316624166251662616627166281662916630166311663216633166341663516636166371663816639166401664116642166431664416645166461664716648166491665016651166521665316654166551665616657166581665916660166611666216663166641666516666166671666816669166701667116672166731667416675166761667716678166791668016681166821668316684166851668616687166881668916690166911669216693166941669516696166971669816699167001670116702167031670416705167061670716708167091671016711167121671316714167151671616717167181671916720167211672216723167241672516726167271672816729167301673116732167331673416735167361673716738167391674016741167421674316744167451674616747167481674916750167511675216753167541675516756167571675816759167601676116762167631676416765167661676716768167691677016771167721677316774167751677616777167781677916780167811678216783167841678516786167871678816789167901679116792167931679416795167961679716798167991680016801168021680316804168051680616807168081680916810168111681216813168141681516816168171681816819168201682116822168231682416825168261682716828168291683016831168321683316834168351683616837168381683916840168411684216843168441684516846168471684816849168501685116852168531685416855168561685716858168591686016861168621686316864168651686616867168681686916870168711687216873168741687516876168771687816879168801688116882168831688416885168861688716888168891689016891168921689316894168951689616897168981689916900169011690216903169041690516906169071690816909169101691116912169131691416915169161691716918169191692016921169221692316924169251692616927169281692916930169311693216933169341693516936169371693816939169401694116942169431694416945169461694716948169491695016951169521695316954169551695616957169581695916960169611696216963169641696516966169671696816969169701697116972169731697416975169761697716978169791698016981169821698316984169851698616987169881698916990169911699216993169941699516996169971699816999170001700117002170031700417005170061700717008170091701017011170121701317014170151701617017170181701917020170211702217023170241702517026170271702817029170301703117032170331703417035170361703717038170391704017041170421704317044170451704617047170481704917050170511705217053170541705517056170571705817059170601706117062170631706417065170661706717068170691707017071170721707317074170751707617077170781707917080170811708217083170841708517086170871708817089170901709117092170931709417095170961709717098170991710017101171021710317104171051710617107171081710917110171111711217113171141711517116171171711817119171201712117122171231712417125171261712717128171291713017131171321713317134171351713617137171381713917140171411714217143171441714517146171471714817149171501715117152171531715417155171561715717158171591716017161171621716317164171651716617167171681716917170171711717217173171741717517176171771717817179171801718117182171831718417185171861718717188171891719017191171921719317194171951719617197171981719917200172011720217203172041720517206172071720817209172101721117212172131721417215172161721717218172191722017221172221722317224172251722617227172281722917230172311723217233172341723517236172371723817239172401724117242172431724417245172461724717248172491725017251172521725317254172551725617257172581725917260172611726217263172641726517266172671726817269172701727117272172731727417275172761727717278172791728017281172821728317284172851728617287172881728917290172911729217293172941729517296172971729817299173001730117302173031730417305173061730717308173091731017311173121731317314173151731617317173181731917320173211732217323173241732517326173271732817329173301733117332173331733417335173361733717338173391734017341173421734317344173451734617347173481734917350173511735217353173541735517356173571735817359173601736117362173631736417365173661736717368173691737017371173721737317374173751737617377173781737917380173811738217383173841738517386173871738817389173901739117392173931739417395173961739717398173991740017401174021740317404174051740617407174081740917410174111741217413174141741517416174171741817419174201742117422174231742417425174261742717428174291743017431174321743317434174351743617437174381743917440174411744217443174441744517446174471744817449174501745117452174531745417455174561745717458174591746017461174621746317464174651746617467174681746917470174711747217473174741747517476174771747817479174801748117482174831748417485174861748717488174891749017491174921749317494174951749617497174981749917500175011750217503175041750517506175071750817509175101751117512175131751417515175161751717518175191752017521175221752317524175251752617527175281752917530175311753217533175341753517536175371753817539175401754117542175431754417545175461754717548175491755017551175521755317554175551755617557175581755917560175611756217563175641756517566175671756817569175701757117572175731757417575175761757717578175791758017581175821758317584175851758617587175881758917590175911759217593175941759517596175971759817599176001760117602176031760417605176061760717608176091761017611176121761317614176151761617617176181761917620176211762217623176241762517626176271762817629176301763117632176331763417635176361763717638176391764017641176421764317644176451764617647176481764917650176511765217653176541765517656176571765817659176601766117662176631766417665176661766717668176691767017671176721767317674176751767617677176781767917680176811768217683176841768517686176871768817689176901769117692176931769417695176961769717698176991770017701177021770317704177051770617707177081770917710177111771217713177141771517716177171771817719177201772117722177231772417725177261772717728177291773017731177321773317734177351773617737177381773917740177411774217743177441774517746177471774817749177501775117752177531775417755177561775717758177591776017761177621776317764177651776617767177681776917770177711777217773177741777517776177771777817779177801778117782177831778417785177861778717788177891779017791177921779317794177951779617797177981779917800178011780217803178041780517806178071780817809178101781117812178131781417815178161781717818178191782017821178221782317824178251782617827178281782917830178311783217833178341783517836178371783817839178401784117842178431784417845178461784717848178491785017851178521785317854178551785617857178581785917860178611786217863178641786517866178671786817869178701787117872178731787417875178761787717878178791788017881178821788317884178851788617887178881788917890178911789217893178941789517896178971789817899179001790117902179031790417905179061790717908179091791017911179121791317914179151791617917179181791917920179211792217923179241792517926179271792817929179301793117932179331793417935179361793717938179391794017941179421794317944179451794617947179481794917950179511795217953179541795517956179571795817959179601796117962179631796417965179661796717968179691797017971179721797317974179751797617977179781797917980179811798217983179841798517986179871798817989179901799117992179931799417995179961799717998179991800018001180021800318004180051800618007180081800918010180111801218013180141801518016180171801818019180201802118022180231802418025180261802718028180291803018031180321803318034180351803618037
  1. {
  2. Copyright (c) 1998-2002 by Florian Klaempfl and Jonas Maebe
  3. This unit contains the peephole optimizer.
  4. This program is free software; you can redistribute it and/or modify
  5. it under the terms of the GNU General Public License as published by
  6. the Free Software Foundation; either version 2 of the License, or
  7. (at your option) any later version.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. GNU General Public License for more details.
  12. You should have received a copy of the GNU General Public License
  13. along with this program; if not, write to the Free Software
  14. Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  15. ****************************************************************************
  16. }
  17. unit aoptx86;
  18. {$i fpcdefs.inc}
  19. { $define DEBUG_AOPTCPU}
  20. {$ifdef EXTDEBUG}
  21. {$define DEBUG_AOPTCPU}
  22. {$endif EXTDEBUG}
  23. interface
  24. uses
  25. globtype,cclasses,
  26. cpubase,
  27. aasmtai,aasmcpu,
  28. cgbase,cgutils,
  29. aopt,aoptobj;
  30. type
  31. TOptsToCheck = (
  32. aoc_MovAnd2Mov_3,
  33. aoc_ForceNewIteration,
  34. aoc_DoPass2JccOpts
  35. );
  36. TX86AsmOptimizer = class(TAsmOptimizer)
  37. { some optimizations are very expensive to check, so the
  38. pre opt pass can be used to set some flags, depending on the found
  39. instructions if it is worth to check a certain optimization }
  40. OptsToCheck : set of TOptsToCheck;
  41. function RegLoadedWithNewValue(reg : tregister; hp : tai) : boolean; override;
  42. function InstructionLoadsFromReg(const reg : TRegister; const hp : tai) : boolean; override;
  43. class function RegReadByInstruction(reg : TRegister; hp : tai) : boolean; static;
  44. function RegInInstruction(Reg: TRegister; p1: tai): Boolean;override;
  45. function GetNextInstructionUsingReg(Current: tai; out Next: tai; reg: TRegister): Boolean;
  46. { Identical to GetNextInstructionUsingReg, but returns a value indicating
  47. how many instructions away that Next is from Current is.
  48. 0 = failure, equivalent to False in GetNextInstructionUsingReg }
  49. function GetNextInstructionUsingRegCount(Current: tai; out Next: tai; reg: TRegister): Cardinal;
  50. { This version of GetNextInstructionUsingReg will look across conditional jumps,
  51. potentially allowing further optimisation (although it might need to know if
  52. it crossed a conditional jump. }
  53. function GetNextInstructionUsingRegCond(Current: tai; out Next: tai; reg: TRegister; var JumpTracking: TLinkedList; var CrossJump: Boolean): Boolean;
  54. {
  55. In comparison with GetNextInstructionUsingReg, GetNextInstructionUsingRegTrackingUse tracks
  56. the use of a register by allocs/dealloc, so it can ignore calls.
  57. In the following example, GetNextInstructionUsingReg will return the second movq,
  58. GetNextInstructionUsingRegTrackingUse won't.
  59. movq %rdi,%rax
  60. # Register rdi released
  61. # Register rdi allocated
  62. movq %rax,%rdi
  63. While in this example:
  64. movq %rdi,%rax
  65. call proc
  66. movq %rdi,%rax
  67. GetNextInstructionUsingRegTrackingUse will return the second instruction while GetNextInstructionUsingReg
  68. won't.
  69. }
  70. function GetNextInstructionUsingRegTrackingUse(Current: tai; out Next: tai; reg: TRegister): Boolean;
  71. function RegModifiedByInstruction(Reg: TRegister; p1: tai): boolean; override;
  72. { returns true if any of the registers in ref are modified by any
  73. instruction between p1 and p2, or if those instructions write to the
  74. reference }
  75. function RefModifiedBetween(Ref: TReference; RefSize: ASizeInt; p1, p2: tai): Boolean;
  76. private
  77. function SkipSimpleInstructions(var hp1: tai): Boolean;
  78. protected
  79. class function IsMOVZXAcceptable: Boolean; static; inline;
  80. function CheckMovMov2MovMov2(const p, hp1: tai): Boolean;
  81. { Attempts to allocate a volatile integer register for use between p and hp,
  82. using AUsedRegs for the current register usage information. Returns NR_NO
  83. if no free register could be found }
  84. function GetIntRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  85. { Attempts to allocate a volatile MM register for use between p and hp,
  86. using AUsedRegs for the current register usage information. Returns NR_NO
  87. if no free register could be found }
  88. function GetMMRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  89. { checks whether loading a new value in reg1 overwrites the entirety of reg2 }
  90. class function Reg1WriteOverwritesReg2Entirely(reg1, reg2: tregister): boolean; static;
  91. { checks whether reading the value in reg1 depends on the value of reg2. This
  92. is very similar to SuperRegisterEquals, except it takes into account that
  93. R_SUBH and R_SUBL are independendent (e.g. reading from AL does not
  94. depend on the value in AH). }
  95. class function Reg1ReadDependsOnReg2(reg1, reg2: tregister): boolean; static;
  96. { Replaces all references to AOldReg in a memory reference to ANewReg }
  97. class function ReplaceRegisterInRef(var ref: TReference; const AOldReg, ANewReg: TRegister): Boolean; static;
  98. { Replaces all references to AOldReg in an operand to ANewReg }
  99. class function ReplaceRegisterInOper(const p: taicpu; const OperIdx: Integer; const AOldReg, ANewReg: TRegister): Boolean; static;
  100. { Replaces all references to AOldReg in an instruction to ANewReg,
  101. except where the register is being written }
  102. class function ReplaceRegisterInInstruction(const p: taicpu; const AOldReg, ANewReg: TRegister): Boolean; static;
  103. { Returns true if the reference only refers to ESP or EBP (or their 64-bit equivalents),
  104. or writes to a global symbol }
  105. class function IsRefSafe(const ref: PReference): Boolean; static;
  106. { Returns true if the given MOV instruction can be safely converted to CMOV }
  107. class function CanBeCMOV(p, cond_p: tai; var RefModified: Boolean) : boolean; static;
  108. { Like UpdateUsedRegs, but ignores deallocations }
  109. class procedure UpdateIntRegsNoDealloc(var AUsedRegs: TAllUsedRegs; p: Tai); static;
  110. { Returns true if the given logic instruction can be converted into a BTx instruction (BT not included) }
  111. class function IsBTXAcceptable(p : tai) : boolean; static;
  112. { Converts the LEA instruction to ADD/INC/SUB/DEC. Returns True if the
  113. conversion was successful }
  114. function ConvertLEA(const p : taicpu): Boolean;
  115. function DeepMOVOpt(const p_mov: taicpu; const hp: taicpu): Boolean;
  116. function FuncMov2Func(var p: tai; const hp1: tai): Boolean;
  117. {$ifdef x86_64}
  118. { If a "mov %reg1d,%reg2d; and %reg1d,%reg1d" is found, we can possibly
  119. replace %reg2q with %reg1q in later instructions }
  120. function DoZeroUpper32Opt(var mov_p: tai; var and_p: tai): Boolean;
  121. {$endif x86_64}
  122. procedure DebugMsg(const s : string; p : tai);inline;
  123. class function IsExitCode(p : tai) : boolean; static;
  124. class function isFoldableArithOp(hp1 : taicpu; reg : tregister) : boolean; static;
  125. class function IsShrMovZFoldable(shr_size, movz_size: topsize; Shift: TCGInt): Boolean; static;
  126. procedure RemoveLastDeallocForFuncRes(p : tai);
  127. function DoArithCombineOpt(var p : tai) : Boolean;
  128. function DoMovCmpMemOpt(var p : tai; const hp1: tai) : Boolean;
  129. function DoSETccLblRETOpt(var p: tai; const hp_label: tai_label) : Boolean;
  130. function HandleSHRMerge(var p: tai; const PostPeephole: Boolean): Boolean;
  131. function PrePeepholeOptSxx(var p : tai) : boolean;
  132. function PrePeepholeOptIMUL(var p : tai) : boolean;
  133. function PrePeepholeOptAND(var p : tai) : boolean;
  134. function OptPass1Test(var p: tai): boolean;
  135. function OptPass1Add(var p: tai): boolean;
  136. function OptPass1AND(var p : tai) : boolean;
  137. function OptPass1CMOVcc(var p: tai): Boolean;
  138. function OptPass1_V_MOVAP(var p : tai) : boolean;
  139. function OptPass1VOP(var p : tai) : boolean;
  140. function OptPass1MOV(var p : tai) : boolean;
  141. function OptPass1MOVD(var p : tai) : boolean;
  142. function OptPass1Movx(var p : tai) : boolean;
  143. function OptPass1MOVXX(var p : tai) : boolean;
  144. function OptPass1OP(var p : tai) : boolean;
  145. function OptPass1LEA(var p : tai) : boolean;
  146. function OptPass1Sub(var p : tai) : boolean;
  147. function OptPass1SHLSAL(var p : tai) : boolean;
  148. function OptPass1SHR(var p : tai) : boolean;
  149. function OptPass1FSTP(var p : tai) : boolean;
  150. function OptPass1FLD(var p : tai) : boolean;
  151. function OptPass1Cmp(var p : tai) : boolean;
  152. function OptPass1PXor(var p : tai) : boolean;
  153. function OptPass1VPXor(var p: tai): boolean;
  154. function OptPass1Imul(var p : tai) : boolean;
  155. function OptPass1Jcc(var p : tai) : boolean;
  156. function OptPass1SHXX(var p: tai): boolean;
  157. function OptPass1VMOVDQ(var p: tai): Boolean;
  158. function OptPass1_V_Cvtss2sd(var p: tai): boolean;
  159. function OptPass1STCCLC(var p: tai): Boolean;
  160. function OptPass2STCCLC(var p: tai): Boolean;
  161. function OptPass2CMOVcc(var p: tai): Boolean;
  162. function OptPass2Movx(var p : tai): Boolean;
  163. function OptPass2MOV(var p : tai) : boolean;
  164. function OptPass2Imul(var p : tai) : boolean;
  165. function OptPass2Jmp(var p : tai) : boolean;
  166. function OptPass2Jcc(var p : tai) : boolean;
  167. function OptPass2Lea(var p: tai): Boolean;
  168. function OptPass2SUB(var p: tai): Boolean;
  169. function OptPass2ADD(var p : tai): Boolean;
  170. function OptPass2SETcc(var p : tai) : boolean;
  171. function OptPass2Cmp(var p: tai): Boolean;
  172. function OptPass2Test(var p: tai): Boolean;
  173. function CheckMemoryWrite(var first_mov, second_mov: taicpu): Boolean;
  174. function PostPeepholeOptMov(var p : tai) : Boolean;
  175. function PostPeepholeOptMovzx(var p : tai) : Boolean;
  176. function PostPeepholeOptXor(var p : tai) : Boolean;
  177. function PostPeepholeOptAnd(var p : tai) : boolean;
  178. function PostPeepholeOptMOVSX(var p : tai) : boolean;
  179. function PostPeepholeOptCmp(var p : tai) : Boolean;
  180. function PostPeepholeOptTestOr(var p : tai) : Boolean;
  181. function PostPeepholeOptCall(var p : tai) : Boolean;
  182. function PostPeepholeOptLea(var p : tai) : Boolean;
  183. function PostPeepholeOptPush(var p: tai): Boolean;
  184. function PostPeepholeOptShr(var p : tai) : boolean;
  185. function PostPeepholeOptADDSUB(var p : tai) : Boolean;
  186. function PostPeepholeOptVPXOR(var p: tai): Boolean;
  187. function PostPeepholeOptRET(var p: tai): Boolean;
  188. procedure ConvertJumpToRET(const p: tai; const ret_p: tai);
  189. function CheckJumpMovTransferOpt(var p: tai; hp1: tai; LoopCount: Integer; out Count: Integer): Boolean;
  190. function TrySwapMovOp(var p, hp1: tai): Boolean;
  191. function TrySwapMovCmp(var p, hp1: tai): Boolean;
  192. function TryCmpCMovOpts(var p, hp1: tai) : Boolean;
  193. function TryJccStcClcOpt(var p, hp1: tai): Boolean;
  194. { Processor-dependent reference optimisation }
  195. class procedure OptimizeRefs(var p: taicpu); static;
  196. end;
  197. function MatchInstruction(const instr: tai; const op: TAsmOp; const opsize: topsizes): boolean;
  198. function MatchInstruction(const instr: tai; const op1,op2: TAsmOp; const opsize: topsizes): boolean;
  199. function MatchInstruction(const instr: tai; const op1,op2,op3: TAsmOp; const opsize: topsizes): boolean;
  200. function MatchInstruction(const instr: tai; const ops: array of TAsmOp; const opsize: topsizes): boolean;
  201. function MatchOperand(const oper: TOper; const reg: TRegister): boolean; inline;
  202. function MatchOperand(const oper: TOper; const a: tcgint): boolean; inline;
  203. function MatchOperand(const oper1: TOper; const oper2: TOper): boolean;
  204. {$if max_operands>2}
  205. function MatchOperand(const oper1: TOper; const oper2: TOper; const oper3: TOper): boolean;
  206. {$endif max_operands>2}
  207. function RefsEqual(const r1, r2: treference): boolean;
  208. { Like RefsEqual, but doesn't compare the offsets }
  209. function RefsAlmostEqual(const r1, r2: treference): boolean;
  210. { Note that Result is set to True if the references COULD overlap but the
  211. compiler cannot be sure (e.g. "(%reg1)" and "4(%reg2)" with a range of 4
  212. might still overlap because %reg2 could be equal to %reg1-4 }
  213. function RefsMightOverlap(const r1, r2: treference; const Range: asizeint): boolean;
  214. function MatchReference(const ref : treference;base,index : TRegister) : Boolean;
  215. { returns true, if ref is a reference using only the registers passed as base and index
  216. and having an offset }
  217. function MatchReferenceWithOffset(const ref : treference;base,index : TRegister) : Boolean;
  218. implementation
  219. uses
  220. cutils,verbose,
  221. systems,
  222. globals,
  223. cpuinfo,
  224. procinfo,
  225. paramgr,
  226. aasmbase,
  227. aoptbase,aoptutils,
  228. symconst,symsym,
  229. cgx86,
  230. itcpugas;
  231. {$ifndef 8086}
  232. const
  233. MAX_CMOV_INSTRUCTIONS = 4;
  234. MAX_CMOV_REGISTERS = 8;
  235. type
  236. TCMovTrackingState = (tsInvalid, tsSimple, tsDetour, tsBranching,
  237. tsDouble, tsDoubleBranchSame, tsDoubleBranchDifferent, tsDoubleSecondBranching,
  238. tsProcessed);
  239. { For OptPass2Jcc }
  240. TCMOVTracking = object
  241. private
  242. CMOVScore, ConstCount: LongInt;
  243. RegWrites: array[0..MAX_CMOV_INSTRUCTIONS*2 - 1] of TRegister;
  244. ConstRegs: array[0..MAX_CMOV_REGISTERS - 1] of TRegister;
  245. ConstVals: array[0..MAX_CMOV_REGISTERS - 1] of TCGInt;
  246. ConstSizes: array[0..MAX_CMOV_REGISTERS - 1] of TSubRegister; { May not match ConstRegs if one is shared over multiple CMOVs. }
  247. ConstMovs: array[0..MAX_CMOV_REGISTERS - 1] of tai; { Location of initialisation instruction }
  248. ConstWriteSizes: array[0..first_int_imreg - 1] of TSubRegister; { Largest size of register written. }
  249. fOptimizer: TX86AsmOptimizer;
  250. fLabel: TAsmSymbol;
  251. fInsertionPoint,
  252. fCondition,
  253. fInitialJump,
  254. fFirstMovBlock,
  255. fFirstMovBlockStop,
  256. fSecondJump,
  257. fThirdJump,
  258. fSecondMovBlock,
  259. fSecondMovBlockStop,
  260. fMidLabel,
  261. fEndLabel,
  262. fAllocationRange: tai;
  263. fState: TCMovTrackingState;
  264. function TryCMOVConst(p, start, stop: tai; var Count: LongInt): Boolean;
  265. function InitialiseBlock(BlockStart, OneBeforeBlock: tai; out BlockStop: tai; out EndJump: tai): Boolean;
  266. function AnalyseMOVBlock(BlockStart, BlockStop, SearchStart: tai): LongInt;
  267. public
  268. RegisterTracking: TAllUsedRegs;
  269. constructor Init(Optimizer: TX86AsmOptimizer; var p_initialjump, p_initialmov: tai; var AFirstLabel: TAsmLabel);
  270. destructor Done;
  271. procedure Process(out new_p: tai);
  272. property State: TCMovTrackingState read fState;
  273. end;
  274. PCMOVTracking = ^TCMOVTracking;
  275. {$endif 8086}
  276. {$ifdef DEBUG_AOPTCPU}
  277. const
  278. SPeepholeOptimization: shortstring = 'Peephole Optimization: ';
  279. {$else DEBUG_AOPTCPU}
  280. { Empty strings help the optimizer to remove string concatenations that won't
  281. ever appear to the user on release builds. [Kit] }
  282. const
  283. SPeepholeOptimization = '';
  284. {$endif DEBUG_AOPTCPU}
  285. LIST_STEP_SIZE = 4;
  286. type
  287. TJumpTrackingItem = class(TLinkedListItem)
  288. private
  289. FSymbol: TAsmSymbol;
  290. FRefs: LongInt;
  291. public
  292. constructor Create(ASymbol: TAsmSymbol);
  293. procedure IncRefs; {$ifdef USEINLINE}inline;{$endif USEINLINE}
  294. property Symbol: TAsmSymbol read FSymbol;
  295. property Refs: LongInt read FRefs;
  296. end;
  297. constructor TJumpTrackingItem.Create(ASymbol: TAsmSymbol);
  298. begin
  299. inherited Create;
  300. FSymbol := ASymbol;
  301. FRefs := 0;
  302. end;
  303. procedure TJumpTrackingItem.IncRefs; {$ifdef USEINLINE}inline;{$endif USEINLINE}
  304. begin
  305. Inc(FRefs);
  306. end;
  307. function MatchInstruction(const instr: tai; const op: TAsmOp; const opsize: topsizes): boolean;
  308. begin
  309. result :=
  310. (instr.typ = ait_instruction) and
  311. (taicpu(instr).opcode = op) and
  312. ((opsize = []) or (taicpu(instr).opsize in opsize));
  313. end;
  314. function MatchInstruction(const instr: tai; const op1,op2: TAsmOp; const opsize: topsizes): boolean;
  315. begin
  316. result :=
  317. (instr.typ = ait_instruction) and
  318. ((taicpu(instr).opcode = op1) or
  319. (taicpu(instr).opcode = op2)
  320. ) and
  321. ((opsize = []) or (taicpu(instr).opsize in opsize));
  322. end;
  323. function MatchInstruction(const instr: tai; const op1,op2,op3: TAsmOp; const opsize: topsizes): boolean;
  324. begin
  325. result :=
  326. (instr.typ = ait_instruction) and
  327. ((taicpu(instr).opcode = op1) or
  328. (taicpu(instr).opcode = op2) or
  329. (taicpu(instr).opcode = op3)
  330. ) and
  331. ((opsize = []) or (taicpu(instr).opsize in opsize));
  332. end;
  333. function MatchInstruction(const instr : tai;const ops : array of TAsmOp;
  334. const opsize : topsizes) : boolean;
  335. var
  336. op : TAsmOp;
  337. begin
  338. result:=false;
  339. if (instr.typ <> ait_instruction) or
  340. ((opsize <> []) and not(taicpu(instr).opsize in opsize)) then
  341. exit;
  342. for op in ops do
  343. begin
  344. if taicpu(instr).opcode = op then
  345. begin
  346. result:=true;
  347. exit;
  348. end;
  349. end;
  350. end;
  351. function MatchOperand(const oper: TOper; const reg: TRegister): boolean; inline;
  352. begin
  353. result := (oper.typ = top_reg) and (oper.reg = reg);
  354. end;
  355. function MatchOperand(const oper: TOper; const a: tcgint): boolean; inline;
  356. begin
  357. result := (oper.typ = top_const) and (oper.val = a);
  358. end;
  359. function MatchOperand(const oper1: TOper; const oper2: TOper): boolean;
  360. begin
  361. result := oper1.typ = oper2.typ;
  362. if result then
  363. case oper1.typ of
  364. top_const:
  365. Result:=oper1.val = oper2.val;
  366. top_reg:
  367. Result:=oper1.reg = oper2.reg;
  368. top_ref:
  369. Result:=RefsEqual(oper1.ref^, oper2.ref^);
  370. else
  371. internalerror(2013102801);
  372. end
  373. end;
  374. function MatchOperand(const oper1: TOper; const oper2: TOper; const oper3: TOper): boolean;
  375. begin
  376. result := (oper1.typ = oper2.typ) and (oper1.typ = oper3.typ);
  377. if result then
  378. case oper1.typ of
  379. top_const:
  380. Result:=(oper1.val = oper2.val) and (oper1.val = oper3.val);
  381. top_reg:
  382. Result:=(oper1.reg = oper2.reg) and (oper1.reg = oper3.reg);
  383. top_ref:
  384. Result:=RefsEqual(oper1.ref^, oper2.ref^) and RefsEqual(oper1.ref^, oper3.ref^);
  385. else
  386. internalerror(2020052401);
  387. end
  388. end;
  389. function RefsEqual(const r1, r2: treference): boolean;
  390. begin
  391. RefsEqual :=
  392. (r1.symbol=r2.symbol) and (r1.refaddr = r2.refaddr) and
  393. (r1.relsymbol = r2.relsymbol) and
  394. (r1.segment = r2.segment) and (r1.base = r2.base) and
  395. (r1.index = r2.index) and (r1.scalefactor = r2.scalefactor) and
  396. (r1.offset = r2.offset) and
  397. (r1.volatility + r2.volatility = []);
  398. end;
  399. function RefsAlmostEqual(const r1, r2: treference): boolean;
  400. begin
  401. RefsAlmostEqual :=
  402. (r1.symbol=r2.symbol) and (r1.refaddr = r2.refaddr) and
  403. (r1.relsymbol = r2.relsymbol) and
  404. (r1.segment = r2.segment) and (r1.base = r2.base) and
  405. (r1.index = r2.index) and (r1.scalefactor = r2.scalefactor) and
  406. { Don't compare the offsets }
  407. (r1.volatility + r2.volatility = []);
  408. end;
  409. function RefsMightOverlap(const r1, r2: treference; const Range: asizeint): boolean;
  410. begin
  411. if (r1.symbol<>r2.symbol) then
  412. { If the index registers are different, there's a chance one could
  413. be set so it equals the other symbol }
  414. Exit((r1.index<>r2.index) or (r1.scalefactor<>r2.scalefactor));
  415. if (r1.symbol=r2.symbol) and (r1.refaddr = r2.refaddr) and
  416. (r1.relsymbol = r2.relsymbol) and
  417. (r1.segment = r2.segment) and (r1.base = r2.base) and
  418. (r1.index = r2.index) and (r1.scalefactor = r2.scalefactor) and
  419. (r1.volatility + r2.volatility = []) then
  420. { In this case, it all depends on the offsets }
  421. Exit(abs(r1.offset - r2.offset) < Range);
  422. { There's a chance things MIGHT overlap, so take no chances }
  423. Result := True;
  424. end;
  425. function MatchReference(const ref : treference;base,index : TRegister) : Boolean;
  426. begin
  427. Result:=(ref.offset=0) and
  428. (ref.scalefactor in [0,1]) and
  429. (ref.segment=NR_NO) and
  430. (ref.symbol=nil) and
  431. (ref.relsymbol=nil) and
  432. ((base=NR_INVALID) or
  433. (ref.base=base)) and
  434. ((index=NR_INVALID) or
  435. (ref.index=index)) and
  436. (ref.volatility=[]);
  437. end;
  438. function MatchReferenceWithOffset(const ref : treference;base,index : TRegister) : Boolean;
  439. begin
  440. Result:=(ref.scalefactor in [0,1]) and
  441. (ref.segment=NR_NO) and
  442. (ref.symbol=nil) and
  443. (ref.relsymbol=nil) and
  444. ((base=NR_INVALID) or
  445. (ref.base=base)) and
  446. ((index=NR_INVALID) or
  447. (ref.index=index)) and
  448. (ref.volatility=[]);
  449. end;
  450. function InstrReadsFlags(p: tai): boolean;
  451. begin
  452. InstrReadsFlags := true;
  453. case p.typ of
  454. ait_instruction:
  455. if InsProp[taicpu(p).opcode].Ch*
  456. [Ch_RCarryFlag,Ch_RParityFlag,Ch_RAuxiliaryFlag,Ch_RZeroFlag,Ch_RSignFlag,Ch_ROverflowFlag,
  457. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag,
  458. Ch_RFlags,Ch_RWFlags,Ch_RFLAGScc,Ch_All]<>[] then
  459. exit;
  460. ait_label:
  461. exit;
  462. else
  463. ;
  464. end;
  465. InstrReadsFlags := false;
  466. end;
  467. function TX86AsmOptimizer.GetNextInstructionUsingReg(Current: tai; out Next: tai; reg: TRegister): Boolean;
  468. begin
  469. Next:=Current;
  470. repeat
  471. Result:=GetNextInstruction(Next,Next);
  472. until not (Result) or
  473. not(cs_opt_level3 in current_settings.optimizerswitches) or
  474. (Next.typ<>ait_instruction) or
  475. RegInInstruction(reg,Next) or
  476. is_calljmp(taicpu(Next).opcode);
  477. end;
  478. function TX86AsmOptimizer.GetNextInstructionUsingRegCount(Current: tai; out Next: tai; reg: TRegister): Cardinal;
  479. var
  480. GetNextResult: Boolean;
  481. begin
  482. Result:=0;
  483. Next:=Current;
  484. repeat
  485. GetNextResult := GetNextInstruction(Next,Next);
  486. if GetNextResult then
  487. Inc(Result)
  488. else
  489. { Must return zero upon hitting the end of the linked list without a match }
  490. Result := 0;
  491. until not (GetNextResult) or
  492. not(cs_opt_level3 in current_settings.optimizerswitches) or
  493. (Next.typ<>ait_instruction) or
  494. RegInInstruction(reg,Next) or
  495. is_calljmp(taicpu(Next).opcode);
  496. end;
  497. function TX86AsmOptimizer.GetNextInstructionUsingRegCond(Current: tai; out Next: tai; reg: TRegister; var JumpTracking: TLinkedList; var CrossJump: Boolean): Boolean;
  498. procedure TrackJump(Symbol: TAsmSymbol);
  499. var
  500. Search: TJumpTrackingItem;
  501. begin
  502. { See if an entry already exists in our jump tracking list
  503. (faster to search backwards due to the higher chance of
  504. matching destinations) }
  505. Search := TJumpTrackingItem(JumpTracking.Last);
  506. while Assigned(Search) do
  507. begin
  508. if Search.Symbol = Symbol then
  509. begin
  510. { Found it - remove it so it can be pushed to the front }
  511. JumpTracking.Remove(Search);
  512. Break;
  513. end;
  514. Search := TJumpTrackingItem(Search.Previous);
  515. end;
  516. if not Assigned(Search) then
  517. Search := TJumpTrackingItem.Create(JumpTargetOp(taicpu(Next))^.ref^.symbol);
  518. JumpTracking.Concat(Search);
  519. Search.IncRefs;
  520. end;
  521. function LabelAccountedFor(Symbol: TAsmSymbol): Boolean;
  522. var
  523. Search: TJumpTrackingItem;
  524. begin
  525. Result := False;
  526. { See if this label appears in the tracking list }
  527. Search := TJumpTrackingItem(JumpTracking.Last);
  528. while Assigned(Search) do
  529. begin
  530. if Search.Symbol = Symbol then
  531. begin
  532. { Found it - let's see what we can discover }
  533. if Search.Symbol.getrefs = Search.Refs then
  534. begin
  535. { Success - all the references are accounted for }
  536. JumpTracking.Remove(Search);
  537. Search.Free;
  538. { It is logically impossible for CrossJump to be false here
  539. because we must have run into a conditional jump for
  540. this label at some point }
  541. if not CrossJump then
  542. InternalError(2022041710);
  543. if JumpTracking.First = nil then
  544. { Tracking list is now empty - no more cross jumps }
  545. CrossJump := False;
  546. Result := True;
  547. Exit;
  548. end;
  549. { If the references don't match, it's possible to enter
  550. this label through other means, so drop out }
  551. Exit;
  552. end;
  553. Search := TJumpTrackingItem(Search.Previous);
  554. end;
  555. end;
  556. var
  557. Next_Label: tai;
  558. begin
  559. { Note, CrossJump keeps its input value if a conditional jump is not found - it doesn't get set to False }
  560. Next := Current;
  561. repeat
  562. Result := GetNextInstruction(Next,Next);
  563. if not Result then
  564. Break;
  565. if (Next.typ=ait_instruction) and is_calljmp(taicpu(Next).opcode) then
  566. if is_calljmpuncondret(taicpu(Next).opcode) then
  567. begin
  568. if (taicpu(Next).opcode = A_JMP) and
  569. { Remove dead code now to save time }
  570. RemoveDeadCodeAfterJump(taicpu(Next)) then
  571. { A jump was removed, but not the current instruction, and
  572. Result doesn't necessarily translate into an optimisation
  573. routine's Result, so use the "Force New Iteration" flag so
  574. mark a new pass }
  575. Include(OptsToCheck, aoc_ForceNewIteration);
  576. if not Assigned(JumpTracking) then
  577. begin
  578. { Cross-label optimisations often causes other optimisations
  579. to perform worse because they're not given the chance to
  580. optimise locally. In this case, don't do the cross-label
  581. optimisations yet, but flag them as a potential possibility
  582. for the next iteration of Pass 1 }
  583. if not NotFirstIteration then
  584. Include(OptsToCheck, aoc_ForceNewIteration);
  585. end
  586. else if IsJumpToLabel(taicpu(Next)) and
  587. GetNextInstruction(Next, Next_Label) then
  588. begin
  589. { If we have JMP .lbl, and the label after it has all of its
  590. references tracked, then this is probably an if-else style of
  591. block and we can keep tracking. If the label for this jump
  592. then appears later and is fully tracked, then it's the end
  593. of the if-else blocks and the code paths converge (thus
  594. marking the end of the cross-jump) }
  595. if (Next_Label.typ = ait_label) then
  596. begin
  597. if LabelAccountedFor(tai_label(Next_Label).labsym) then
  598. begin
  599. TrackJump(JumpTargetOp(taicpu(Next))^.ref^.symbol);
  600. Next := Next_Label;
  601. { CrossJump gets set to false by LabelAccountedFor if the
  602. list is completely emptied (as it indicates that all
  603. code paths have converged). We could avoid this nuance
  604. by moving the TrackJump call to before the
  605. LabelAccountedFor call, but this is slower in situations
  606. where LabelAccountedFor would return False due to the
  607. creation of a new object that is not used and destroyed
  608. soon after. }
  609. CrossJump := True;
  610. Continue;
  611. end;
  612. end
  613. else if (Next_Label.typ <> ait_marker) then
  614. { We just did a RemoveDeadCodeAfterJump, so either we find
  615. a label, the end of the procedure or some kind of marker}
  616. InternalError(2022041720);
  617. end;
  618. Result := False;
  619. Exit;
  620. end
  621. else
  622. begin
  623. if not Assigned(JumpTracking) then
  624. begin
  625. { Cross-label optimisations often causes other optimisations
  626. to perform worse because they're not given the chance to
  627. optimise locally. In this case, don't do the cross-label
  628. optimisations yet, but flag them as a potential possibility
  629. for the next iteration of Pass 1 }
  630. if not NotFirstIteration then
  631. Include(OptsToCheck, aoc_ForceNewIteration);
  632. end
  633. else if IsJumpToLabel(taicpu(Next)) then
  634. TrackJump(JumpTargetOp(taicpu(Next))^.ref^.symbol)
  635. else
  636. { Conditional jumps should always be a jump to label }
  637. InternalError(2022041701);
  638. CrossJump := True;
  639. Continue;
  640. end;
  641. if Next.typ = ait_label then
  642. begin
  643. if not Assigned(JumpTracking) then
  644. begin
  645. { Cross-label optimisations often causes other optimisations
  646. to perform worse because they're not given the chance to
  647. optimise locally. In this case, don't do the cross-label
  648. optimisations yet, but flag them as a potential possibility
  649. for the next iteration of Pass 1 }
  650. if not NotFirstIteration then
  651. Include(OptsToCheck, aoc_ForceNewIteration);
  652. end
  653. else if LabelAccountedFor(tai_label(Next).labsym) then
  654. Continue;
  655. { If we reach here, we're at a label that hasn't been seen before
  656. (or JumpTracking was nil) }
  657. Break;
  658. end;
  659. until not Result or
  660. not (cs_opt_level3 in current_settings.optimizerswitches) or
  661. not (Next.typ in [ait_label, ait_instruction]) or
  662. RegInInstruction(reg,Next);
  663. end;
  664. function TX86AsmOptimizer.GetNextInstructionUsingRegTrackingUse(Current: tai; out Next: tai; reg: TRegister): Boolean;
  665. begin
  666. if not(cs_opt_level3 in current_settings.optimizerswitches) then
  667. begin
  668. Result:=GetNextInstruction(Current,Next);
  669. exit;
  670. end;
  671. Next:=tai(Current.Next);
  672. Result:=false;
  673. while assigned(Next) do
  674. begin
  675. if ((Next.typ=ait_instruction) and is_calljmp(taicpu(Next).opcode) and not(taicpu(Next).opcode=A_CALL)) or
  676. ((Next.typ=ait_regalloc) and (getsupreg(tai_regalloc(Next).reg)=getsupreg(reg))) or
  677. ((Next.typ=ait_label) and not(labelCanBeSkipped(Tai_Label(Next)))) then
  678. exit
  679. else if (Next.typ=ait_instruction) and RegInInstruction(reg,Next) and not(taicpu(Next).opcode=A_CALL) then
  680. begin
  681. Result:=true;
  682. exit;
  683. end;
  684. Next:=tai(Next.Next);
  685. end;
  686. end;
  687. function TX86AsmOptimizer.InstructionLoadsFromReg(const reg: TRegister;const hp: tai): boolean;
  688. begin
  689. Result:=RegReadByInstruction(reg,hp);
  690. end;
  691. class function TX86AsmOptimizer.RegReadByInstruction(reg: TRegister; hp: tai): boolean;
  692. var
  693. p: taicpu;
  694. opcount: longint;
  695. begin
  696. RegReadByInstruction := false;
  697. if hp.typ <> ait_instruction then
  698. exit;
  699. p := taicpu(hp);
  700. case p.opcode of
  701. A_CALL:
  702. regreadbyinstruction := true;
  703. A_IMUL:
  704. case p.ops of
  705. 1:
  706. regReadByInstruction := RegInOp(reg,p.oper[0]^) or
  707. (
  708. ((getregtype(reg)=R_INTREGISTER) and (getsupreg(reg)=RS_EAX)) and
  709. ((getsubreg(reg)<>R_SUBH) or (p.opsize<>S_B))
  710. );
  711. 2,3:
  712. regReadByInstruction :=
  713. reginop(reg,p.oper[0]^) or
  714. reginop(reg,p.oper[1]^);
  715. else
  716. InternalError(2019112801);
  717. end;
  718. A_MUL:
  719. begin
  720. regReadByInstruction := RegInOp(reg,p.oper[0]^) or
  721. (
  722. ((getregtype(reg)=R_INTREGISTER) and (getsupreg(reg)=RS_EAX)) and
  723. ((getsubreg(reg)<>R_SUBH) or (p.opsize<>S_B))
  724. );
  725. end;
  726. A_IDIV,A_DIV:
  727. begin
  728. regReadByInstruction := RegInOp(reg,p.oper[0]^) or
  729. (
  730. (getregtype(reg)=R_INTREGISTER) and
  731. (
  732. (getsupreg(reg)=RS_EAX) or ((getsupreg(reg)=RS_EDX) and (p.opsize<>S_B))
  733. )
  734. );
  735. end;
  736. else
  737. begin
  738. if (p.opcode=A_LEA) and is_segment_reg(reg) then
  739. begin
  740. RegReadByInstruction := false;
  741. exit;
  742. end;
  743. for opcount := 0 to p.ops-1 do
  744. if (p.oper[opCount]^.typ = top_ref) and
  745. RegInRef(reg,p.oper[opcount]^.ref^) then
  746. begin
  747. RegReadByInstruction := true;
  748. exit
  749. end;
  750. { special handling for SSE MOVSD }
  751. if (p.opcode=A_MOVSD) and (p.ops>0) then
  752. begin
  753. if p.ops<>2 then
  754. internalerror(2017042702);
  755. regReadByInstruction := reginop(reg,p.oper[0]^) or
  756. (
  757. (p.oper[1]^.typ=top_reg) and (p.oper[0]^.typ=top_reg) and reginop(reg, p.oper[1]^)
  758. );
  759. exit;
  760. end;
  761. with insprop[p.opcode] do
  762. begin
  763. case getregtype(reg) of
  764. R_INTREGISTER:
  765. begin
  766. case getsupreg(reg) of
  767. RS_EAX:
  768. if [Ch_REAX,Ch_RWEAX,Ch_MEAX,Ch_WRAX,Ch_RWRAX,Ch_MRAX]*Ch<>[] then
  769. begin
  770. RegReadByInstruction := true;
  771. exit
  772. end;
  773. RS_ECX:
  774. if [Ch_RECX,Ch_RWECX,Ch_MECX,Ch_WRCX,Ch_RWRCX,Ch_MRCX]*Ch<>[] then
  775. begin
  776. RegReadByInstruction := true;
  777. exit
  778. end;
  779. RS_EDX:
  780. if [Ch_REDX,Ch_RWEDX,Ch_MEDX,Ch_WRDX,Ch_RWRDX,Ch_MRDX]*Ch<>[] then
  781. begin
  782. RegReadByInstruction := true;
  783. exit
  784. end;
  785. RS_EBX:
  786. if [Ch_REBX,Ch_RWEBX,Ch_MEBX,Ch_WRBX,Ch_RWRBX,Ch_MRBX]*Ch<>[] then
  787. begin
  788. RegReadByInstruction := true;
  789. exit
  790. end;
  791. RS_ESP:
  792. if [Ch_RESP,Ch_RWESP,Ch_MESP,Ch_WRSP,Ch_RWRSP,Ch_MRSP]*Ch<>[] then
  793. begin
  794. RegReadByInstruction := true;
  795. exit
  796. end;
  797. RS_EBP:
  798. if [Ch_REBP,Ch_RWEBP,Ch_MEBP,Ch_WRBP,Ch_RWRBP,Ch_MRBP]*Ch<>[] then
  799. begin
  800. RegReadByInstruction := true;
  801. exit
  802. end;
  803. RS_ESI:
  804. if [Ch_RESI,Ch_RWESI,Ch_MESI,Ch_WRSI,Ch_RWRSI,Ch_MRSI]*Ch<>[] then
  805. begin
  806. RegReadByInstruction := true;
  807. exit
  808. end;
  809. RS_EDI:
  810. if [Ch_REDI,Ch_RWEDI,Ch_MEDI,Ch_WRDI,Ch_RWRDI,Ch_MRDI]*Ch<>[] then
  811. begin
  812. RegReadByInstruction := true;
  813. exit
  814. end;
  815. end;
  816. end;
  817. R_MMREGISTER:
  818. begin
  819. case getsupreg(reg) of
  820. RS_XMM0:
  821. if [Ch_RXMM0,Ch_RWXMM0,Ch_MXMM0]*Ch<>[] then
  822. begin
  823. RegReadByInstruction := true;
  824. exit
  825. end;
  826. end;
  827. end;
  828. else
  829. ;
  830. end;
  831. if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  832. begin
  833. if (Ch_RFLAGScc in Ch) and not(getsubreg(reg) in [R_SUBW,R_SUBD,R_SUBQ]) then
  834. begin
  835. case p.condition of
  836. C_A,C_NBE, { CF=0 and ZF=0 }
  837. C_BE,C_NA: { CF=1 or ZF=1 }
  838. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGCARRY,R_SUBFLAGZERO];
  839. C_AE,C_NB,C_NC, { CF=0 }
  840. C_B,C_NAE,C_C: { CF=1 }
  841. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGCARRY];
  842. C_NE,C_NZ, { ZF=0 }
  843. C_E,C_Z: { ZF=1 }
  844. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGZERO];
  845. C_G,C_NLE, { ZF=0 and SF=OF }
  846. C_LE,C_NG: { ZF=1 or SF<>OF }
  847. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGZERO,R_SUBFLAGSIGN,R_SUBFLAGOVERFLOW];
  848. C_GE,C_NL, { SF=OF }
  849. C_L,C_NGE: { SF<>OF }
  850. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGSIGN,R_SUBFLAGOVERFLOW];
  851. C_NO, { OF=0 }
  852. C_O: { OF=1 }
  853. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGOVERFLOW];
  854. C_NP,C_PO, { PF=0 }
  855. C_P,C_PE: { PF=1 }
  856. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGPARITY];
  857. C_NS, { SF=0 }
  858. C_S: { SF=1 }
  859. RegReadByInstruction:=getsubreg(reg) in [R_SUBFLAGSIGN];
  860. else
  861. internalerror(2017042701);
  862. end;
  863. if RegReadByInstruction then
  864. exit;
  865. end;
  866. case getsubreg(reg) of
  867. R_SUBW,R_SUBD,R_SUBQ:
  868. RegReadByInstruction :=
  869. [Ch_RCarryFlag,Ch_RParityFlag,Ch_RAuxiliaryFlag,Ch_RZeroFlag,Ch_RSignFlag,Ch_ROverflowFlag,
  870. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag,
  871. Ch_RDirFlag,Ch_RFlags,Ch_RWFlags,Ch_RFLAGScc]*Ch<>[];
  872. R_SUBFLAGCARRY:
  873. RegReadByInstruction:=[Ch_RCarryFlag,Ch_RWCarryFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  874. R_SUBFLAGPARITY:
  875. RegReadByInstruction:=[Ch_RParityFlag,Ch_RWParityFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  876. R_SUBFLAGAUXILIARY:
  877. RegReadByInstruction:=[Ch_RAuxiliaryFlag,Ch_RWAuxiliaryFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  878. R_SUBFLAGZERO:
  879. RegReadByInstruction:=[Ch_RZeroFlag,Ch_RWZeroFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  880. R_SUBFLAGSIGN:
  881. RegReadByInstruction:=[Ch_RSignFlag,Ch_RWSignFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  882. R_SUBFLAGOVERFLOW:
  883. RegReadByInstruction:=[Ch_ROverflowFlag,Ch_RWOverflowFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  884. R_SUBFLAGINTERRUPT:
  885. RegReadByInstruction:=[Ch_RFlags,Ch_RWFlags]*Ch<>[];
  886. R_SUBFLAGDIRECTION:
  887. RegReadByInstruction:=[Ch_RDirFlag,Ch_RFlags,Ch_RWFlags]*Ch<>[];
  888. else
  889. internalerror(2017042601);
  890. end;
  891. exit;
  892. end;
  893. if (Ch_NoReadIfEqualRegs in Ch) and (p.ops=2) and
  894. (p.oper[0]^.typ=top_reg) and (p.oper[1]^.typ=top_reg) and
  895. (p.oper[0]^.reg=p.oper[1]^.reg) then
  896. exit;
  897. if ([CH_RWOP1,CH_ROP1,CH_MOP1]*Ch<>[]) and reginop(reg,p.oper[0]^) then
  898. begin
  899. RegReadByInstruction := true;
  900. exit
  901. end;
  902. if ([Ch_RWOP2,Ch_ROP2,Ch_MOP2]*Ch<>[]) and reginop(reg,p.oper[1]^) then
  903. begin
  904. RegReadByInstruction := true;
  905. exit
  906. end;
  907. if ([Ch_RWOP3,Ch_ROP3,Ch_MOP3]*Ch<>[]) and reginop(reg,p.oper[2]^) then
  908. begin
  909. RegReadByInstruction := true;
  910. exit
  911. end;
  912. if ([Ch_RWOP4,Ch_ROP4,Ch_MOP4]*Ch<>[]) and reginop(reg,p.oper[3]^) then
  913. begin
  914. RegReadByInstruction := true;
  915. exit
  916. end;
  917. end;
  918. end;
  919. end;
  920. end;
  921. function TX86AsmOptimizer.RegInInstruction(Reg: TRegister; p1: tai): Boolean;
  922. begin
  923. result:=false;
  924. if p1.typ<>ait_instruction then
  925. exit;
  926. if (Ch_All in insprop[taicpu(p1).opcode].Ch) then
  927. exit(true);
  928. if (getregtype(reg)=R_INTREGISTER) and
  929. { change information for xmm movsd are not correct }
  930. ((taicpu(p1).opcode<>A_MOVSD) or (taicpu(p1).ops=0)) then
  931. begin
  932. { Handle instructions that behave differently depending on the size and operand count }
  933. case taicpu(p1).opcode of
  934. A_MUL, A_DIV, A_IDIV:
  935. if taicpu(p1).opsize = S_B then
  936. Result := (getsupreg(Reg) = RS_EAX)
  937. else
  938. Result := (getsupreg(Reg) in [RS_EAX, RS_EDX]);
  939. A_IMUL:
  940. if taicpu(p1).ops = 1 then
  941. begin
  942. if taicpu(p1).opsize = S_B then
  943. Result := (getsupreg(Reg) = RS_EAX)
  944. else
  945. Result := (getsupreg(Reg) in [RS_EAX, RS_EDX]);
  946. end;
  947. { If ops are greater than 1, call inherited method }
  948. else
  949. case getsupreg(reg) of
  950. { RS_EAX = RS_RAX on x86-64 }
  951. RS_EAX:
  952. result:=([Ch_REAX,Ch_RRAX,Ch_WEAX,Ch_WRAX,Ch_RWEAX,Ch_RWRAX,Ch_MEAX,Ch_MRAX]*insprop[taicpu(p1).opcode].Ch)<>[];
  953. RS_ECX:
  954. result:=([Ch_RECX,Ch_RRCX,Ch_WECX,Ch_WRCX,Ch_RWECX,Ch_RWRCX,Ch_MECX,Ch_MRCX]*insprop[taicpu(p1).opcode].Ch)<>[];
  955. RS_EDX:
  956. result:=([Ch_REDX,Ch_RRDX,Ch_WEDX,Ch_WRDX,Ch_RWEDX,Ch_RWRDX,Ch_MEDX,Ch_MRDX]*insprop[taicpu(p1).opcode].Ch)<>[];
  957. RS_EBX:
  958. result:=([Ch_REBX,Ch_RRBX,Ch_WEBX,Ch_WRBX,Ch_RWEBX,Ch_RWRBX,Ch_MEBX,Ch_MRBX]*insprop[taicpu(p1).opcode].Ch)<>[];
  959. RS_ESP:
  960. result:=([Ch_RESP,Ch_RRSP,Ch_WESP,Ch_WRSP,Ch_RWESP,Ch_RWRSP,Ch_MESP,Ch_MRSP]*insprop[taicpu(p1).opcode].Ch)<>[];
  961. RS_EBP:
  962. result:=([Ch_REBP,Ch_RRBP,Ch_WEBP,Ch_WRBP,Ch_RWEBP,Ch_RWRBP,Ch_MEBP,Ch_MRBP]*insprop[taicpu(p1).opcode].Ch)<>[];
  963. RS_ESI:
  964. result:=([Ch_RESI,Ch_RRSI,Ch_WESI,Ch_WRSI,Ch_RWESI,Ch_RWRSI,Ch_MESI,Ch_MRSI,Ch_RMemEDI]*insprop[taicpu(p1).opcode].Ch)<>[];
  965. RS_EDI:
  966. result:=([Ch_REDI,Ch_RRDI,Ch_WEDI,Ch_WRDI,Ch_RWEDI,Ch_RWRDI,Ch_MEDI,Ch_MRDI,Ch_WMemEDI]*insprop[taicpu(p1).opcode].Ch)<>[];
  967. else
  968. ;
  969. end;
  970. end;
  971. if result then
  972. exit;
  973. end
  974. else if getregtype(reg)=R_MMREGISTER then
  975. begin
  976. case getsupreg(reg) of
  977. RS_XMM0:
  978. result:=([Ch_RXMM0,Ch_WXMM0,Ch_RWXMM0,Ch_MXMM0]*insprop[taicpu(p1).opcode].Ch)<>[];
  979. else
  980. ;
  981. end;
  982. if result then
  983. exit;
  984. end
  985. else if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  986. begin
  987. if ([Ch_RFlags,Ch_WFlags,Ch_RWFlags,Ch_RFLAGScc]*insprop[taicpu(p1).opcode].Ch)<>[] then
  988. exit(true);
  989. case getsubreg(reg) of
  990. R_SUBFLAGCARRY:
  991. Result:=([Ch_RCarryFlag,Ch_RWCarryFlag,Ch_W0CarryFlag,Ch_W1CarryFlag,Ch_WCarryFlag,Ch_WUCarryFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  992. R_SUBFLAGPARITY:
  993. Result:=([Ch_RParityFlag,Ch_RWParityFlag,Ch_W0ParityFlag,Ch_W1ParityFlag,Ch_WParityFlag,Ch_WUParityFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  994. R_SUBFLAGAUXILIARY:
  995. Result:=([Ch_RAuxiliaryFlag,Ch_RWAuxiliaryFlag,Ch_W0AuxiliaryFlag,Ch_W1AuxiliaryFlag,Ch_WAuxiliaryFlag,Ch_WUAuxiliaryFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  996. R_SUBFLAGZERO:
  997. Result:=([Ch_RZeroFlag,Ch_RWZeroFlag,Ch_W0ZeroFlag,Ch_W1ZeroFlag,Ch_WZeroFlag,Ch_WUZeroFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  998. R_SUBFLAGSIGN:
  999. Result:=([Ch_RSignFlag,Ch_RWSignFlag,Ch_W0SignFlag,Ch_W1SignFlag,Ch_WSignFlag,Ch_WUSignFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  1000. R_SUBFLAGOVERFLOW:
  1001. Result:=([Ch_ROverflowFlag,Ch_RWOverflowFlag,Ch_W0OverflowFlag,Ch_W1OverflowFlag,Ch_WOverflowFlag,Ch_WUOverflowFlag]*insprop[taicpu(p1).opcode].Ch)<>[];
  1002. R_SUBFLAGINTERRUPT:
  1003. Result:=([Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags]*insprop[taicpu(p1).opcode].Ch)<>[];
  1004. R_SUBFLAGDIRECTION:
  1005. Result:=([Ch_RDirFlag,Ch_W0DirFlag,Ch_W1DirFlag,Ch_WFlags]*insprop[taicpu(p1).opcode].Ch)<>[];
  1006. R_SUBW,R_SUBD,R_SUBQ:
  1007. { Everything except the direction bits }
  1008. Result:=
  1009. ([Ch_RCarryFlag,Ch_RParityFlag,Ch_RAuxiliaryFlag,Ch_RZeroFlag,Ch_RSignFlag,Ch_ROverflowFlag,
  1010. Ch_WCarryFlag,Ch_WParityFlag,Ch_WAuxiliaryFlag,Ch_WZeroFlag,Ch_WSignFlag,Ch_WOverflowFlag,
  1011. Ch_W0CarryFlag,Ch_W0ParityFlag,Ch_W0AuxiliaryFlag,Ch_W0ZeroFlag,Ch_W0SignFlag,Ch_W0OverflowFlag,
  1012. Ch_W1CarryFlag,Ch_W1ParityFlag,Ch_W1AuxiliaryFlag,Ch_W1ZeroFlag,Ch_W1SignFlag,Ch_W1OverflowFlag,
  1013. Ch_WUCarryFlag,Ch_WUParityFlag,Ch_WUAuxiliaryFlag,Ch_WUZeroFlag,Ch_WUSignFlag,Ch_WUOverflowFlag,
  1014. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag
  1015. ]*insprop[taicpu(p1).opcode].Ch)<>[];
  1016. else
  1017. ;
  1018. end;
  1019. if result then
  1020. exit;
  1021. end
  1022. else if (getregtype(reg)=R_FPUREGISTER) and (Ch_FPU in insprop[taicpu(p1).opcode].Ch) then
  1023. exit(true);
  1024. Result:=inherited RegInInstruction(Reg, p1);
  1025. end;
  1026. function TX86AsmOptimizer.RegModifiedByInstruction(Reg: TRegister; p1: tai): boolean;
  1027. const
  1028. WriteOps: array[0..3] of set of TInsChange =
  1029. ([CH_RWOP1,CH_WOP1,CH_MOP1],
  1030. [Ch_RWOP2,Ch_WOP2,Ch_MOP2],
  1031. [Ch_RWOP3,Ch_WOP3,Ch_MOP3],
  1032. [Ch_RWOP4,Ch_WOP4,Ch_MOP4]);
  1033. var
  1034. OperIdx: Integer;
  1035. begin
  1036. Result := False;
  1037. if p1.typ <> ait_instruction then
  1038. exit;
  1039. with insprop[taicpu(p1).opcode] do
  1040. if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  1041. begin
  1042. case getsubreg(reg) of
  1043. R_SUBW,R_SUBD,R_SUBQ:
  1044. Result :=
  1045. [Ch_WCarryFlag,Ch_WParityFlag,Ch_WAuxiliaryFlag,Ch_WZeroFlag,Ch_WSignFlag,Ch_WOverflowFlag,
  1046. Ch_W0CarryFlag,Ch_W0ParityFlag,Ch_W0AuxiliaryFlag,Ch_W0ZeroFlag,Ch_W0SignFlag,Ch_W0OverflowFlag,
  1047. Ch_W1CarryFlag,Ch_W1ParityFlag,Ch_W1AuxiliaryFlag,Ch_W1ZeroFlag,Ch_W1SignFlag,Ch_W1OverflowFlag,
  1048. Ch_WUCarryFlag,Ch_WUParityFlag,Ch_WUAuxiliaryFlag,Ch_WUZeroFlag,Ch_WUSignFlag,Ch_WUOverflowFlag,
  1049. Ch_RWCarryFlag,Ch_RWParityFlag,Ch_RWAuxiliaryFlag,Ch_RWZeroFlag,Ch_RWSignFlag,Ch_RWOverflowFlag,
  1050. Ch_W0DirFlag,Ch_W1DirFlag,Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1051. R_SUBFLAGCARRY:
  1052. Result:=[Ch_WCarryFlag,Ch_W0CarryFlag,Ch_W1CarryFlag,Ch_WUCarryFlag,Ch_RWCarryFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1053. R_SUBFLAGPARITY:
  1054. Result:=[Ch_WParityFlag,Ch_W0ParityFlag,Ch_W1ParityFlag,Ch_WUParityFlag,Ch_RWParityFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1055. R_SUBFLAGAUXILIARY:
  1056. Result:=[Ch_WAuxiliaryFlag,Ch_W0AuxiliaryFlag,Ch_W1AuxiliaryFlag,Ch_WUAuxiliaryFlag,Ch_RWAuxiliaryFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1057. R_SUBFLAGZERO:
  1058. Result:=[Ch_WZeroFlag,Ch_W0ZeroFlag,Ch_W1ZeroFlag,Ch_WUZeroFlag,Ch_RWZeroFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1059. R_SUBFLAGSIGN:
  1060. Result:=[Ch_WSignFlag,Ch_W0SignFlag,Ch_W1SignFlag,Ch_WUSignFlag,Ch_RWSignFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1061. R_SUBFLAGOVERFLOW:
  1062. Result:=[Ch_WOverflowFlag,Ch_W0OverflowFlag,Ch_W1OverflowFlag,Ch_WUOverflowFlag,Ch_RWOverflowFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1063. R_SUBFLAGINTERRUPT:
  1064. Result:=[Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1065. R_SUBFLAGDIRECTION:
  1066. Result:=[Ch_W0DirFlag,Ch_W1DirFlag,Ch_WFlags,Ch_RWFlags]*Ch<>[];
  1067. else
  1068. internalerror(2017042602);
  1069. end;
  1070. exit;
  1071. end;
  1072. case taicpu(p1).opcode of
  1073. A_CALL:
  1074. { We could potentially set Result to False if the register in
  1075. question is non-volatile for the subroutine's calling convention,
  1076. but this would require detecting the calling convention in use and
  1077. also assuming that the routine doesn't contain malformed assembly
  1078. language, for example... so it could only be done under -O4 as it
  1079. would be considered a side-effect. [Kit] }
  1080. Result := True;
  1081. A_MOVSD:
  1082. { special handling for SSE MOVSD }
  1083. if (taicpu(p1).ops>0) then
  1084. begin
  1085. if taicpu(p1).ops<>2 then
  1086. internalerror(2017042703);
  1087. Result := (taicpu(p1).oper[1]^.typ=top_reg) and RegInOp(reg,taicpu(p1).oper[1]^);
  1088. end;
  1089. { VMOVSS and VMOVSD has two and three operand flavours, this cannot modelled by x86ins.dat
  1090. so fix it here (FK)
  1091. }
  1092. A_VMOVSS,
  1093. A_VMOVSD:
  1094. begin
  1095. Result := (taicpu(p1).ops=3) and (taicpu(p1).oper[2]^.typ=top_reg) and RegInOp(reg,taicpu(p1).oper[2]^);
  1096. exit;
  1097. end;
  1098. A_MUL, A_DIV, A_IDIV:
  1099. begin
  1100. if taicpu(p1).opsize = S_B then
  1101. Result := (getsupreg(Reg) = RS_EAX)
  1102. else
  1103. Result := (getsupreg(Reg) in [RS_EAX, RS_EDX]);
  1104. end;
  1105. A_IMUL:
  1106. begin
  1107. if taicpu(p1).ops = 1 then
  1108. begin
  1109. Result := (getsupreg(Reg) in [RS_EAX, RS_EDX]);
  1110. end
  1111. else
  1112. Result := (taicpu(p1).oper[taicpu(p1).ops-1]^.typ=top_reg) and RegInOp(reg,taicpu(p1).oper[taicpu(p1).ops-1]^);
  1113. Exit;
  1114. end;
  1115. else
  1116. ;
  1117. end;
  1118. if Result then
  1119. exit;
  1120. with insprop[taicpu(p1).opcode] do
  1121. begin
  1122. if getregtype(reg)=R_INTREGISTER then
  1123. begin
  1124. case getsupreg(reg) of
  1125. RS_EAX:
  1126. if [Ch_WEAX,Ch_RWEAX,Ch_MEAX,Ch_WRAX,Ch_RWRAX,Ch_MRAX]*Ch<>[] then
  1127. begin
  1128. Result := True;
  1129. exit
  1130. end;
  1131. RS_ECX:
  1132. if [Ch_WECX,Ch_RWECX,Ch_MECX,Ch_WRCX,Ch_RWRCX,Ch_MRCX]*Ch<>[] then
  1133. begin
  1134. Result := True;
  1135. exit
  1136. end;
  1137. RS_EDX:
  1138. if [Ch_WEDX,Ch_RWEDX,Ch_MEDX,Ch_WRDX,Ch_RWRDX,Ch_MRDX]*Ch<>[] then
  1139. begin
  1140. Result := True;
  1141. exit
  1142. end;
  1143. RS_EBX:
  1144. if [Ch_WEBX,Ch_RWEBX,Ch_MEBX,Ch_WRBX,Ch_RWRBX,Ch_MRBX]*Ch<>[] then
  1145. begin
  1146. Result := True;
  1147. exit
  1148. end;
  1149. RS_ESP:
  1150. if [Ch_WESP,Ch_RWESP,Ch_MESP,Ch_WRSP,Ch_RWRSP,Ch_MRSP]*Ch<>[] then
  1151. begin
  1152. Result := True;
  1153. exit
  1154. end;
  1155. RS_EBP:
  1156. if [Ch_WEBP,Ch_RWEBP,Ch_MEBP,Ch_WRBP,Ch_RWRBP,Ch_MRBP]*Ch<>[] then
  1157. begin
  1158. Result := True;
  1159. exit
  1160. end;
  1161. RS_ESI:
  1162. if [Ch_WESI,Ch_RWESI,Ch_MESI,Ch_WRSI,Ch_RWRSI,Ch_MRSI]*Ch<>[] then
  1163. begin
  1164. Result := True;
  1165. exit
  1166. end;
  1167. RS_EDI:
  1168. if [Ch_WEDI,Ch_RWEDI,Ch_MEDI,Ch_WRDI,Ch_RWRDI,Ch_MRDI]*Ch<>[] then
  1169. begin
  1170. Result := True;
  1171. exit
  1172. end;
  1173. end;
  1174. end;
  1175. for OperIdx := 0 to taicpu(p1).ops - 1 do
  1176. if (WriteOps[OperIdx]*Ch<>[]) and
  1177. { The register doesn't get modified inside a reference }
  1178. (taicpu(p1).oper[OperIdx]^.typ = top_reg) and
  1179. SuperRegistersEqual(reg,taicpu(p1).oper[OperIdx]^.reg) then
  1180. begin
  1181. Result := true;
  1182. exit
  1183. end;
  1184. end;
  1185. end;
  1186. function TX86AsmOptimizer.RefModifiedBetween(Ref: TReference; RefSize: ASizeInt; p1, p2: tai): Boolean;
  1187. const
  1188. WriteOps: array[0..3] of set of TInsChange =
  1189. ([CH_RWOP1,CH_WOP1,CH_MOP1],
  1190. [Ch_RWOP2,Ch_WOP2,Ch_MOP2],
  1191. [Ch_RWOP3,Ch_WOP3,Ch_MOP3],
  1192. [Ch_RWOP4,Ch_WOP4,Ch_MOP4]);
  1193. var
  1194. X: Integer;
  1195. CurrentP1Size: asizeint;
  1196. begin
  1197. Result := (
  1198. (Ref.base <> NR_NO) and
  1199. {$ifdef x86_64}
  1200. (Ref.base <> NR_RIP) and
  1201. {$endif x86_64}
  1202. RegModifiedBetween(Ref.base, p1, p2)
  1203. ) or
  1204. (
  1205. (Ref.index <> NR_NO) and
  1206. (Ref.index <> Ref.base) and
  1207. RegModifiedBetween(Ref.index, p1, p2)
  1208. );
  1209. { Now check to see if the memory itself is written to }
  1210. if not Result then
  1211. begin
  1212. while assigned(p1) and assigned(p2) and GetNextInstruction(p1,p1) and (p1<>p2) do
  1213. if p1.typ = ait_instruction then
  1214. begin
  1215. CurrentP1Size := topsize2memsize[taicpu(p1).opsize] shr 3; { Convert to bytes }
  1216. with insprop[taicpu(p1).opcode] do
  1217. for X := 0 to taicpu(p1).ops - 1 do
  1218. if (taicpu(p1).oper[X]^.typ = top_ref) and
  1219. RefsAlmostEqual(Ref, taicpu(p1).oper[X]^.ref^) and
  1220. { Catch any potential overlaps }
  1221. (
  1222. (RefSize = 0) or
  1223. ((taicpu(p1).oper[X]^.ref^.offset - Ref.offset) < RefSize)
  1224. ) and
  1225. (
  1226. (CurrentP1Size = 0) or
  1227. ((Ref.offset - taicpu(p1).oper[X]^.ref^.offset) < CurrentP1Size)
  1228. ) and
  1229. { Reference is used, but does the instruction write to it? }
  1230. (
  1231. (Ch_All in Ch) or
  1232. ((WriteOps[X] * Ch) <> [])
  1233. ) then
  1234. begin
  1235. Result := True;
  1236. Break;
  1237. end;
  1238. end;
  1239. end;
  1240. end;
  1241. {$ifdef DEBUG_AOPTCPU}
  1242. procedure TX86AsmOptimizer.DebugMsg(const s: string;p : tai);
  1243. begin
  1244. asml.insertbefore(tai_comment.Create(strpnew(s)), p);
  1245. end;
  1246. function debug_tostr(i: tcgint): string; inline;
  1247. begin
  1248. Result := tostr(i);
  1249. end;
  1250. function debug_hexstr(i: tcgint): string;
  1251. begin
  1252. Result := '0x';
  1253. case i of
  1254. 0..$FF:
  1255. Result := Result + hexstr(i, 2);
  1256. $100..$FFFF:
  1257. Result := Result + hexstr(i, 4);
  1258. $10000..$FFFFFF:
  1259. Result := Result + hexstr(i, 6);
  1260. $1000000..$FFFFFFFF:
  1261. Result := Result + hexstr(i, 8);
  1262. else
  1263. Result := Result + hexstr(i, 16);
  1264. end;
  1265. end;
  1266. function debug_regname(r: TRegister): string; inline;
  1267. begin
  1268. Result := '%' + std_regname(r);
  1269. end;
  1270. { Debug output function - creates a string representation of an operator }
  1271. function debug_operstr(oper: TOper): string;
  1272. begin
  1273. case oper.typ of
  1274. top_const:
  1275. Result := '$' + debug_tostr(oper.val);
  1276. top_reg:
  1277. Result := debug_regname(oper.reg);
  1278. top_ref:
  1279. begin
  1280. if oper.ref^.offset <> 0 then
  1281. Result := debug_tostr(oper.ref^.offset) + '('
  1282. else
  1283. Result := '(';
  1284. if (oper.ref^.base <> NR_INVALID) and (oper.ref^.base <> NR_NO) then
  1285. begin
  1286. Result := Result + debug_regname(oper.ref^.base);
  1287. if (oper.ref^.index <> NR_INVALID) and (oper.ref^.index <> NR_NO) then
  1288. Result := Result + ',' + debug_regname(oper.ref^.index);
  1289. end
  1290. else
  1291. if (oper.ref^.index <> NR_INVALID) and (oper.ref^.index <> NR_NO) then
  1292. Result := Result + debug_regname(oper.ref^.index);
  1293. if (oper.ref^.scalefactor > 1) then
  1294. Result := Result + ',' + debug_tostr(oper.ref^.scalefactor) + ')'
  1295. else
  1296. Result := Result + ')';
  1297. end;
  1298. else
  1299. Result := '[UNKNOWN]';
  1300. end;
  1301. end;
  1302. function debug_op2str(opcode: tasmop): string; inline;
  1303. begin
  1304. Result := std_op2str[opcode];
  1305. end;
  1306. function debug_opsize2str(opsize: topsize): string; inline;
  1307. begin
  1308. Result := gas_opsize2str[opsize];
  1309. end;
  1310. {$else DEBUG_AOPTCPU}
  1311. procedure TX86AsmOptimizer.DebugMsg(const s: string;p : tai);inline;
  1312. begin
  1313. end;
  1314. function debug_tostr(i: tcgint): string; inline;
  1315. begin
  1316. Result := '';
  1317. end;
  1318. function debug_hexstr(i: tcgint): string; inline;
  1319. begin
  1320. Result := '';
  1321. end;
  1322. function debug_regname(r: TRegister): string; inline;
  1323. begin
  1324. Result := '';
  1325. end;
  1326. function debug_operstr(oper: TOper): string; inline;
  1327. begin
  1328. Result := '';
  1329. end;
  1330. function debug_op2str(opcode: tasmop): string; inline;
  1331. begin
  1332. Result := '';
  1333. end;
  1334. function debug_opsize2str(opsize: topsize): string; inline;
  1335. begin
  1336. Result := '';
  1337. end;
  1338. {$endif DEBUG_AOPTCPU}
  1339. class function TX86AsmOptimizer.IsMOVZXAcceptable: Boolean; inline;
  1340. begin
  1341. {$ifdef x86_64}
  1342. { Always fine on x86-64 }
  1343. Result := True;
  1344. {$else x86_64}
  1345. Result :=
  1346. {$ifdef i8086}
  1347. (current_settings.cputype >= cpu_386) and
  1348. {$endif i8086}
  1349. (
  1350. { Always accept if optimising for size }
  1351. (cs_opt_size in current_settings.optimizerswitches) or
  1352. { From the Pentium II onwards, MOVZX only takes 1 cycle. [Kit] }
  1353. (current_settings.optimizecputype >= cpu_Pentium2)
  1354. );
  1355. {$endif x86_64}
  1356. end;
  1357. { Attempts to allocate a volatile integer register for use between p and hp,
  1358. using AUsedRegs for the current register usage information. Returns NR_NO
  1359. if no free register could be found }
  1360. function TX86AsmOptimizer.GetIntRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  1361. var
  1362. RegSet: TCPURegisterSet;
  1363. CurrentSuperReg: Integer;
  1364. CurrentReg: TRegister;
  1365. Currentp: tai;
  1366. Breakout: Boolean;
  1367. begin
  1368. Result := NR_NO;
  1369. RegSet :=
  1370. paramanager.get_volatile_registers_int(current_procinfo.procdef.proccalloption) +
  1371. current_procinfo.saved_regs_int;
  1372. (*
  1373. { Don't use the frame register unless explicitly allowed (fixes i40111) }
  1374. if ([cs_useebp, cs_userbp] * current_settings.optimizerswitches) = [] then
  1375. Exclude(RegSet, RS_FRAME_POINTER_REG);
  1376. *)
  1377. for CurrentSuperReg in RegSet do
  1378. begin
  1379. CurrentReg := newreg(R_INTREGISTER, TSuperRegister(CurrentSuperReg), RegSize);
  1380. if not AUsedRegs[R_INTREGISTER].IsUsed(CurrentReg)
  1381. {$if defined(i386) or defined(i8086)}
  1382. { If the target size is 8-bit, make sure we can actually encode it }
  1383. and (
  1384. (RegSize >= R_SUBW) or { Not R_SUBL or R_SUBH }
  1385. (GetSupReg(CurrentReg) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX])
  1386. )
  1387. {$endif i386 or i8086}
  1388. then
  1389. begin
  1390. Currentp := p;
  1391. Breakout := False;
  1392. while not Breakout and GetNextInstruction(Currentp, Currentp) and (Currentp <> hp) do
  1393. begin
  1394. case Currentp.typ of
  1395. ait_instruction:
  1396. begin
  1397. if RegInInstruction(CurrentReg, Currentp) then
  1398. begin
  1399. Breakout := True;
  1400. Break;
  1401. end;
  1402. { Cannot allocate across an unconditional jump }
  1403. if is_calljmpuncondret(taicpu(Currentp).opcode) then
  1404. Exit;
  1405. end;
  1406. ait_marker:
  1407. { Don't try anything more if a marker is hit }
  1408. Exit;
  1409. ait_regalloc:
  1410. if (tai_regalloc(Currentp).ratype <> ra_dealloc) and SuperRegistersEqual(CurrentReg, tai_regalloc(Currentp).reg) then
  1411. begin
  1412. Breakout := True;
  1413. Break;
  1414. end;
  1415. else
  1416. ;
  1417. end;
  1418. end;
  1419. if Breakout then
  1420. { Try the next register }
  1421. Continue;
  1422. { We have a free register available }
  1423. Result := CurrentReg;
  1424. if not DontAlloc then
  1425. AllocRegBetween(CurrentReg, p, hp, AUsedRegs);
  1426. Exit;
  1427. end;
  1428. end;
  1429. end;
  1430. { Attempts to allocate a volatile MM register for use between p and hp,
  1431. using AUsedRegs for the current register usage information. Returns NR_NO
  1432. if no free register could be found }
  1433. function TX86AsmOptimizer.GetMMRegisterBetween(RegSize: TSubRegister; var AUsedRegs: TAllUsedRegs; p, hp: tai; DontAlloc: Boolean = False): TRegister;
  1434. var
  1435. RegSet: TCPURegisterSet;
  1436. CurrentSuperReg: Integer;
  1437. CurrentReg: TRegister;
  1438. Currentp: tai;
  1439. Breakout: Boolean;
  1440. begin
  1441. Result := NR_NO;
  1442. RegSet :=
  1443. paramanager.get_volatile_registers_mm(current_procinfo.procdef.proccalloption) +
  1444. current_procinfo.saved_regs_mm;
  1445. for CurrentSuperReg in RegSet do
  1446. begin
  1447. CurrentReg := newreg(R_MMREGISTER, TSuperRegister(CurrentSuperReg), RegSize);
  1448. if not AUsedRegs[R_MMREGISTER].IsUsed(CurrentReg) then
  1449. begin
  1450. Currentp := p;
  1451. Breakout := False;
  1452. while not Breakout and GetNextInstruction(Currentp, Currentp) and (Currentp <> hp) do
  1453. begin
  1454. case Currentp.typ of
  1455. ait_instruction:
  1456. begin
  1457. if RegInInstruction(CurrentReg, Currentp) then
  1458. begin
  1459. Breakout := True;
  1460. Break;
  1461. end;
  1462. { Cannot allocate across an unconditional jump }
  1463. if is_calljmpuncondret(taicpu(Currentp).opcode) then
  1464. Exit;
  1465. end;
  1466. ait_marker:
  1467. { Don't try anything more if a marker is hit }
  1468. Exit;
  1469. ait_regalloc:
  1470. if (tai_regalloc(Currentp).ratype <> ra_dealloc) and SuperRegistersEqual(CurrentReg, tai_regalloc(Currentp).reg) then
  1471. begin
  1472. Breakout := True;
  1473. Break;
  1474. end;
  1475. else
  1476. ;
  1477. end;
  1478. end;
  1479. if Breakout then
  1480. { Try the next register }
  1481. Continue;
  1482. { We have a free register available }
  1483. Result := CurrentReg;
  1484. if not DontAlloc then
  1485. AllocRegBetween(CurrentReg, p, hp, AUsedRegs);
  1486. Exit;
  1487. end;
  1488. end;
  1489. end;
  1490. class function TX86AsmOptimizer.Reg1WriteOverwritesReg2Entirely(reg1, reg2: tregister): boolean;
  1491. begin
  1492. if not SuperRegistersEqual(reg1,reg2) then
  1493. exit(false);
  1494. if getregtype(reg1)<>R_INTREGISTER then
  1495. exit(true); {because SuperRegisterEqual is true}
  1496. case getsubreg(reg1) of
  1497. { A write to R_SUBL doesn't change R_SUBH and if reg2 is R_SUBW or
  1498. higher, it preserves the high bits, so the new value depends on
  1499. reg2's previous value. In other words, it is equivalent to doing:
  1500. reg2 := (reg2 and $ffffff00) or byte(reg1); }
  1501. R_SUBL:
  1502. exit(getsubreg(reg2)=R_SUBL);
  1503. { A write to R_SUBH doesn't change R_SUBL and if reg2 is R_SUBW or
  1504. higher, it actually does a:
  1505. reg2 := (reg2 and $ffff00ff) or (reg1 and $ff00); }
  1506. R_SUBH:
  1507. exit(getsubreg(reg2)=R_SUBH);
  1508. { If reg2 is R_SUBD or larger, a write to R_SUBW preserves the high 16
  1509. bits of reg2:
  1510. reg2 := (reg2 and $ffff0000) or word(reg1); }
  1511. R_SUBW:
  1512. exit(getsubreg(reg2) in [R_SUBL,R_SUBH,R_SUBW]);
  1513. { a write to R_SUBD always overwrites every other subregister,
  1514. because it clears the high 32 bits of R_SUBQ on x86_64 }
  1515. R_SUBD,
  1516. R_SUBQ:
  1517. exit(true);
  1518. else
  1519. internalerror(2017042801);
  1520. end;
  1521. end;
  1522. class function TX86AsmOptimizer.Reg1ReadDependsOnReg2(reg1, reg2: tregister): boolean;
  1523. begin
  1524. if not SuperRegistersEqual(reg1,reg2) then
  1525. exit(false);
  1526. if getregtype(reg1)<>R_INTREGISTER then
  1527. exit(true); {because SuperRegisterEqual is true}
  1528. case getsubreg(reg1) of
  1529. R_SUBL:
  1530. exit(getsubreg(reg2)<>R_SUBH);
  1531. R_SUBH:
  1532. exit(getsubreg(reg2)<>R_SUBL);
  1533. R_SUBW,
  1534. R_SUBD,
  1535. R_SUBQ:
  1536. exit(true);
  1537. else
  1538. internalerror(2017042802);
  1539. end;
  1540. end;
  1541. function TX86AsmOptimizer.PrePeepholeOptSxx(var p : tai) : boolean;
  1542. var
  1543. hp1 : tai;
  1544. l : TCGInt;
  1545. begin
  1546. result:=false;
  1547. if not(GetNextInstruction(p, hp1)) then
  1548. exit;
  1549. { changes the code sequence
  1550. shr/sar const1, x
  1551. shl const2, x
  1552. to
  1553. either "sar/and", "shl/and" or just "and" depending on const1 and const2 }
  1554. if (taicpu(p).oper[0]^.typ = top_const) and
  1555. MatchInstruction(hp1,A_SHL,[]) and
  1556. (taicpu(hp1).oper[0]^.typ = top_const) and
  1557. (taicpu(hp1).opsize = taicpu(p).opsize) and
  1558. (taicpu(hp1).oper[1]^.typ = taicpu(p).oper[1]^.typ) and
  1559. OpsEqual(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^) then
  1560. begin
  1561. if (taicpu(p).oper[0]^.val > taicpu(hp1).oper[0]^.val) and
  1562. not(cs_opt_size in current_settings.optimizerswitches)
  1563. {$ifdef x86_64}
  1564. and (
  1565. (taicpu(p).opsize <> S_Q) or
  1566. { 64-bit AND can only store signed 32-bit immediates }
  1567. (taicpu(p).oper[0]^.val < 32)
  1568. )
  1569. {$endif x86_64}
  1570. then
  1571. begin
  1572. { shr/sar const1, %reg
  1573. shl const2, %reg
  1574. with const1 > const2 }
  1575. DebugMsg(SPeepholeOptimization + 'SxrShl2SxrAnd 1 done',p);
  1576. taicpu(p).loadConst(0,taicpu(p).oper[0]^.val-taicpu(hp1).oper[0]^.val);
  1577. taicpu(hp1).opcode := A_AND;
  1578. l := (1 shl (taicpu(hp1).oper[0]^.val)) - 1;
  1579. case taicpu(p).opsize Of
  1580. S_B: taicpu(hp1).loadConst(0,l Xor $ff);
  1581. S_W: taicpu(hp1).loadConst(0,l Xor $ffff);
  1582. S_L: taicpu(hp1).loadConst(0,l Xor tcgint($ffffffff));
  1583. S_Q: taicpu(hp1).loadConst(0,l Xor tcgint($ffffffffffffffff));
  1584. else
  1585. Internalerror(2017050703)
  1586. end;
  1587. end
  1588. else if (taicpu(p).oper[0]^.val<taicpu(hp1).oper[0]^.val) and
  1589. not(cs_opt_size in current_settings.optimizerswitches)
  1590. {$ifdef x86_64}
  1591. and (
  1592. (taicpu(p).opsize <> S_Q) or
  1593. { 64-bit AND can only store signed 32-bit immediates }
  1594. (taicpu(p).oper[0]^.val < 32)
  1595. )
  1596. {$endif x86_64}
  1597. then
  1598. begin
  1599. { shr/sar const1, %reg
  1600. shl const2, %reg
  1601. with const1 < const2 }
  1602. DebugMsg(SPeepholeOptimization + 'SxrShl2SxrAnd 2 done',p);
  1603. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val-taicpu(p).oper[0]^.val);
  1604. taicpu(p).opcode := A_AND;
  1605. l := (1 shl (taicpu(p).oper[0]^.val))-1;
  1606. case taicpu(p).opsize Of
  1607. S_B: taicpu(p).loadConst(0,l Xor $ff);
  1608. S_W: taicpu(p).loadConst(0,l Xor $ffff);
  1609. S_L: taicpu(p).loadConst(0,l Xor tcgint($ffffffff));
  1610. S_Q: taicpu(p).loadConst(0,l Xor tcgint($ffffffffffffffff));
  1611. else
  1612. Internalerror(2017050702)
  1613. end;
  1614. end
  1615. else if (taicpu(p).oper[0]^.val = taicpu(hp1).oper[0]^.val)
  1616. {$ifdef x86_64}
  1617. and (
  1618. (taicpu(p).opsize <> S_Q) or
  1619. { 64-bit AND can only store signed 32-bit immediates }
  1620. (taicpu(p).oper[0]^.val < 32)
  1621. )
  1622. {$endif x86_64}
  1623. then
  1624. begin
  1625. { shr/sar const1, %reg
  1626. shl const2, %reg
  1627. with const1 = const2 }
  1628. DebugMsg(SPeepholeOptimization + 'SxrShl2And done',p);
  1629. taicpu(p).opcode := A_AND;
  1630. l := (1 shl (taicpu(p).oper[0]^.val))-1;
  1631. case taicpu(p).opsize Of
  1632. S_B: taicpu(p).loadConst(0,l Xor $ff);
  1633. S_W: taicpu(p).loadConst(0,l Xor $ffff);
  1634. S_L: taicpu(p).loadConst(0,l Xor tcgint($ffffffff));
  1635. S_Q: taicpu(p).loadConst(0,l Xor tcgint($ffffffffffffffff));
  1636. else
  1637. Internalerror(2017050701)
  1638. end;
  1639. RemoveInstruction(hp1);
  1640. end;
  1641. end;
  1642. end;
  1643. function TX86AsmOptimizer.PrePeepholeOptIMUL(var p : tai) : boolean;
  1644. var
  1645. opsize : topsize;
  1646. hp1, hp2 : tai;
  1647. tmpref : treference;
  1648. ShiftValue : Cardinal;
  1649. BaseValue : TCGInt;
  1650. begin
  1651. result:=false;
  1652. opsize:=taicpu(p).opsize;
  1653. { changes certain "imul const, %reg"'s to lea sequences }
  1654. if (MatchOpType(taicpu(p),top_const,top_reg) or
  1655. MatchOpType(taicpu(p),top_const,top_reg,top_reg)) and
  1656. (opsize in [S_L{$ifdef x86_64},S_Q{$endif x86_64}]) then
  1657. if (taicpu(p).oper[0]^.val = 1) then
  1658. if (taicpu(p).ops = 2) then
  1659. { remove "imul $1, reg" }
  1660. begin
  1661. DebugMsg(SPeepholeOptimization + 'Imul2Nop done',p);
  1662. Result := RemoveCurrentP(p);
  1663. end
  1664. else
  1665. { change "imul $1, reg1, reg2" to "mov reg1, reg2" }
  1666. begin
  1667. hp1 := taicpu.Op_Reg_Reg(A_MOV, opsize, taicpu(p).oper[1]^.reg,taicpu(p).oper[2]^.reg);
  1668. taicpu(hp1).fileinfo := taicpu(p).fileinfo;
  1669. asml.InsertAfter(hp1, p);
  1670. DebugMsg(SPeepholeOptimization + 'Imul2Mov done',p);
  1671. RemoveCurrentP(p, hp1);
  1672. Result := True;
  1673. end
  1674. else if ((taicpu(p).ops <= 2) or
  1675. (taicpu(p).oper[2]^.typ = Top_Reg)) and
  1676. not(cs_opt_size in current_settings.optimizerswitches) and
  1677. (not(GetNextInstruction(p, hp1)) or
  1678. not((tai(hp1).typ = ait_instruction) and
  1679. ((taicpu(hp1).opcode=A_Jcc) and
  1680. (taicpu(hp1).condition in [C_O,C_NO])))) then
  1681. begin
  1682. {
  1683. imul X, reg1, reg2 to
  1684. lea (reg1,reg1,Y), reg2
  1685. shl ZZ,reg2
  1686. imul XX, reg1 to
  1687. lea (reg1,reg1,YY), reg1
  1688. shl ZZ,reg2
  1689. This optimziation makes sense for pretty much every x86, except the VIA Nano3000: it has IMUL latency 2, lea/shl pair as well,
  1690. it does not exist as a separate optimization target in FPC though.
  1691. This optimziation can be applied as long as only two bits are set in the constant and those two bits are separated by
  1692. at most two zeros
  1693. }
  1694. reference_reset(tmpref,1,[]);
  1695. if (PopCnt(QWord(taicpu(p).oper[0]^.val))=2) and (BsrQWord(taicpu(p).oper[0]^.val)-BsfQWord(taicpu(p).oper[0]^.val)<=3) then
  1696. begin
  1697. ShiftValue:=BsfQWord(taicpu(p).oper[0]^.val);
  1698. BaseValue:=taicpu(p).oper[0]^.val shr ShiftValue;
  1699. TmpRef.base := taicpu(p).oper[1]^.reg;
  1700. TmpRef.index := taicpu(p).oper[1]^.reg;
  1701. if not(BaseValue in [3,5,9]) then
  1702. Internalerror(2018110101);
  1703. TmpRef.ScaleFactor := BaseValue-1;
  1704. if (taicpu(p).ops = 2) then
  1705. hp1 := taicpu.op_ref_reg(A_LEA, opsize, TmpRef, taicpu(p).oper[1]^.reg)
  1706. else
  1707. hp1 := taicpu.op_ref_reg(A_LEA, opsize, TmpRef, taicpu(p).oper[2]^.reg);
  1708. AsmL.InsertAfter(hp1,p);
  1709. DebugMsg(SPeepholeOptimization + 'Imul2LeaShl done',p);
  1710. taicpu(hp1).fileinfo:=taicpu(p).fileinfo;
  1711. RemoveCurrentP(p, hp1);
  1712. if ShiftValue>0 then
  1713. begin
  1714. hp2 := taicpu.op_const_reg(A_SHL, opsize, ShiftValue, taicpu(hp1).oper[1]^.reg);
  1715. AsmL.InsertAfter(hp2,hp1);
  1716. taicpu(hp2).fileinfo:=taicpu(hp1).fileinfo;
  1717. end;
  1718. Result := True;
  1719. end;
  1720. end;
  1721. end;
  1722. function TX86AsmOptimizer.PrePeepholeOptAND(var p : tai) : boolean;
  1723. begin
  1724. Result := False;
  1725. if MatchOperand(taicpu(p).oper[0]^, 0) and
  1726. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  1727. begin
  1728. DebugMsg(SPeepholeOptimization + 'AND 0 -> MOV 0', p);
  1729. taicpu(p).opcode := A_MOV;
  1730. Result := True;
  1731. end;
  1732. end;
  1733. function TX86AsmOptimizer.RegLoadedWithNewValue(reg: tregister; hp: tai): boolean;
  1734. var
  1735. p: taicpu absolute hp; { Implicit typecast }
  1736. i: Integer;
  1737. begin
  1738. Result := False;
  1739. if not assigned(hp) or
  1740. (hp.typ <> ait_instruction) then
  1741. Exit;
  1742. Prefetch(insprop[p.opcode]);
  1743. if SuperRegistersEqual(reg,NR_DEFAULTFLAGS) then
  1744. with insprop[p.opcode] do
  1745. begin
  1746. case getsubreg(reg) of
  1747. R_SUBW,R_SUBD,R_SUBQ:
  1748. Result:=
  1749. { ZF, CF, OF, SF, PF and AF must all be set in some way (ordered so the most
  1750. uncommon flags are checked first }
  1751. ([Ch_W0AuxiliaryFlag,Ch_W1AuxiliaryFlag,Ch_WAuxiliaryFlag,Ch_WUAuxiliaryFlag,Ch_WFlags] * Ch <> []) and
  1752. ([Ch_W0ParityFlag,Ch_W1ParityFlag,Ch_WParityFlag,Ch_WUParityFlag,Ch_WFlags]*Ch <> []) and
  1753. ([Ch_W0SignFlag,Ch_W1SignFlag,Ch_WSignFlag,Ch_WUSignFlag,Ch_WFlags]*Ch <> []) and
  1754. ([Ch_W0OverflowFlag,Ch_W1OverflowFlag,Ch_WOverflowFlag,Ch_WUOverflowFlag,Ch_WFlags]*Ch <> []) and
  1755. ([Ch_W0CarryFlag,Ch_W1CarryFlag,Ch_WCarryFlag,Ch_WUCarryFlag,Ch_WFlags]*Ch <> []) and
  1756. ([Ch_W0ZeroFlag,Ch_W1ZeroFlag,Ch_WZeroFlag,Ch_WUZeroFlag,Ch_WFlags]*Ch <> []);
  1757. R_SUBFLAGCARRY:
  1758. Result:=[Ch_W0CarryFlag,Ch_W1CarryFlag,Ch_WCarryFlag,Ch_WUCarryFlag,Ch_WFlags]*Ch<>[];
  1759. R_SUBFLAGPARITY:
  1760. Result:=[Ch_W0ParityFlag,Ch_W1ParityFlag,Ch_WParityFlag,Ch_WUParityFlag,Ch_WFlags]*Ch<>[];
  1761. R_SUBFLAGAUXILIARY:
  1762. Result:=[Ch_W0AuxiliaryFlag,Ch_W1AuxiliaryFlag,Ch_WAuxiliaryFlag,Ch_WUAuxiliaryFlag,Ch_WFlags]*Ch<>[];
  1763. R_SUBFLAGZERO:
  1764. Result:=[Ch_W0ZeroFlag,Ch_W1ZeroFlag,Ch_WZeroFlag,Ch_WUZeroFlag,Ch_WFlags]*Ch<>[];
  1765. R_SUBFLAGSIGN:
  1766. Result:=[Ch_W0SignFlag,Ch_W1SignFlag,Ch_WSignFlag,Ch_WUSignFlag,Ch_WFlags]*Ch<>[];
  1767. R_SUBFLAGOVERFLOW:
  1768. Result:=[Ch_W0OverflowFlag,Ch_W1OverflowFlag,Ch_WOverflowFlag,Ch_WUOverflowFlag,Ch_WFlags]*Ch<>[];
  1769. R_SUBFLAGINTERRUPT:
  1770. Result:=[Ch_W0IntFlag,Ch_W1IntFlag,Ch_WFlags]*Ch<>[];
  1771. R_SUBFLAGDIRECTION:
  1772. Result:=[Ch_W0DirFlag,Ch_W1DirFlag,Ch_WFlags]*Ch<>[];
  1773. else
  1774. internalerror(2017050501);
  1775. end;
  1776. exit;
  1777. end;
  1778. { Handle special cases first }
  1779. case p.opcode of
  1780. A_MOV, A_MOVZX, A_MOVSX, A_LEA, A_VMOVSS, A_VMOVSD, A_VMOVAPD,
  1781. A_VMOVAPS, A_VMOVQ, A_MOVSS, A_MOVSD, A_MOVQ, A_MOVAPD, A_MOVAPS:
  1782. begin
  1783. Result :=
  1784. (p.ops=2) and { A_MOVSD can have zero operands, so this check is needed }
  1785. (p.oper[1]^.typ = top_reg) and
  1786. (Reg1WriteOverwritesReg2Entirely(p.oper[1]^.reg,reg)) and
  1787. (
  1788. (p.oper[0]^.typ = top_const) or
  1789. (
  1790. (p.oper[0]^.typ = top_reg) and
  1791. not(Reg1ReadDependsOnReg2(p.oper[0]^.reg,reg))
  1792. ) or (
  1793. (p.oper[0]^.typ = top_ref) and
  1794. not RegInRef(reg,p.oper[0]^.ref^)
  1795. )
  1796. );
  1797. end;
  1798. A_MUL, A_IMUL:
  1799. Result :=
  1800. (
  1801. (p.ops=3) and { IMUL only }
  1802. (Reg1WriteOverwritesReg2Entirely(p.oper[2]^.reg,reg)) and
  1803. (
  1804. (
  1805. (p.oper[1]^.typ=top_reg) and
  1806. not Reg1ReadDependsOnReg2(p.oper[1]^.reg,reg)
  1807. ) or (
  1808. (p.oper[1]^.typ=top_ref) and
  1809. not RegInRef(reg,p.oper[1]^.ref^)
  1810. )
  1811. )
  1812. ) or (
  1813. (
  1814. (p.ops=1) and
  1815. (
  1816. (
  1817. (
  1818. (p.oper[0]^.typ=top_reg) and
  1819. not Reg1ReadDependsOnReg2(p.oper[0]^.reg,reg)
  1820. )
  1821. ) or (
  1822. (p.oper[0]^.typ=top_ref) and
  1823. not RegInRef(reg,p.oper[0]^.ref^)
  1824. )
  1825. ) and (
  1826. (
  1827. (p.opsize=S_B) and
  1828. Reg1WriteOverwritesReg2Entirely(NR_AX,reg) and
  1829. not Reg1ReadDependsOnReg2(NR_AL,reg)
  1830. ) or (
  1831. (p.opsize=S_W) and
  1832. Reg1WriteOverwritesReg2Entirely(NR_DX,reg)
  1833. ) or (
  1834. (p.opsize=S_L) and
  1835. Reg1WriteOverwritesReg2Entirely(NR_EDX,reg)
  1836. {$ifdef x86_64}
  1837. ) or (
  1838. (p.opsize=S_Q) and
  1839. Reg1WriteOverwritesReg2Entirely(NR_RDX,reg)
  1840. {$endif x86_64}
  1841. )
  1842. )
  1843. )
  1844. );
  1845. A_CBW:
  1846. Result := Reg1WriteOverwritesReg2Entirely(NR_AX,reg) and not(Reg1ReadDependsOnReg2(NR_AL,reg));
  1847. {$ifndef x86_64}
  1848. A_LDS:
  1849. Result := (reg=NR_DS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1850. A_LES:
  1851. Result := (reg=NR_ES) and not(RegInRef(reg,p.oper[0]^.ref^));
  1852. {$endif not x86_64}
  1853. A_LFS:
  1854. Result := (reg=NR_FS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1855. A_LGS:
  1856. Result := (reg=NR_GS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1857. A_LSS:
  1858. Result := (reg=NR_SS) and not(RegInRef(reg,p.oper[0]^.ref^));
  1859. A_LAHF{$ifndef x86_64}, A_AAM{$endif not x86_64}:
  1860. Result := Reg1WriteOverwritesReg2Entirely(NR_AH,reg);
  1861. A_LODSB:
  1862. Result := Reg1WriteOverwritesReg2Entirely(NR_AL,reg);
  1863. A_LODSW:
  1864. Result := Reg1WriteOverwritesReg2Entirely(NR_AX,reg);
  1865. {$ifdef x86_64}
  1866. A_LODSQ:
  1867. Result := Reg1WriteOverwritesReg2Entirely(NR_RAX,reg);
  1868. {$endif x86_64}
  1869. A_LODSD:
  1870. Result := Reg1WriteOverwritesReg2Entirely(NR_EAX,reg);
  1871. A_FSTSW, A_FNSTSW:
  1872. Result := (p.oper[0]^.typ=top_reg) and Reg1WriteOverwritesReg2Entirely(p.oper[0]^.reg,reg);
  1873. else
  1874. begin
  1875. with insprop[p.opcode] do
  1876. begin
  1877. if (
  1878. { xor %reg,%reg etc. is classed as a new value }
  1879. (([Ch_NoReadIfEqualRegs]*Ch)<>[]) and
  1880. MatchOpType(p, top_reg, top_reg) and
  1881. (p.oper[0]^.reg = p.oper[1]^.reg) and
  1882. Reg1WriteOverwritesReg2Entirely(p.oper[1]^.reg,reg)
  1883. ) then
  1884. begin
  1885. Result := True;
  1886. Exit;
  1887. end;
  1888. { Make sure the entire register is overwritten }
  1889. if (getregtype(reg) = R_INTREGISTER) then
  1890. begin
  1891. if (p.ops > 0) then
  1892. begin
  1893. if RegInOp(reg, p.oper[0]^) then
  1894. begin
  1895. if (p.oper[0]^.typ = top_ref) then
  1896. begin
  1897. if RegInRef(reg, p.oper[0]^.ref^) then
  1898. begin
  1899. Result := False;
  1900. Exit;
  1901. end;
  1902. end
  1903. else if (p.oper[0]^.typ = top_reg) then
  1904. begin
  1905. if ([Ch_ROp1, Ch_RWOp1, Ch_MOp1]*Ch<>[]) then
  1906. begin
  1907. Result := False;
  1908. Exit;
  1909. end
  1910. else if ([Ch_WOp1]*Ch<>[]) then
  1911. begin
  1912. if Reg1WriteOverwritesReg2Entirely(p.oper[0]^.reg, reg) then
  1913. Result := True
  1914. else
  1915. begin
  1916. Result := False;
  1917. Exit;
  1918. end;
  1919. end;
  1920. end;
  1921. end;
  1922. if (p.ops > 1) then
  1923. begin
  1924. if RegInOp(reg, p.oper[1]^) then
  1925. begin
  1926. if (p.oper[1]^.typ = top_ref) then
  1927. begin
  1928. if RegInRef(reg, p.oper[1]^.ref^) then
  1929. begin
  1930. Result := False;
  1931. Exit;
  1932. end;
  1933. end
  1934. else if (p.oper[1]^.typ = top_reg) then
  1935. begin
  1936. if ([Ch_ROp2, Ch_RWOp2, Ch_MOp2]*Ch<>[]) then
  1937. begin
  1938. Result := False;
  1939. Exit;
  1940. end
  1941. else if ([Ch_WOp2]*Ch<>[]) then
  1942. begin
  1943. if Reg1WriteOverwritesReg2Entirely(p.oper[1]^.reg, reg) then
  1944. Result := True
  1945. else
  1946. begin
  1947. Result := False;
  1948. Exit;
  1949. end;
  1950. end;
  1951. end;
  1952. end;
  1953. if (p.ops > 2) then
  1954. begin
  1955. if RegInOp(reg, p.oper[2]^) then
  1956. begin
  1957. if (p.oper[2]^.typ = top_ref) then
  1958. begin
  1959. if RegInRef(reg, p.oper[2]^.ref^) then
  1960. begin
  1961. Result := False;
  1962. Exit;
  1963. end;
  1964. end
  1965. else if (p.oper[2]^.typ = top_reg) then
  1966. begin
  1967. if ([Ch_ROp3, Ch_RWOp3, Ch_MOp3]*Ch<>[]) then
  1968. begin
  1969. Result := False;
  1970. Exit;
  1971. end
  1972. else if ([Ch_WOp3]*Ch<>[]) then
  1973. begin
  1974. if Reg1WriteOverwritesReg2Entirely(p.oper[2]^.reg, reg) then
  1975. Result := True
  1976. else
  1977. begin
  1978. Result := False;
  1979. Exit;
  1980. end;
  1981. end;
  1982. end;
  1983. end;
  1984. if (p.ops > 3) and RegInOp(reg, p.oper[3]^) then
  1985. begin
  1986. if (p.oper[3]^.typ = top_ref) then
  1987. begin
  1988. if RegInRef(reg, p.oper[3]^.ref^) then
  1989. begin
  1990. Result := False;
  1991. Exit;
  1992. end;
  1993. end
  1994. else if (p.oper[3]^.typ = top_reg) then
  1995. begin
  1996. if ([Ch_ROp4, Ch_RWOp4, Ch_MOp4]*Ch<>[]) then
  1997. begin
  1998. Result := False;
  1999. Exit;
  2000. end
  2001. else if ([Ch_WOp4]*Ch<>[]) then
  2002. begin
  2003. if Reg1WriteOverwritesReg2Entirely(p.oper[3]^.reg, reg) then
  2004. Result := True
  2005. else
  2006. begin
  2007. Result := False;
  2008. Exit;
  2009. end;
  2010. end;
  2011. end;
  2012. end;
  2013. end;
  2014. end;
  2015. end;
  2016. { Don't do these ones first in case an input operand is equal to an explicit output register }
  2017. case getsupreg(reg) of
  2018. RS_EAX:
  2019. if ([Ch_WEAX{$ifdef x86_64},Ch_WRAX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EAX, reg) then
  2020. begin
  2021. Result := True;
  2022. Exit;
  2023. end;
  2024. RS_ECX:
  2025. if ([Ch_WECX{$ifdef x86_64},Ch_WRCX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_ECX, reg) then
  2026. begin
  2027. Result := True;
  2028. Exit;
  2029. end;
  2030. RS_EDX:
  2031. if ([Ch_REDX{$ifdef x86_64},Ch_WRDX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EDX, reg) then
  2032. begin
  2033. Result := True;
  2034. Exit;
  2035. end;
  2036. RS_EBX:
  2037. if ([Ch_WEBX{$ifdef x86_64},Ch_WRBX{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EBX, reg) then
  2038. begin
  2039. Result := True;
  2040. Exit;
  2041. end;
  2042. RS_ESP:
  2043. if ([Ch_WESP{$ifdef x86_64},Ch_WRSP{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_ESP, reg) then
  2044. begin
  2045. Result := True;
  2046. Exit;
  2047. end;
  2048. RS_EBP:
  2049. if ([Ch_WEBP{$ifdef x86_64},Ch_WRBP{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EBP, reg) then
  2050. begin
  2051. Result := True;
  2052. Exit;
  2053. end;
  2054. RS_ESI:
  2055. if ([Ch_WESI{$ifdef x86_64},Ch_WRSI{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_ESI, reg) then
  2056. begin
  2057. Result := True;
  2058. Exit;
  2059. end;
  2060. RS_EDI:
  2061. if ([Ch_WEDI{$ifdef x86_64},Ch_WRDI{$endif x86_64}]*Ch<>[]) and Reg1WriteOverwritesReg2Entirely(NR_EDI, reg) then
  2062. begin
  2063. Result := True;
  2064. Exit;
  2065. end;
  2066. else
  2067. ;
  2068. end;
  2069. end;
  2070. end;
  2071. end;
  2072. end;
  2073. end;
  2074. class function TX86AsmOptimizer.IsExitCode(p : tai) : boolean;
  2075. var
  2076. hp2,hp3 : tai;
  2077. begin
  2078. { some x86-64 issue a NOP before the real exit code }
  2079. if MatchInstruction(p,A_NOP,[]) then
  2080. GetNextInstruction(p,p);
  2081. result:=assigned(p) and (p.typ=ait_instruction) and
  2082. ((taicpu(p).opcode = A_RET) or
  2083. ((taicpu(p).opcode=A_LEAVE) and
  2084. GetNextInstruction(p,hp2) and
  2085. MatchInstruction(hp2,A_RET,[S_NO])
  2086. ) or
  2087. (((taicpu(p).opcode=A_LEA) and
  2088. MatchOpType(taicpu(p),top_ref,top_reg) and
  2089. (taicpu(p).oper[0]^.ref^.base=NR_STACK_POINTER_REG) and
  2090. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG)
  2091. ) and
  2092. GetNextInstruction(p,hp2) and
  2093. MatchInstruction(hp2,A_RET,[S_NO])
  2094. ) or
  2095. ((((taicpu(p).opcode=A_MOV) and
  2096. MatchOpType(taicpu(p),top_reg,top_reg) and
  2097. (taicpu(p).oper[0]^.reg=current_procinfo.framepointer) and
  2098. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG)) or
  2099. ((taicpu(p).opcode=A_LEA) and
  2100. MatchOpType(taicpu(p),top_ref,top_reg) and
  2101. (taicpu(p).oper[0]^.ref^.base=current_procinfo.framepointer) and
  2102. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG)
  2103. )
  2104. ) and
  2105. GetNextInstruction(p,hp2) and
  2106. MatchInstruction(hp2,A_POP,[reg2opsize(current_procinfo.framepointer)]) and
  2107. MatchOpType(taicpu(hp2),top_reg) and
  2108. (taicpu(hp2).oper[0]^.reg=current_procinfo.framepointer) and
  2109. GetNextInstruction(hp2,hp3) and
  2110. MatchInstruction(hp3,A_RET,[S_NO])
  2111. )
  2112. );
  2113. end;
  2114. class function TX86AsmOptimizer.isFoldableArithOp(hp1: taicpu; reg: tregister): boolean;
  2115. begin
  2116. isFoldableArithOp := False;
  2117. case hp1.opcode of
  2118. A_ADD,A_SUB,A_OR,A_XOR,A_AND,A_SHL,A_SHR,A_SAR:
  2119. isFoldableArithOp :=
  2120. ((taicpu(hp1).oper[0]^.typ = top_const) or
  2121. ((taicpu(hp1).oper[0]^.typ = top_reg) and
  2122. (taicpu(hp1).oper[0]^.reg <> reg))) and
  2123. (taicpu(hp1).oper[1]^.typ = top_reg) and
  2124. (taicpu(hp1).oper[1]^.reg = reg);
  2125. A_INC,A_DEC,A_NEG,A_NOT:
  2126. isFoldableArithOp :=
  2127. (taicpu(hp1).oper[0]^.typ = top_reg) and
  2128. (taicpu(hp1).oper[0]^.reg = reg);
  2129. else
  2130. ;
  2131. end;
  2132. end;
  2133. procedure TX86AsmOptimizer.RemoveLastDeallocForFuncRes(p: tai);
  2134. procedure DoRemoveLastDeallocForFuncRes( supreg: tsuperregister);
  2135. var
  2136. hp2: tai;
  2137. begin
  2138. hp2 := p;
  2139. repeat
  2140. hp2 := tai(hp2.previous);
  2141. if assigned(hp2) and
  2142. (hp2.typ = ait_regalloc) and
  2143. (tai_regalloc(hp2).ratype=ra_dealloc) and
  2144. (getregtype(tai_regalloc(hp2).reg) = R_INTREGISTER) and
  2145. (getsupreg(tai_regalloc(hp2).reg) = supreg) then
  2146. begin
  2147. RemoveInstruction(hp2);
  2148. break;
  2149. end;
  2150. until not(assigned(hp2)) or regInInstruction(newreg(R_INTREGISTER,supreg,R_SUBWHOLE),hp2);
  2151. end;
  2152. begin
  2153. case current_procinfo.procdef.returndef.typ of
  2154. arraydef,recorddef,pointerdef,
  2155. stringdef,enumdef,procdef,objectdef,errordef,
  2156. filedef,setdef,procvardef,
  2157. classrefdef,forwarddef:
  2158. DoRemoveLastDeallocForFuncRes(RS_EAX);
  2159. orddef:
  2160. if current_procinfo.procdef.returndef.size <> 0 then
  2161. begin
  2162. DoRemoveLastDeallocForFuncRes(RS_EAX);
  2163. { for int64/qword }
  2164. if current_procinfo.procdef.returndef.size = 8 then
  2165. DoRemoveLastDeallocForFuncRes(RS_EDX);
  2166. end;
  2167. else
  2168. ;
  2169. end;
  2170. end;
  2171. function TX86AsmOptimizer.OptPass1CMOVcc(var p: tai): Boolean;
  2172. var
  2173. hp1: tai;
  2174. operswap: poper;
  2175. begin
  2176. Result := False;
  2177. { Optimise:
  2178. cmov(c) %reg1,%reg2
  2179. mov %reg2,%reg1
  2180. (%reg2 dealloc.)
  2181. To:
  2182. cmov(~c) %reg2,%reg1
  2183. }
  2184. if (taicpu(p).oper[0]^.typ = top_reg) then
  2185. while GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[1]^.reg) and
  2186. MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  2187. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) and
  2188. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[0]^.reg) do
  2189. begin
  2190. TransferUsedRegs(TmpUsedRegs);
  2191. UpdateUsedRegsBetween(TmpUsedRegs, p, hp1);
  2192. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) then
  2193. begin
  2194. DebugMsg(SPeepholeOptimization + 'CMOV(c) %reg1,%reg2; MOV %reg2,%reg1 -> CMOV(~c) %reg2,%reg1 (CMovMov2CMov)', p);
  2195. { Save time by swapping the pointers (they're both registers, so
  2196. we don't need to worry about reference counts) }
  2197. operswap := taicpu(p).oper[0];
  2198. taicpu(p).oper[0] := taicpu(p).oper[1];
  2199. taicpu(p).oper[1] := operswap;
  2200. taicpu(p).condition := inverse_cond(taicpu(p).condition);
  2201. RemoveInstruction(hp1);
  2202. { It's still a CMOV, so we can look further ahead }
  2203. Include(OptsToCheck, aoc_ForceNewIteration);
  2204. { But first, let's see if this will get optimised again
  2205. (probably won't happen, but best to be sure) }
  2206. Continue;
  2207. end;
  2208. Break;
  2209. end;
  2210. end;
  2211. function TX86AsmOptimizer.OptPass1_V_MOVAP(var p : tai) : boolean;
  2212. var
  2213. hp1,hp2 : tai;
  2214. begin
  2215. result:=false;
  2216. if MatchOpType(taicpu(p),top_reg,top_reg) then
  2217. begin
  2218. { vmova* reg1,reg1
  2219. =>
  2220. <nop> }
  2221. if taicpu(p).oper[0]^.reg = taicpu(p).oper[1]^.reg then
  2222. begin
  2223. RemoveCurrentP(p);
  2224. result:=true;
  2225. exit;
  2226. end;
  2227. if GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg) and
  2228. (hp1.typ = ait_instruction) and
  2229. (
  2230. { Under -O2 and below, the instructions are always adjacent }
  2231. not (cs_opt_level3 in current_settings.optimizerswitches) or
  2232. (taicpu(hp1).ops <= 1) or
  2233. not RegInOp(taicpu(p).oper[0]^.reg, taicpu(hp1).oper[1]^) or
  2234. { If reg1 = reg3, reg1 must not be modified in between }
  2235. not RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp1)
  2236. ) then
  2237. begin
  2238. if MatchInstruction(hp1,[taicpu(p).opcode],[S_NO]) and
  2239. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) then
  2240. begin
  2241. { vmova* reg1,reg2
  2242. ...
  2243. vmova* reg2,reg3
  2244. dealloc reg2
  2245. =>
  2246. vmova* reg1,reg3 }
  2247. TransferUsedRegs(TmpUsedRegs);
  2248. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2249. if MatchOpType(taicpu(hp1),top_reg,top_reg) and
  2250. not RegUsedBetween(taicpu(hp1).oper[1]^.reg, p, hp1) and
  2251. not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  2252. begin
  2253. DebugMsg(SPeepholeOptimization + '(V)MOVA*(V)MOVA*2(V)MOVA* 1',p);
  2254. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  2255. TransferUsedRegs(TmpUsedRegs);
  2256. AllocRegBetween(taicpu(hp1).oper[1]^.reg, p, hp1, TmpUsedRegs);
  2257. RemoveInstruction(hp1);
  2258. result:=true;
  2259. exit;
  2260. end;
  2261. { special case:
  2262. vmova* reg1,<op>
  2263. ...
  2264. vmova* <op>,reg1
  2265. =>
  2266. vmova* reg1,<op> }
  2267. if MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[1]^) and
  2268. ((taicpu(p).oper[0]^.typ<>top_ref) or
  2269. (not(vol_read in taicpu(p).oper[0]^.ref^.volatility))
  2270. ) then
  2271. begin
  2272. DebugMsg(SPeepholeOptimization + '(V)MOVA*(V)MOVA*2(V)MOVA* 2',p);
  2273. RemoveInstruction(hp1);
  2274. result:=true;
  2275. exit;
  2276. end
  2277. end
  2278. else if ((MatchInstruction(p,[A_MOVAPS,A_VMOVAPS],[S_NO]) and
  2279. MatchInstruction(hp1,[A_MOVSS,A_VMOVSS],[S_NO])) or
  2280. ((MatchInstruction(p,[A_MOVAPD,A_VMOVAPD],[S_NO]) and
  2281. MatchInstruction(hp1,[A_MOVSD,A_VMOVSD],[S_NO])))
  2282. ) and
  2283. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) then
  2284. begin
  2285. { vmova* reg1,reg2
  2286. ...
  2287. vmovs* reg2,<op>
  2288. dealloc reg2
  2289. =>
  2290. vmovs* reg1,<op> }
  2291. TransferUsedRegs(TmpUsedRegs);
  2292. UpdateUsedRegsBetween(TmpUsedRegs, p, hp1);
  2293. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  2294. begin
  2295. DebugMsg(SPeepholeOptimization + '(V)MOVA*(V)MOVS*2(V)MOVS* 1',p);
  2296. taicpu(p).opcode:=taicpu(hp1).opcode;
  2297. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  2298. TransferUsedRegs(TmpUsedRegs);
  2299. AllocRegBetween(taicpu(p).oper[0]^.reg, p, hp1, TmpUsedRegs);
  2300. RemoveInstruction(hp1);
  2301. result:=true;
  2302. exit;
  2303. end
  2304. end;
  2305. if MatchInstruction(hp1,[A_VFMADDPD,
  2306. A_VFMADD132PD,
  2307. A_VFMADD132PS,
  2308. A_VFMADD132SD,
  2309. A_VFMADD132SS,
  2310. A_VFMADD213PD,
  2311. A_VFMADD213PS,
  2312. A_VFMADD213SD,
  2313. A_VFMADD213SS,
  2314. A_VFMADD231PD,
  2315. A_VFMADD231PS,
  2316. A_VFMADD231SD,
  2317. A_VFMADD231SS,
  2318. A_VFMADDSUB132PD,
  2319. A_VFMADDSUB132PS,
  2320. A_VFMADDSUB213PD,
  2321. A_VFMADDSUB213PS,
  2322. A_VFMADDSUB231PD,
  2323. A_VFMADDSUB231PS,
  2324. A_VFMSUB132PD,
  2325. A_VFMSUB132PS,
  2326. A_VFMSUB132SD,
  2327. A_VFMSUB132SS,
  2328. A_VFMSUB213PD,
  2329. A_VFMSUB213PS,
  2330. A_VFMSUB213SD,
  2331. A_VFMSUB213SS,
  2332. A_VFMSUB231PD,
  2333. A_VFMSUB231PS,
  2334. A_VFMSUB231SD,
  2335. A_VFMSUB231SS,
  2336. A_VFMSUBADD132PD,
  2337. A_VFMSUBADD132PS,
  2338. A_VFMSUBADD213PD,
  2339. A_VFMSUBADD213PS,
  2340. A_VFMSUBADD231PD,
  2341. A_VFMSUBADD231PS,
  2342. A_VFNMADD132PD,
  2343. A_VFNMADD132PS,
  2344. A_VFNMADD132SD,
  2345. A_VFNMADD132SS,
  2346. A_VFNMADD213PD,
  2347. A_VFNMADD213PS,
  2348. A_VFNMADD213SD,
  2349. A_VFNMADD213SS,
  2350. A_VFNMADD231PD,
  2351. A_VFNMADD231PS,
  2352. A_VFNMADD231SD,
  2353. A_VFNMADD231SS,
  2354. A_VFNMSUB132PD,
  2355. A_VFNMSUB132PS,
  2356. A_VFNMSUB132SD,
  2357. A_VFNMSUB132SS,
  2358. A_VFNMSUB213PD,
  2359. A_VFNMSUB213PS,
  2360. A_VFNMSUB213SD,
  2361. A_VFNMSUB213SS,
  2362. A_VFNMSUB231PD,
  2363. A_VFNMSUB231PS,
  2364. A_VFNMSUB231SD,
  2365. A_VFNMSUB231SS],[S_NO]) and
  2366. { we mix single and double opperations here because we assume that the compiler
  2367. generates vmovapd only after double operations and vmovaps only after single operations }
  2368. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[2]^.reg) and
  2369. GetNextInstructionUsingReg(hp1, hp2, taicpu(hp1).oper[2]^.reg) and
  2370. MatchInstruction(hp2,[A_VMOVAPD,A_VMOVAPS,A_MOVAPD,A_MOVAPS],[S_NO]) and
  2371. MatchOperand(taicpu(p).oper[0]^,taicpu(hp2).oper[1]^) then
  2372. begin
  2373. TransferUsedRegs(TmpUsedRegs);
  2374. UpdateUsedRegsBetween(TmpUsedRegs, p, hp2);
  2375. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  2376. begin
  2377. taicpu(hp1).loadoper(2,taicpu(p).oper[0]^);
  2378. if (cs_opt_level3 in current_settings.optimizerswitches) then
  2379. RemoveCurrentP(p)
  2380. else
  2381. RemoveCurrentP(p, hp1); // hp1 is guaranteed to be the immediate next instruction in this case.
  2382. RemoveInstruction(hp2);
  2383. end;
  2384. end
  2385. else if (hp1.typ = ait_instruction) and
  2386. (((taicpu(p).opcode=A_MOVAPS) and
  2387. ((taicpu(hp1).opcode=A_ADDSS) or (taicpu(hp1).opcode=A_SUBSS) or
  2388. (taicpu(hp1).opcode=A_MULSS) or (taicpu(hp1).opcode=A_DIVSS))) or
  2389. ((taicpu(p).opcode=A_MOVAPD) and
  2390. ((taicpu(hp1).opcode=A_ADDSD) or (taicpu(hp1).opcode=A_SUBSD) or
  2391. (taicpu(hp1).opcode=A_MULSD) or (taicpu(hp1).opcode=A_DIVSD)))
  2392. ) and
  2393. GetNextInstructionUsingReg(hp1, hp2, taicpu(hp1).oper[1]^.reg) and
  2394. MatchInstruction(hp2,taicpu(p).opcode,[]) and
  2395. OpsEqual(taicpu(hp2).oper[1]^, taicpu(p).oper[0]^) and
  2396. MatchOpType(taicpu(hp2),top_reg,top_reg) and
  2397. MatchOperand(taicpu(hp2).oper[0]^,taicpu(p).oper[1]^) then
  2398. { change
  2399. movapX reg,reg2
  2400. addsX/subsX/... reg3, reg2
  2401. movapX reg2,reg
  2402. to
  2403. addsX/subsX/... reg3,reg
  2404. }
  2405. begin
  2406. TransferUsedRegs(TmpUsedRegs);
  2407. UpdateUsedRegsBetween(TmpUsedRegs, p, hp2);
  2408. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  2409. begin
  2410. DebugMsg(SPeepholeOptimization + 'MovapXOpMovapX2Op ('+
  2411. debug_op2str(taicpu(p).opcode)+' '+
  2412. debug_op2str(taicpu(hp1).opcode)+' '+
  2413. debug_op2str(taicpu(hp2).opcode)+') done',p);
  2414. { we cannot eliminate the first move if
  2415. the operations uses the same register for source and dest }
  2416. if not(OpsEqual(taicpu(hp1).oper[1]^,taicpu(hp1).oper[0]^)) then
  2417. { Remember that hp1 is not necessarily the immediate
  2418. next instruction }
  2419. RemoveCurrentP(p);
  2420. taicpu(hp1).loadoper(1, taicpu(hp2).oper[1]^);
  2421. RemoveInstruction(hp2);
  2422. result:=true;
  2423. end;
  2424. end
  2425. else if (hp1.typ = ait_instruction) and
  2426. (((taicpu(p).opcode=A_VMOVAPD) and
  2427. (taicpu(hp1).opcode=A_VCOMISD)) or
  2428. ((taicpu(p).opcode=A_VMOVAPS) and
  2429. ((taicpu(hp1).opcode=A_VCOMISS))
  2430. )
  2431. ) and not(OpsEqual(taicpu(hp1).oper[1]^,taicpu(hp1).oper[0]^)) then
  2432. { change
  2433. movapX reg,reg1
  2434. vcomisX reg1,reg1
  2435. to
  2436. vcomisX reg,reg
  2437. }
  2438. begin
  2439. TransferUsedRegs(TmpUsedRegs);
  2440. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2441. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  2442. begin
  2443. DebugMsg(SPeepholeOptimization + 'MovapXComisX2ComisX2 ('+
  2444. debug_op2str(taicpu(p).opcode)+' '+
  2445. debug_op2str(taicpu(hp1).opcode)+') done',p);
  2446. if OpsEqual(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) then
  2447. taicpu(hp1).loadoper(0, taicpu(p).oper[0]^);
  2448. if OpsEqual(taicpu(p).oper[1]^,taicpu(hp1).oper[1]^) then
  2449. taicpu(hp1).loadoper(1, taicpu(p).oper[0]^);
  2450. RemoveCurrentP(p);
  2451. result:=true;
  2452. exit;
  2453. end;
  2454. end
  2455. end;
  2456. end;
  2457. end;
  2458. function TX86AsmOptimizer.OptPass1VOP(var p : tai) : boolean;
  2459. var
  2460. hp1 : tai;
  2461. begin
  2462. result:=false;
  2463. { replace
  2464. V<Op>X %mreg1,%mreg2,%mreg3
  2465. VMovX %mreg3,%mreg4
  2466. dealloc %mreg3
  2467. by
  2468. V<Op>X %mreg1,%mreg2,%mreg4
  2469. ?
  2470. }
  2471. if GetNextInstruction(p,hp1) and
  2472. { we mix single and double operations here because we assume that the compiler
  2473. generates vmovapd only after double operations and vmovaps only after single operations }
  2474. MatchInstruction(hp1,A_VMOVAPD,A_VMOVAPS,[S_NO]) and
  2475. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  2476. (taicpu(hp1).oper[1]^.typ=top_reg) then
  2477. begin
  2478. TransferUsedRegs(TmpUsedRegs);
  2479. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  2480. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  2481. begin
  2482. taicpu(p).loadoper(2,taicpu(hp1).oper[1]^);
  2483. DebugMsg(SPeepholeOptimization + 'VOpVmov2VOp done',p);
  2484. RemoveInstruction(hp1);
  2485. result:=true;
  2486. end;
  2487. end;
  2488. end;
  2489. { Replaces all references to AOldReg in a memory reference to ANewReg }
  2490. class function TX86AsmOptimizer.ReplaceRegisterInRef(var ref: TReference; const AOldReg, ANewReg: TRegister): Boolean;
  2491. begin
  2492. Result := False;
  2493. { For safety reasons, only check for exact register matches }
  2494. { Check base register }
  2495. if (ref.base = AOldReg) then
  2496. begin
  2497. ref.base := ANewReg;
  2498. Result := True;
  2499. end;
  2500. { Check index register }
  2501. if (ref.index = AOldReg) and (getsupreg(ANewReg)<>RS_ESP) then
  2502. begin
  2503. ref.index := ANewReg;
  2504. Result := True;
  2505. end;
  2506. end;
  2507. { Replaces all references to AOldReg in an operand to ANewReg }
  2508. class function TX86AsmOptimizer.ReplaceRegisterInOper(const p: taicpu; const OperIdx: Integer; const AOldReg, ANewReg: TRegister): Boolean;
  2509. var
  2510. OldSupReg, NewSupReg: TSuperRegister;
  2511. OldSubReg, NewSubReg: TSubRegister;
  2512. OldRegType: TRegisterType;
  2513. ThisOper: POper;
  2514. begin
  2515. ThisOper := p.oper[OperIdx]; { Faster to access overall }
  2516. Result := False;
  2517. if (AOldReg = NR_NO) or (ANewReg = NR_NO) then
  2518. InternalError(2020011801);
  2519. OldSupReg := getsupreg(AOldReg);
  2520. OldSubReg := getsubreg(AOldReg);
  2521. OldRegType := getregtype(AOldReg);
  2522. NewSupReg := getsupreg(ANewReg);
  2523. NewSubReg := getsubreg(ANewReg);
  2524. if OldRegType <> getregtype(ANewReg) then
  2525. InternalError(2020011802);
  2526. if OldSubReg <> NewSubReg then
  2527. InternalError(2020011803);
  2528. case ThisOper^.typ of
  2529. top_reg:
  2530. if (
  2531. (ThisOper^.reg = AOldReg) or
  2532. (
  2533. (OldRegType = R_INTREGISTER) and
  2534. (getsupreg(ThisOper^.reg) = OldSupReg) and
  2535. (getregtype(ThisOper^.reg) = R_INTREGISTER) and
  2536. (
  2537. (getsubreg(ThisOper^.reg) <= OldSubReg)
  2538. {$ifndef x86_64}
  2539. and (
  2540. { Under i386 and i8086, ESI, EDI, EBP and ESP
  2541. don't have an 8-bit representation }
  2542. (getsubreg(ThisOper^.reg) >= R_SUBW) or
  2543. not (NewSupReg in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  2544. )
  2545. {$endif x86_64}
  2546. )
  2547. )
  2548. ) then
  2549. begin
  2550. ThisOper^.reg := newreg(getregtype(ANewReg), NewSupReg, getsubreg(p.oper[OperIdx]^.reg));
  2551. Result := True;
  2552. end;
  2553. top_ref:
  2554. if ReplaceRegisterInRef(ThisOper^.ref^, AOldReg, ANewReg) then
  2555. Result := True;
  2556. else
  2557. ;
  2558. end;
  2559. end;
  2560. { Replaces all references to AOldReg in an instruction to ANewReg }
  2561. class function TX86AsmOptimizer.ReplaceRegisterInInstruction(const p: taicpu; const AOldReg, ANewReg: TRegister): Boolean;
  2562. const
  2563. ReadFlag: array[0..3] of TInsChange = (Ch_Rop1, Ch_Rop2, Ch_Rop3, Ch_Rop4);
  2564. var
  2565. OperIdx: Integer;
  2566. begin
  2567. Result := False;
  2568. for OperIdx := 0 to p.ops - 1 do
  2569. if (ReadFlag[OperIdx] in InsProp[p.Opcode].Ch) then
  2570. begin
  2571. { The shift and rotate instructions can only use CL }
  2572. if not (
  2573. (OperIdx = 0) and
  2574. { This second condition just helps to avoid unnecessarily
  2575. calling MatchInstruction for 10 different opcodes }
  2576. (p.oper[0]^.reg = NR_CL) and
  2577. MatchInstruction(p, [A_RCL, A_RCR, A_ROL, A_ROR, A_SAL, A_SAR, A_SHL, A_SHLD, A_SHR, A_SHRD], [])
  2578. ) then
  2579. Result := ReplaceRegisterInOper(p, OperIdx, AOldReg, ANewReg) or Result;
  2580. end
  2581. else if p.oper[OperIdx]^.typ = top_ref then
  2582. { It's okay to replace registers in references that get written to }
  2583. Result := ReplaceRegisterInOper(p, OperIdx, AOldReg, ANewReg) or Result;
  2584. end;
  2585. class function TX86AsmOptimizer.IsRefSafe(const ref: PReference): Boolean;
  2586. begin
  2587. Result :=
  2588. (ref^.index = NR_NO) and
  2589. (
  2590. {$ifdef x86_64}
  2591. (
  2592. (ref^.base = NR_RIP) and
  2593. (ref^.refaddr in [addr_pic, addr_pic_no_got])
  2594. ) or
  2595. {$endif x86_64}
  2596. (ref^.refaddr = addr_full) or
  2597. (ref^.base = NR_STACK_POINTER_REG) or
  2598. (ref^.base = current_procinfo.framepointer)
  2599. );
  2600. end;
  2601. function TX86AsmOptimizer.ConvertLEA(const p: taicpu): Boolean;
  2602. var
  2603. l: asizeint;
  2604. begin
  2605. Result := False;
  2606. { Should have been checked previously }
  2607. if p.opcode <> A_LEA then
  2608. InternalError(2020072501);
  2609. { do not mess with the stack point as adjusting it by lea is recommend, except if we optimize for size }
  2610. if (p.oper[1]^.reg=NR_STACK_POINTER_REG) and
  2611. not(cs_opt_size in current_settings.optimizerswitches) then
  2612. exit;
  2613. with p.oper[0]^.ref^ do
  2614. begin
  2615. if (base <> p.oper[1]^.reg) or
  2616. (index <> NR_NO) or
  2617. assigned(symbol) then
  2618. exit;
  2619. l:=offset;
  2620. if (l=1) and UseIncDec then
  2621. begin
  2622. p.opcode:=A_INC;
  2623. p.loadreg(0,p.oper[1]^.reg);
  2624. p.ops:=1;
  2625. DebugMsg(SPeepholeOptimization + 'Lea2Inc done',p);
  2626. end
  2627. else if (l=-1) and UseIncDec then
  2628. begin
  2629. p.opcode:=A_DEC;
  2630. p.loadreg(0,p.oper[1]^.reg);
  2631. p.ops:=1;
  2632. DebugMsg(SPeepholeOptimization + 'Lea2Dec done',p);
  2633. end
  2634. else
  2635. begin
  2636. if (l<0) and (l<>-2147483648) then
  2637. begin
  2638. p.opcode:=A_SUB;
  2639. p.loadConst(0,-l);
  2640. DebugMsg(SPeepholeOptimization + 'Lea2Sub done',p);
  2641. end
  2642. else
  2643. begin
  2644. p.opcode:=A_ADD;
  2645. p.loadConst(0,l);
  2646. DebugMsg(SPeepholeOptimization + 'Lea2Add done',p);
  2647. end;
  2648. end;
  2649. end;
  2650. Result := True;
  2651. end;
  2652. function TX86AsmOptimizer.DeepMOVOpt(const p_mov: taicpu; const hp: taicpu): Boolean;
  2653. var
  2654. CurrentReg, ReplaceReg: TRegister;
  2655. begin
  2656. Result := False;
  2657. ReplaceReg := taicpu(p_mov).oper[0]^.reg;
  2658. CurrentReg := taicpu(p_mov).oper[1]^.reg;
  2659. case hp.opcode of
  2660. A_FSTSW, A_FNSTSW,
  2661. A_IN, A_INS, A_OUT, A_OUTS,
  2662. A_CMPS, A_LODS, A_MOVS, A_SCAS, A_STOS:
  2663. { These routines have explicit operands, but they are restricted in
  2664. what they can be (e.g. IN and OUT can only read from AL, AX or
  2665. EAX. }
  2666. Exit;
  2667. A_IMUL:
  2668. begin
  2669. { The 1-operand version writes to implicit registers
  2670. The 2-operand version reads from the first operator, and reads
  2671. from and writes to the second (equivalent to Ch_ROp1, ChRWOp2).
  2672. the 3-operand version reads from a register that it doesn't write to
  2673. }
  2674. case hp.ops of
  2675. 1:
  2676. if (
  2677. (
  2678. (hp.opsize = S_B) and (getsupreg(CurrentReg) <> RS_EAX)
  2679. ) or
  2680. not (getsupreg(CurrentReg) in [RS_EAX, RS_EDX])
  2681. ) and ReplaceRegisterInOper(hp, 0, CurrentReg, ReplaceReg) then
  2682. begin
  2683. Result := True;
  2684. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovIMul2MovIMul 1)', hp);
  2685. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2686. end;
  2687. 2:
  2688. { Only modify the first parameter }
  2689. if ReplaceRegisterInOper(hp, 0, CurrentReg, ReplaceReg) then
  2690. begin
  2691. Result := True;
  2692. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovIMul2MovIMul 2)', hp);
  2693. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2694. end;
  2695. 3:
  2696. { Only modify the second parameter }
  2697. if ReplaceRegisterInOper(hp, 1, CurrentReg, ReplaceReg) then
  2698. begin
  2699. Result := True;
  2700. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovIMul2MovIMul 3)', hp);
  2701. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2702. end;
  2703. else
  2704. InternalError(2020012901);
  2705. end;
  2706. end;
  2707. else
  2708. if (hp.ops > 0) and
  2709. ReplaceRegisterInInstruction(hp, CurrentReg, ReplaceReg) then
  2710. begin
  2711. Result := True;
  2712. DebugMsg(SPeepholeOptimization + debug_regname(CurrentReg) + ' = ' + debug_regname(ReplaceReg) + '; changed to minimise pipeline stall (MovXXX2MovXXX)', hp);
  2713. AllocRegBetween(ReplaceReg, p_mov, hp, UsedRegs);
  2714. end;
  2715. end;
  2716. end;
  2717. function TX86AsmOptimizer.FuncMov2Func(var p: tai; const hp1: tai): Boolean;
  2718. var
  2719. hp2, hp_regalloc: tai;
  2720. p_SourceReg, p_TargetReg: TRegister;
  2721. begin
  2722. Result := False;
  2723. { Backward optimisation. If we have:
  2724. func. %reg1,%reg2
  2725. mov %reg2,%reg3
  2726. (dealloc %reg2)
  2727. Change to:
  2728. func. %reg1,%reg3 (see comment below for what a valid func. is)
  2729. Perform similar optimisations with 1, 3 and 4-operand instructions
  2730. that only have one output.
  2731. }
  2732. if MatchOpType(taicpu(p), top_reg, top_reg) then
  2733. begin
  2734. p_SourceReg := taicpu(p).oper[0]^.reg;
  2735. p_TargetReg := taicpu(p).oper[1]^.reg;
  2736. TransferUsedRegs(TmpUsedRegs);
  2737. if not RegUsedAfterInstruction(p_SourceReg, p, TmpUsedRegs) and
  2738. GetLastInstruction(p, hp2) and
  2739. (hp2.typ = ait_instruction) and
  2740. { Have to make sure it's an instruction that only reads from
  2741. the first operands and only writes (not reads or modifies) to
  2742. the last one; in essence, a pure function such as BSR, POPCNT
  2743. or ANDN }
  2744. (
  2745. (
  2746. (taicpu(hp2).ops = 1) and
  2747. (insprop[taicpu(hp2).opcode].Ch * [Ch_Wop1] = [Ch_Wop1])
  2748. ) or
  2749. (
  2750. (taicpu(hp2).ops = 2) and
  2751. (insprop[taicpu(hp2).opcode].Ch * [Ch_Rop1, Ch_Wop2] = [Ch_Rop1, Ch_Wop2])
  2752. ) or
  2753. (
  2754. (taicpu(hp2).ops = 3) and
  2755. (insprop[taicpu(hp2).opcode].Ch * [Ch_Rop1, Ch_Rop2, Ch_Wop3] = [Ch_Rop1, Ch_Rop2, Ch_Wop3])
  2756. ) or
  2757. (
  2758. (taicpu(hp2).ops = 4) and
  2759. (insprop[taicpu(hp2).opcode].Ch * [Ch_Rop1, Ch_Rop2, Ch_Rop3, Ch_Wop4] = [Ch_Rop1, Ch_Rop2, Ch_Rop3, Ch_Wop4])
  2760. )
  2761. ) and
  2762. (taicpu(hp2).oper[taicpu(hp2).ops-1]^.typ = top_reg) and
  2763. (taicpu(hp2).oper[taicpu(hp2).ops-1]^.reg = p_SourceReg) then
  2764. begin
  2765. case taicpu(hp2).opcode of
  2766. A_FSTSW, A_FNSTSW,
  2767. A_IN, A_INS, A_OUT, A_OUTS,
  2768. A_CMPS, A_LODS, A_MOVS, A_SCAS, A_STOS:
  2769. { These routines have explicit operands, but they are restricted in
  2770. what they can be (e.g. IN and OUT can only read from AL, AX or
  2771. EAX. }
  2772. ;
  2773. else
  2774. begin
  2775. DebugMsg(SPeepholeOptimization + 'Removed MOV and changed destination on previous instruction to optimise register usage (FuncMov2Func)', p);
  2776. { if %reg2 (p_SourceReg) is allocated before func., remove it completely }
  2777. hp_regalloc := FindRegAllocBackward(p_SourceReg, hp2);
  2778. if Assigned(hp_regalloc) then
  2779. begin
  2780. Asml.Remove(hp_regalloc);
  2781. if Assigned(FindRegDealloc(p_SourceReg, p)) then
  2782. begin
  2783. ExcludeRegFromUsedRegs(p_SourceReg, UsedRegs);
  2784. hp_regalloc.Free;
  2785. end
  2786. else
  2787. { If the register is not explicitly deallocated, it's
  2788. being reused, so move the allocation to after func. }
  2789. AsmL.InsertAfter(hp_regalloc, hp2);
  2790. end;
  2791. if not RegInInstruction(p_TargetReg, hp2) then
  2792. begin
  2793. TransferUsedRegs(TmpUsedRegs);
  2794. AllocRegBetween(p_TargetReg, hp2, p, TmpUsedRegs);
  2795. end;
  2796. { Actually make the changes }
  2797. taicpu(hp2).oper[taicpu(hp2).ops-1]^.reg := p_TargetReg;
  2798. RemoveCurrentp(p, hp1);
  2799. { If the Func was another MOV instruction, we might get
  2800. "mov %reg,%reg" that doesn't get removed in Pass 2
  2801. otherwise, so deal with it here (also do something
  2802. similar with lea (%reg),%reg}
  2803. if (taicpu(hp2).opcode = A_MOV) and MatchOperand(taicpu(hp2).oper[0]^, taicpu(hp2).oper[1]^.reg) then
  2804. begin
  2805. DebugMsg(SPeepholeOptimization + 'Mov2Nop 1a done', hp2);
  2806. if p = hp2 then
  2807. RemoveCurrentp(p)
  2808. else
  2809. RemoveInstruction(hp2);
  2810. end;
  2811. Result := True;
  2812. Exit;
  2813. end;
  2814. end;
  2815. end;
  2816. end;
  2817. end;
  2818. function TX86AsmOptimizer.CheckMovMov2MovMov2(const p, hp1: tai) : boolean;
  2819. begin
  2820. Result := False;
  2821. if MatchOpType(taicpu(p),top_ref,top_reg) and
  2822. MatchOpType(taicpu(hp1),top_ref,top_reg) and
  2823. (taicpu(p).opsize = taicpu(hp1).opsize) and
  2824. RefsEqual(taicpu(p).oper[0]^.ref^,taicpu(hp1).oper[0]^.ref^) and
  2825. (taicpu(p).oper[0]^.ref^.volatility=[]) and
  2826. (taicpu(hp1).oper[0]^.ref^.volatility=[]) and
  2827. not(SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.ref^.base)) and
  2828. not(SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.ref^.index)) then
  2829. begin
  2830. DebugMsg(SPeepholeOptimization + 'MovMov2MovMov 2',p);
  2831. taicpu(hp1).loadReg(0,taicpu(p).oper[1]^.reg);
  2832. Result := True;
  2833. Include(OptsToCheck, aoc_ForceNewIteration);
  2834. end;
  2835. end;
  2836. function TX86AsmOptimizer.OptPass1MOV(var p : tai) : boolean;
  2837. var
  2838. hp1, hp2, hp3, hp4, last_hp1: tai;
  2839. GetNextInstruction_p, DoOptimisation, TempBool: Boolean;
  2840. p_SourceReg, p_TargetReg, NewMMReg: TRegister;
  2841. {$ifdef x86_64}
  2842. NewConst: TCGInt;
  2843. {$endif x86_64}
  2844. procedure convert_mov_value(signed_movop: tasmop; max_value: tcgint); inline;
  2845. begin
  2846. if taicpu(hp1).opcode = signed_movop then
  2847. begin
  2848. if taicpu(p).oper[0]^.val > max_value shr 1 then
  2849. taicpu(p).oper[0]^.val:=taicpu(p).oper[0]^.val - max_value - 1 { Convert to signed }
  2850. end
  2851. else
  2852. taicpu(p).oper[0]^.val:=taicpu(p).oper[0]^.val and max_value; { Trim to unsigned }
  2853. end;
  2854. function GetNextHp1(const in_p: tai): Boolean;
  2855. begin
  2856. if NotFirstIteration and (cs_opt_level3 in current_settings.optimizerswitches) then
  2857. GetNextInstruction_p := GetNextInstructionUsingReg(in_p, hp1, p_TargetReg)
  2858. else
  2859. GetNextInstruction_p := GetNextInstruction(in_p, hp1);
  2860. Result := GetNextInstruction_p and (hp1.typ = ait_instruction);
  2861. end;
  2862. function TryConstMerge(var p1, p2: tai): Boolean;
  2863. var
  2864. ThisRef: TReference;
  2865. begin
  2866. Result := False;
  2867. ThisRef := taicpu(p2).oper[1]^.ref^;
  2868. { Only permit writes to the stack, since we can guarantee alignment with that }
  2869. if (ThisRef.index = NR_NO) and
  2870. (
  2871. (ThisRef.base = NR_STACK_POINTER_REG) or
  2872. (ThisRef.base = current_procinfo.framepointer)
  2873. ) then
  2874. begin
  2875. case taicpu(p).opsize of
  2876. S_B:
  2877. begin
  2878. { Word writes must be on a 2-byte boundary }
  2879. if (taicpu(p1).oper[1]^.ref^.offset mod 2) = 0 then
  2880. begin
  2881. { Reduce offset of second reference to see if it is sequential with the first }
  2882. Dec(ThisRef.offset, 1);
  2883. if RefsEqual(taicpu(p1).oper[1]^.ref^, ThisRef) then
  2884. begin
  2885. { Make sure the constants aren't represented as a
  2886. negative number, as these won't merge properly }
  2887. taicpu(p1).opsize := S_W;
  2888. taicpu(p1).oper[0]^.val := (taicpu(p1).oper[0]^.val and $FF) or ((taicpu(p2).oper[0]^.val and $FF) shl 8);
  2889. DebugMsg(SPeepholeOptimization + 'Merged two byte-sized constant writes to stack (MovMov2Mov 2a)', p1);
  2890. RemoveInstruction(p2);
  2891. Result := True;
  2892. end;
  2893. end;
  2894. end;
  2895. S_W:
  2896. begin
  2897. { Longword writes must be on a 4-byte boundary }
  2898. if (taicpu(p1).oper[1]^.ref^.offset mod 4) = 0 then
  2899. begin
  2900. { Reduce offset of second reference to see if it is sequential with the first }
  2901. Dec(ThisRef.offset, 2);
  2902. if RefsEqual(taicpu(p1).oper[1]^.ref^, ThisRef) then
  2903. begin
  2904. { Make sure the constants aren't represented as a
  2905. negative number, as these won't merge properly }
  2906. taicpu(p1).opsize := S_L;
  2907. taicpu(p1).oper[0]^.val := (taicpu(p1).oper[0]^.val and $FFFF) or ((taicpu(p2).oper[0]^.val and $FFFF) shl 16);
  2908. DebugMsg(SPeepholeOptimization + 'Merged two word-sized constant writes to stack (MovMov2Mov 2b)', p1);
  2909. RemoveInstruction(p2);
  2910. Result := True;
  2911. end;
  2912. end;
  2913. end;
  2914. {$ifdef x86_64}
  2915. S_L:
  2916. begin
  2917. { Only sign-extended 32-bit constants can be written to 64-bit memory directly, so check to
  2918. see if the constants can be encoded this way. }
  2919. NewConst := (taicpu(p1).oper[0]^.val and $FFFFFFFF) or (taicpu(p2).oper[0]^.val shl 32);
  2920. if (NewConst >= -2147483648) and (NewConst <= 2147483647) and
  2921. { Quadword writes must be on an 8-byte boundary }
  2922. ((taicpu(p1).oper[1]^.ref^.offset mod 8) = 0) then
  2923. begin
  2924. { Reduce offset of second reference to see if it is sequential with the first }
  2925. Dec(ThisRef.offset, 4);
  2926. if RefsEqual(taicpu(p1).oper[1]^.ref^, ThisRef) then
  2927. begin
  2928. { Make sure the constants aren't represented as a
  2929. negative number, as these won't merge properly }
  2930. taicpu(p1).opsize := S_Q;
  2931. { Force a typecast into a 32-bit signed integer (that will then be sign-extended to 64-bit) }
  2932. taicpu(p1).oper[0]^.val := NewConst;
  2933. DebugMsg(SPeepholeOptimization + 'Merged two longword-sized constant writes to stack (MovMov2Mov 2c)', p1);
  2934. RemoveInstruction(p2);
  2935. Result := True;
  2936. end;
  2937. end;
  2938. end;
  2939. {$endif x86_64}
  2940. else
  2941. ;
  2942. end;
  2943. end;
  2944. end;
  2945. var
  2946. TempRegUsed, CrossJump: Boolean;
  2947. PreMessage, RegName1, RegName2, InputVal, MaskNum: string;
  2948. NewSize: topsize; NewOffset: asizeint;
  2949. SourceRef, TargetRef: TReference;
  2950. MovAligned, MovUnaligned: TAsmOp;
  2951. JumpTracking: TLinkedList;
  2952. begin
  2953. Result:=false;
  2954. { remove mov reg1,reg1? }
  2955. if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^)
  2956. then
  2957. begin
  2958. DebugMsg(SPeepholeOptimization + 'Mov2Nop 1 done',p);
  2959. { take care of the register (de)allocs following p }
  2960. RemoveCurrentP(p);
  2961. Result := True;
  2962. exit;
  2963. end;
  2964. { Prevent compiler warnings }
  2965. p_SourceReg := NR_NO;
  2966. p_TargetReg := NR_NO;
  2967. hp1 := nil;
  2968. if taicpu(p).oper[1]^.typ = top_reg then
  2969. begin
  2970. { Saves on a large number of dereferences }
  2971. p_TargetReg := taicpu(p).oper[1]^.reg;
  2972. TransferUsedRegs(TmpUsedRegs);
  2973. last_hp1 := p;
  2974. if GetNextHp1(p) then
  2975. while True do
  2976. begin
  2977. if (taicpu(hp1).opcode = A_AND) and
  2978. (taicpu(hp1).oper[1]^.typ = top_reg) and
  2979. SuperRegistersEqual(p_TargetReg, taicpu(hp1).oper[1]^.reg) then
  2980. begin
  2981. UpdateUsedRegsBetween(TmpUsedRegs, last_hp1, hp1);
  2982. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  2983. (taicpu(hp1).oper[0]^.typ = top_const) and
  2984. (taicpu(p).opsize = taicpu(hp1).opsize) then
  2985. begin
  2986. case taicpu(p).opsize of
  2987. S_L:
  2988. if (taicpu(hp1).oper[0]^.val = $ffffffff) then
  2989. begin
  2990. { Optimize out:
  2991. mov x, %reg
  2992. and ffffffffh, %reg
  2993. }
  2994. DebugMsg(SPeepholeOptimization + 'MovAnd2Mov 1 done',p);
  2995. hp2 := tai(hp1.Previous);
  2996. RemoveInstruction(hp1);
  2997. //Include(OptsToCheck, aoc_ForceNewIteration);
  2998. if GetNextHp1(hp2) then
  2999. Continue
  3000. else
  3001. Exit;
  3002. end;
  3003. S_Q: { TODO: Confirm if this is even possible }
  3004. if (taicpu(hp1).oper[0]^.val = $ffffffffffffffff) then
  3005. begin
  3006. { Optimize out:
  3007. mov x, %reg
  3008. and ffffffffffffffffh, %reg
  3009. }
  3010. DebugMsg(SPeepholeOptimization + 'MovAnd2Mov 2 done',p);
  3011. hp2 := tai(hp1.Previous);
  3012. RemoveInstruction(hp1);
  3013. //Include(OptsToCheck, aoc_ForceNewIteration);
  3014. if GetNextHp1(hp2) then
  3015. Continue
  3016. else
  3017. Exit;
  3018. end;
  3019. else
  3020. ;
  3021. end;
  3022. if (
  3023. { Make sure that if a reference is used, its registers
  3024. are not modified in between }
  3025. (
  3026. (taicpu(p).oper[0]^.typ = top_reg) and
  3027. not RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp1)
  3028. ) or
  3029. (
  3030. (taicpu(p).oper[0]^.typ = top_ref) and
  3031. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) and
  3032. not RefModifiedBetween(taicpu(p).oper[0]^.ref^, topsize2memsize[taicpu(p).opsize] shr 3, p, hp1)
  3033. )
  3034. ) and
  3035. GetNextInstruction(hp1,hp2) and
  3036. MatchInstruction(hp2,A_TEST,[]) and
  3037. (
  3038. MatchOperand(taicpu(hp1).oper[1]^,taicpu(hp2).oper[1]^) or
  3039. (
  3040. { If the register being tested is smaller than the one
  3041. that received a bitwise AND, permit it if the constant
  3042. fits into the smaller size }
  3043. (taicpu(hp1).oper[1]^.typ = top_reg) and (taicpu(hp2).oper[1]^.typ = top_reg) and
  3044. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg,taicpu(hp2).oper[1]^.reg) and
  3045. (taicpu(hp1).oper[0]^.typ = top_const) and (taicpu(hp1).oper[0]^.val >= 0) and
  3046. (GetSubReg(taicpu(hp2).oper[1]^.reg) < GetSubReg(taicpu(hp1).oper[1]^.reg)) and
  3047. (
  3048. (
  3049. (GetSubReg(taicpu(hp2).oper[1]^.reg) = R_SUBL) and
  3050. (taicpu(hp1).oper[0]^.val <= $FF)
  3051. ) or
  3052. (
  3053. (GetSubReg(taicpu(hp2).oper[1]^.reg) = R_SUBW) and
  3054. (taicpu(hp1).oper[0]^.val <= $FFFF)
  3055. {$ifdef x86_64}
  3056. ) or
  3057. (
  3058. (GetSubReg(taicpu(hp2).oper[1]^.reg) = R_SUBD) and
  3059. (taicpu(hp1).oper[0]^.val <= $FFFFFFFF)
  3060. {$endif x86_64}
  3061. )
  3062. )
  3063. )
  3064. ) and
  3065. (
  3066. MatchOperand(taicpu(hp2).oper[0]^,taicpu(hp2).oper[1]^) or
  3067. MatchOperand(taicpu(hp2).oper[0]^,-1)
  3068. ) and
  3069. GetNextInstruction(hp2,hp3) and
  3070. MatchInstruction(hp3,A_Jcc,A_Setcc,[]) and
  3071. (taicpu(hp3).condition in [C_E,C_NE]) then
  3072. begin
  3073. TransferUsedRegs(TmpUsedRegs);
  3074. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  3075. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  3076. if not(RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp2, TmpUsedRegs)) then
  3077. begin
  3078. DebugMsg(SPeepholeOptimization + 'MovAndTest2Test done',p);
  3079. taicpu(hp1).loadoper(1,taicpu(p).oper[0]^);
  3080. taicpu(hp1).opcode:=A_TEST;
  3081. { Shrink the TEST instruction down to the smallest possible size }
  3082. case taicpu(hp1).oper[0]^.val of
  3083. 0..255:
  3084. if (taicpu(hp1).opsize <> S_B)
  3085. {$ifndef x86_64}
  3086. and (
  3087. (taicpu(hp1).oper[1]^.typ <> top_reg) or
  3088. { Cannot encode byte-sized ESI, EDI, EBP or ESP under i386 }
  3089. (GetSupReg(taicpu(hp1).oper[1]^.reg) in [RS_EAX, RS_EBX, RS_ECX, RS_EDX])
  3090. )
  3091. {$endif x86_64}
  3092. then
  3093. begin
  3094. if taicpu(hp1).opsize <> taicpu(hp2).opsize then
  3095. { Only print debug message if the TEST instruction
  3096. is a different size before and after }
  3097. DebugMsg(SPeepholeOptimization + 'test' + debug_opsize2str(taicpu(hp1).opsize) + ' -> testb to reduce instruction size (Test2Test 1a)' , p);
  3098. taicpu(hp1).opsize := S_B;
  3099. if (taicpu(hp1).oper[1]^.typ = top_reg) then
  3100. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBL);
  3101. end;
  3102. 256..65535:
  3103. if (taicpu(hp1).opsize <> S_W) then
  3104. begin
  3105. if taicpu(hp1).opsize <> taicpu(hp2).opsize then
  3106. { Only print debug message if the TEST instruction
  3107. is a different size before and after }
  3108. DebugMsg(SPeepholeOptimization + 'test' + debug_opsize2str(taicpu(hp1).opsize) + ' -> testw to reduce instruction size (Test2Test 1b)' , p);
  3109. taicpu(hp1).opsize := S_W;
  3110. if (taicpu(hp1).oper[1]^.typ = top_reg) then
  3111. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBW);
  3112. end;
  3113. {$ifdef x86_64}
  3114. 65536..$7FFFFFFF:
  3115. if (taicpu(hp1).opsize <> S_L) then
  3116. begin
  3117. if taicpu(hp1).opsize <> taicpu(hp2).opsize then
  3118. { Only print debug message if the TEST instruction
  3119. is a different size before and after }
  3120. DebugMsg(SPeepholeOptimization + 'test' + debug_opsize2str(taicpu(hp1).opsize) + ' -> testl to reduce instruction size (Test2Test 1c)' , p);
  3121. taicpu(hp1).opsize := S_L;
  3122. if (taicpu(hp1).oper[1]^.typ = top_reg) then
  3123. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  3124. end;
  3125. {$endif x86_64}
  3126. else
  3127. ;
  3128. end;
  3129. RemoveInstruction(hp2);
  3130. RemoveCurrentP(p);
  3131. Result:=true;
  3132. exit;
  3133. end;
  3134. end;
  3135. end;
  3136. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  3137. (taicpu(p).opsize = taicpu(hp1).opsize) and
  3138. (taicpu(hp1).oper[0]^.typ <> top_ref) and
  3139. MatchOperand(taicpu(p).oper[0]^, taicpu(hp1).oper[0]^) and
  3140. MatchOperand(taicpu(p).oper[1]^, taicpu(hp1).oper[1]^) and
  3141. (
  3142. not (cs_opt_level3 in current_settings.optimizerswitches) or
  3143. (taicpu(hp1).oper[0]^.typ = top_const) or
  3144. not RegModifiedBetween(taicpu(hp1).oper[0]^.reg, p, hp1)
  3145. ) then
  3146. begin
  3147. { With:
  3148. mov %reg1,%reg2
  3149. ...
  3150. and %reg1,%reg2
  3151. Or:
  3152. mov $x,%reg2
  3153. ...
  3154. and $x,%reg2
  3155. Remove the 'and' instruction
  3156. }
  3157. DebugMsg(SPeepholeOptimization + 'MovAnd2Mov 4 done',hp1);
  3158. hp2 := tai(hp1.Previous);
  3159. RemoveInstruction(hp1);
  3160. //Include(OptsToCheck, aoc_ForceNewIteration);
  3161. if GetNextHp1(hp2) then
  3162. Continue
  3163. else
  3164. Exit;
  3165. end;
  3166. if IsMOVZXAcceptable and
  3167. (taicpu(p).oper[0]^.typ <> top_const) then { MOVZX only supports registers and memory, not immediates (use MOV for that!) }
  3168. begin
  3169. InputVal := debug_operstr(taicpu(p).oper[0]^);
  3170. MaskNum := debug_tostr(taicpu(hp1).oper[0]^.val);
  3171. case taicpu(p).opsize of
  3172. S_B:
  3173. if (taicpu(hp1).oper[0]^.val = $ff) then
  3174. begin
  3175. { Convert:
  3176. movb x, %regl movb x, %regl
  3177. andw ffh, %regw andl ffh, %regd
  3178. To:
  3179. movzbw x, %regd movzbl x, %regd
  3180. (Identical registers, just different sizes)
  3181. }
  3182. RegName1 := debug_regname(taicpu(p).oper[1]^.reg); { 8-bit register name }
  3183. RegName2 := debug_regname(taicpu(hp1).oper[1]^.reg); { 16/32-bit register name }
  3184. case taicpu(hp1).opsize of
  3185. S_W: NewSize := S_BW;
  3186. S_L: NewSize := S_BL;
  3187. {$ifdef x86_64}
  3188. S_Q: NewSize := S_BQ;
  3189. {$endif x86_64}
  3190. else
  3191. InternalError(2018011510);
  3192. end;
  3193. end
  3194. else
  3195. NewSize := S_NO;
  3196. S_W:
  3197. if (taicpu(hp1).oper[0]^.val = $ffff) then
  3198. begin
  3199. { Convert:
  3200. movw x, %regw
  3201. andl ffffh, %regd
  3202. To:
  3203. movzwl x, %regd
  3204. (Identical registers, just different sizes)
  3205. }
  3206. RegName1 := debug_regname(taicpu(p).oper[1]^.reg); { 16-bit register name }
  3207. RegName2 := debug_regname(taicpu(hp1).oper[1]^.reg); { 32-bit register name }
  3208. case taicpu(hp1).opsize of
  3209. S_L: NewSize := S_WL;
  3210. {$ifdef x86_64}
  3211. S_Q: NewSize := S_WQ;
  3212. {$endif x86_64}
  3213. else
  3214. InternalError(2018011511);
  3215. end;
  3216. end
  3217. else
  3218. NewSize := S_NO;
  3219. else
  3220. NewSize := S_NO;
  3221. end;
  3222. if NewSize <> S_NO then
  3223. begin
  3224. PreMessage := 'mov' + debug_opsize2str(taicpu(p).opsize) + ' ' + InputVal + ',' + RegName1;
  3225. { The actual optimization }
  3226. taicpu(p).opcode := A_MOVZX;
  3227. taicpu(p).changeopsize(NewSize);
  3228. taicpu(p).loadoper(1, taicpu(hp1).oper[1]^);
  3229. { Make sure we deal with any reference counts that were increased }
  3230. if taicpu(hp1).oper[1]^.typ = top_ref then
  3231. begin
  3232. if Assigned(taicpu(hp1).oper[1]^.ref^.symbol) then
  3233. taicpu(hp1).oper[1]^.ref^.symbol.decrefs;
  3234. if Assigned(taicpu(hp1).oper[1]^.ref^.relsymbol) then
  3235. taicpu(hp1).oper[1]^.ref^.relsymbol.decrefs;
  3236. end;
  3237. { Safeguard if "and" is followed by a conditional command }
  3238. TransferUsedRegs(TmpUsedRegs);
  3239. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.next), hp1);
  3240. if (RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)) then
  3241. begin
  3242. { At this point, the "and" command is effectively equivalent to
  3243. "test %reg,%reg". This will be handled separately by the
  3244. Peephole Optimizer. [Kit] }
  3245. DebugMsg(SPeepholeOptimization + PreMessage +
  3246. ' -> movz' + debug_opsize2str(NewSize) + ' ' + InputVal + ',' + RegName2, p);
  3247. end
  3248. else
  3249. begin
  3250. DebugMsg(SPeepholeOptimization + PreMessage + '; and' + debug_opsize2str(taicpu(hp1).opsize) + ' $' + MaskNum + ',' + RegName2 +
  3251. ' -> movz' + debug_opsize2str(NewSize) + ' ' + InputVal + ',' + RegName2, p);
  3252. RemoveInstruction(hp1);
  3253. end;
  3254. Result := True;
  3255. Exit;
  3256. { Go through DeepMOVOpt again (jump to "while True do") }
  3257. Continue;
  3258. end;
  3259. end;
  3260. end;
  3261. if taicpu(p).oper[0]^.typ = top_reg then
  3262. begin
  3263. p_SourceReg := taicpu(p).oper[0]^.reg;
  3264. { Look for:
  3265. mov %reg1,%reg2
  3266. ??? %reg2,r/m
  3267. Change to:
  3268. mov %reg1,%reg2
  3269. ??? %reg1,r/m
  3270. }
  3271. if RegReadByInstruction(p_TargetReg, hp1) and
  3272. not RegModifiedBetween(p_SourceReg, p, hp1) and
  3273. DeepMOVOpt(taicpu(p), taicpu(hp1)) then
  3274. begin
  3275. { A change has occurred, just not in p }
  3276. Include(OptsToCheck, aoc_ForceNewIteration);
  3277. TransferUsedRegs(TmpUsedRegs);
  3278. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  3279. if not RegUsedAfterInstruction(p_TargetReg, hp1, TmpUsedRegs) and
  3280. { Just in case something didn't get modified (e.g. an
  3281. implicit register) }
  3282. not RegReadByInstruction(p_TargetReg, hp1) then
  3283. begin
  3284. { We can remove the original MOV }
  3285. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3 done',p);
  3286. RemoveCurrentP(p);
  3287. { UsedRegs got updated by RemoveCurrentp }
  3288. Result := True;
  3289. Exit;
  3290. end;
  3291. { If we know a MOV instruction has become a null operation, we might as well
  3292. get rid of it now to save time. }
  3293. if (taicpu(hp1).opcode = A_MOV) and
  3294. (taicpu(hp1).oper[1]^.typ = top_reg) and
  3295. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[0]^.reg) and
  3296. { Just being a register is enough to confirm it's a null operation }
  3297. (taicpu(hp1).oper[0]^.typ = top_reg) then
  3298. begin
  3299. Result := True;
  3300. { Speed-up to reduce a pipeline stall... if we had something like...
  3301. movl %eax,%edx
  3302. movw %dx,%ax
  3303. ... the second instruction would change to movw %ax,%ax, but
  3304. given that it is now %ax that's active rather than %eax,
  3305. penalties might occur due to a partial register write, so instead,
  3306. change it to a MOVZX instruction when optimising for speed.
  3307. }
  3308. if not (cs_opt_size in current_settings.optimizerswitches) and
  3309. IsMOVZXAcceptable and
  3310. (taicpu(hp1).opsize < taicpu(p).opsize)
  3311. {$ifdef x86_64}
  3312. { operations already implicitly set the upper 64 bits to zero }
  3313. and not ((taicpu(hp1).opsize = S_L) and (taicpu(p).opsize = S_Q))
  3314. {$endif x86_64}
  3315. then
  3316. begin
  3317. DebugMsg(SPeepholeOptimization + 'Zero-extension to minimise pipeline stall (Mov2Movz)',hp1);
  3318. case taicpu(p).opsize of
  3319. S_W:
  3320. if taicpu(hp1).opsize = S_B then
  3321. taicpu(hp1).opsize := S_BL
  3322. else
  3323. InternalError(2020012911);
  3324. S_L{$ifdef x86_64}, S_Q{$endif x86_64}:
  3325. case taicpu(hp1).opsize of
  3326. S_B:
  3327. taicpu(hp1).opsize := S_BL;
  3328. S_W:
  3329. taicpu(hp1).opsize := S_WL;
  3330. else
  3331. InternalError(2020012912);
  3332. end;
  3333. else
  3334. InternalError(2020012910);
  3335. end;
  3336. taicpu(hp1).opcode := A_MOVZX;
  3337. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  3338. end
  3339. else
  3340. begin
  3341. GetNextInstruction_p := GetNextInstruction(hp1, hp2);
  3342. DebugMsg(SPeepholeOptimization + 'Mov2Nop 4 done',hp1);
  3343. RemoveInstruction(hp1);
  3344. { The instruction after what was hp1 is now the immediate next instruction,
  3345. so we can continue to make optimisations if it's present }
  3346. if not GetNextInstruction_p or (hp2.typ <> ait_instruction) then
  3347. Exit;
  3348. hp1 := hp2;
  3349. end;
  3350. end;
  3351. end;
  3352. {$ifdef x86_64}
  3353. { Change:
  3354. movl %reg1l,%reg2l
  3355. movq %reg2q,%reg1q
  3356. To:
  3357. movl %reg1l,%reg2l
  3358. andl %reg1l,%reg1l
  3359. }
  3360. if (taicpu(p).opsize = S_L) and MatchInstruction(hp1,A_MOV,[S_Q]) and
  3361. not RegModifiedBetween(p_SourceReg, p, hp1) and
  3362. MatchOpType(taicpu(hp1), top_reg, top_reg) and
  3363. SuperRegistersEqual(p_TargetReg, taicpu(hp1).oper[0]^.reg) and
  3364. SuperRegistersEqual(p_SourceReg, taicpu(hp1).oper[1]^.reg) then
  3365. begin
  3366. TransferUsedRegs(TmpUsedRegs);
  3367. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  3368. taicpu(hp1).opsize := S_L;
  3369. taicpu(hp1).loadreg(0, p_SourceReg);
  3370. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  3371. AllocRegBetween(p_SourceReg, p, hp1, UsedRegs);
  3372. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlAndl 1)', hp1);
  3373. taicpu(hp1).opcode := A_AND;
  3374. { We may be able to do more and replace references
  3375. to %reg2q with %reg1q etc. }
  3376. if (cs_opt_level3 in current_settings.optimizerswitches) and
  3377. { p_TargetReg is not used between, otherwise the earlier
  3378. GetNextInstructionUsingReg would have stopped sooner }
  3379. DoZeroUpper32Opt(p,hp1) then
  3380. begin
  3381. Result := True;
  3382. Exit;
  3383. end;
  3384. end;
  3385. {
  3386. If we have the following already in the code...
  3387. movl %reg1l,%reg2l
  3388. andl %reg1l,%reg1l
  3389. ...we may be able to do more and replace references to
  3390. %reg2q with %reg1q etc. (program flow won't reach this
  3391. point if the second instruction was originally a MOV
  3392. and just got changed to AND)
  3393. }
  3394. if (cs_opt_level3 in current_settings.optimizerswitches) and
  3395. (taicpu(p).opsize = S_L) and MatchInstruction(hp1,A_AND,[S_L]) and
  3396. not RegModifiedBetween(p_SourceReg, p, hp1) and
  3397. { p_TargetReg is not used between, otherwise the earlier
  3398. GetNextInstructionUsingReg would have stopped sooner }
  3399. MatchOperand(taicpu(hp1).oper[1]^, p_SourceReg) and
  3400. (
  3401. MatchOperand(taicpu(hp1).oper[0]^, p_SourceReg) or
  3402. MatchOperand(taicpu(hp1).oper[0]^, $ffffffff)
  3403. ) and
  3404. DoZeroUpper32Opt(p,hp1) then
  3405. begin
  3406. Result := True;
  3407. Exit;
  3408. end;
  3409. {$endif x86_64}
  3410. end
  3411. else if taicpu(p).oper[0]^.typ = top_const then
  3412. begin
  3413. if (taicpu(hp1).opcode = A_OR) and
  3414. (taicpu(p).oper[1]^.typ = top_reg) and
  3415. MatchOperand(taicpu(p).oper[0]^, 0) and
  3416. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) then
  3417. begin
  3418. { mov 0, %reg
  3419. or ###,%reg
  3420. Change to (only if the flags are not used):
  3421. mov ###,%reg
  3422. }
  3423. TransferUsedRegs(TmpUsedRegs);
  3424. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  3425. DoOptimisation := True;
  3426. { Even if the flags are used, we might be able to do the optimisation
  3427. if the conditions are predictable }
  3428. if RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  3429. begin
  3430. { Only perform if ### = %reg (the same register) or equal to 0,
  3431. so %reg is guaranteed to still have a value of zero }
  3432. if MatchOperand(taicpu(hp1).oper[0]^, 0) or
  3433. MatchOperand(taicpu(hp1).oper[0]^, taicpu(hp1).oper[1]^.reg) then
  3434. begin
  3435. hp2 := hp1;
  3436. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  3437. while RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  3438. GetNextInstruction(hp2, hp3) do
  3439. begin
  3440. { Don't continue modifying if the flags state is getting changed }
  3441. if RegModifiedByInstruction(NR_DEFAULTFLAGS, hp3) then
  3442. Break;
  3443. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  3444. if MatchInstruction(hp3, A_Jcc, A_SETcc, A_CMOVcc, []) then
  3445. begin
  3446. if condition_in(C_E, taicpu(hp3).condition) or (taicpu(hp3).condition in [C_NC, C_NS, C_NO]) then
  3447. begin
  3448. { Condition is always true }
  3449. case taicpu(hp3).opcode of
  3450. A_Jcc:
  3451. begin
  3452. { Check for jump shortcuts before we destroy the condition }
  3453. hp4 := hp3;
  3454. DoJumpOptimizations(hp3, TempBool);
  3455. { Make sure hp3 hasn't changed }
  3456. if (hp4 = hp3) then
  3457. begin
  3458. DebugMsg(SPeepholeOptimization + 'Condition is always true (jump made unconditional)', hp3);
  3459. MakeUnconditional(taicpu(hp3));
  3460. end;
  3461. Result := True;
  3462. end;
  3463. A_CMOVcc:
  3464. begin
  3465. DebugMsg(SPeepholeOptimization + 'Condition is always true (CMOVcc -> MOV)', hp3);
  3466. taicpu(hp3).opcode := A_MOV;
  3467. taicpu(hp3).condition := C_None;
  3468. Result := True;
  3469. end;
  3470. A_SETcc:
  3471. begin
  3472. DebugMsg(SPeepholeOptimization + 'Condition is always true (changed to MOV 1)', hp3);
  3473. { Convert "set(c) %reg" instruction to "movb 1,%reg" }
  3474. taicpu(hp3).opcode := A_MOV;
  3475. taicpu(hp3).ops := 2;
  3476. taicpu(hp3).condition := C_None;
  3477. taicpu(hp3).opsize := S_B;
  3478. taicpu(hp3).loadreg(1,taicpu(hp3).oper[0]^.reg);
  3479. taicpu(hp3).loadconst(0, 1);
  3480. Result := True;
  3481. end;
  3482. else
  3483. InternalError(2021090701);
  3484. end;
  3485. end
  3486. else if (taicpu(hp3).condition in [C_A, C_B, C_C, C_G, C_L, C_NE, C_NZ, C_O, C_S]) then
  3487. begin
  3488. { Condition is always false }
  3489. case taicpu(hp3).opcode of
  3490. A_Jcc:
  3491. begin
  3492. DebugMsg(SPeepholeOptimization + 'Condition is always false (jump removed)', hp3);
  3493. TAsmLabel(taicpu(hp3).oper[0]^.ref^.symbol).decrefs;
  3494. RemoveInstruction(hp3);
  3495. Result := True;
  3496. { Since hp3 was deleted, hp2 must not be updated }
  3497. Continue;
  3498. end;
  3499. A_CMOVcc:
  3500. begin
  3501. DebugMsg(SPeepholeOptimization + 'Condition is always false (conditional load removed)', hp3);
  3502. RemoveInstruction(hp3);
  3503. Result := True;
  3504. { Since hp3 was deleted, hp2 must not be updated }
  3505. Continue;
  3506. end;
  3507. A_SETcc:
  3508. begin
  3509. DebugMsg(SPeepholeOptimization + 'Condition is always false (changed to MOV 0)', hp3);
  3510. { Convert "set(c) %reg" instruction to "movb 0,%reg" }
  3511. taicpu(hp3).opcode := A_MOV;
  3512. taicpu(hp3).ops := 2;
  3513. taicpu(hp3).condition := C_None;
  3514. taicpu(hp3).opsize := S_B;
  3515. taicpu(hp3).loadreg(1,taicpu(hp3).oper[0]^.reg);
  3516. taicpu(hp3).loadconst(0, 0);
  3517. Result := True;
  3518. end;
  3519. else
  3520. InternalError(2021090702);
  3521. end;
  3522. end
  3523. else
  3524. { Uncertain what to do - don't optimise (although optimise other conditional statements if present) }
  3525. DoOptimisation := False;
  3526. end;
  3527. hp2 := hp3;
  3528. end;
  3529. if DoOptimisation then
  3530. begin
  3531. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  3532. if RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  3533. { Flags are still in use - don't optimise }
  3534. DoOptimisation := False;
  3535. end;
  3536. end
  3537. else
  3538. DoOptimisation := False;
  3539. end;
  3540. if DoOptimisation then
  3541. begin
  3542. {$ifdef x86_64}
  3543. { OR only supports 32-bit sign-extended constants for 64-bit
  3544. instructions, so compensate for this if the constant is
  3545. encoded as a value greater than or equal to 2^31 }
  3546. if (taicpu(hp1).opsize = S_Q) and
  3547. (taicpu(hp1).oper[0]^.typ = top_const) and
  3548. (taicpu(hp1).oper[0]^.val >= $80000000) then
  3549. taicpu(hp1).oper[0]^.val := taicpu(hp1).oper[0]^.val or $FFFFFFFF00000000;
  3550. {$endif x86_64}
  3551. DebugMsg(SPeepholeOptimization + 'MOV 0 / OR -> MOV', p);
  3552. taicpu(hp1).opcode := A_MOV;
  3553. RemoveCurrentP(p);
  3554. Result := True;
  3555. Exit;
  3556. end;
  3557. end;
  3558. end
  3559. else if
  3560. { oper[0] is a reference }
  3561. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) then
  3562. begin
  3563. if MatchInstruction(hp1,A_LEA,[S_L{$ifdef x86_64},S_Q{$endif x86_64}]) then
  3564. begin
  3565. if ((MatchReference(Taicpu(hp1).oper[0]^.ref^,Taicpu(hp1).oper[1]^.reg,Taicpu(p).oper[1]^.reg) and
  3566. (Taicpu(hp1).oper[0]^.ref^.base<>Taicpu(p).oper[1]^.reg)
  3567. ) or
  3568. (MatchReference(Taicpu(hp1).oper[0]^.ref^,Taicpu(p).oper[1]^.reg,Taicpu(hp1).oper[1]^.reg) and
  3569. (Taicpu(hp1).oper[0]^.ref^.index<>Taicpu(p).oper[1]^.reg)
  3570. )
  3571. ) and
  3572. not RegModifiedBetween(Taicpu(hp1).oper[1]^.reg, p, hp1) then
  3573. { mov ref,reg1
  3574. lea (reg1,reg2),reg2
  3575. to
  3576. add ref,reg2 }
  3577. begin
  3578. TransferUsedRegs(TmpUsedRegs);
  3579. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  3580. { If the flags register is in use, don't change the instruction to an
  3581. ADD otherwise this will scramble the flags. [Kit] }
  3582. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  3583. { reg1 may not be used afterwards }
  3584. not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs)) then
  3585. begin
  3586. Taicpu(hp1).opcode:=A_ADD;
  3587. Taicpu(hp1).oper[0]^.ref^:=Taicpu(p).oper[0]^.ref^;
  3588. DebugMsg(SPeepholeOptimization + 'MovLea2Add done',hp1);
  3589. RemoveCurrentp(p);
  3590. result:=true;
  3591. exit;
  3592. end;
  3593. end;
  3594. { If the LEA instruction can be converted into an arithmetic instruction,
  3595. it may be possible to then fold it in the next optimisation. }
  3596. if ConvertLEA(taicpu(hp1)) then
  3597. Include(OptsToCheck, aoc_ForceNewIteration);
  3598. end;
  3599. {
  3600. mov ref,reg0
  3601. <op> reg0,reg1
  3602. dealloc reg0
  3603. to
  3604. <op> ref,reg1
  3605. }
  3606. if MatchOpType(taicpu(hp1),top_reg,top_reg) and
  3607. (taicpu(hp1).oper[0]^.reg = p_TargetReg) and
  3608. MatchInstruction(hp1, [A_AND, A_OR, A_XOR, A_ADD, A_SUB, A_CMP, A_TEST, A_CMOVcc, A_BSR, A_BSF, A_POPCNT, A_LZCNT], [taicpu(p).opsize]) and
  3609. not SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, p_TargetReg) and
  3610. not RefModifiedBetween(taicpu(p).oper[0]^.ref^, topsize2memsize[taicpu(p).opsize] shr 3, p, hp1) then
  3611. begin
  3612. TransferUsedRegs(TmpUsedRegs);
  3613. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  3614. if not RegUsedAfterInstruction(p_TargetReg, hp1, TmpUsedRegs) then
  3615. begin
  3616. taicpu(hp1).loadref(0,taicpu(p).oper[0]^.ref^);
  3617. { loadref increases the reference count, so decrement it again }
  3618. if Assigned(taicpu(p).oper[0]^.ref^.symbol) then
  3619. taicpu(p).oper[0]^.ref^.symbol.decrefs;
  3620. if Assigned(taicpu(p).oper[0]^.ref^.relsymbol) then
  3621. taicpu(p).oper[0]^.ref^.relsymbol.decrefs;
  3622. DebugMsg(SPeepholeOptimization + 'MovOp2Op done',hp1);
  3623. { See if we can remove the allocation of reg0 }
  3624. if not RegInRef(p_TargetReg, taicpu(p).oper[0]^.ref^) then
  3625. TryRemoveRegAlloc(p_TargetReg, p, hp1);
  3626. RemoveCurrentp(p);
  3627. Result:=true;
  3628. exit;
  3629. end;
  3630. end;
  3631. end;
  3632. { Depending on the DeepMOVOpt above, it may turn out that hp1 completely
  3633. overwrites the original destination register. e.g.
  3634. movl ###,%reg2d
  3635. movslq ###,%reg2q (### doesn't have to be the same as the first one)
  3636. In this case, we can remove the MOV (Go to "Mov2Nop 5" below)
  3637. }
  3638. if MatchInstruction(hp1, [A_LEA, A_MOV, A_MOVSX, A_MOVZX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}], []) and
  3639. (taicpu(hp1).oper[1]^.typ = top_reg) and
  3640. Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[1]^.reg) then
  3641. begin
  3642. if RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^) then
  3643. begin
  3644. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  3645. case taicpu(p).oper[0]^.typ of
  3646. top_const:
  3647. { We have something like:
  3648. movb $x, %regb
  3649. movzbl %regb,%regd
  3650. Change to:
  3651. movl $x, %regd
  3652. }
  3653. begin
  3654. case taicpu(hp1).opsize of
  3655. S_BW:
  3656. begin
  3657. convert_mov_value(A_MOVSX, $FF);
  3658. setsubreg(taicpu(p).oper[1]^.reg, R_SUBW);
  3659. taicpu(p).opsize := S_W;
  3660. end;
  3661. S_BL:
  3662. begin
  3663. convert_mov_value(A_MOVSX, $FF);
  3664. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  3665. taicpu(p).opsize := S_L;
  3666. end;
  3667. S_WL:
  3668. begin
  3669. convert_mov_value(A_MOVSX, $FFFF);
  3670. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  3671. taicpu(p).opsize := S_L;
  3672. end;
  3673. {$ifdef x86_64}
  3674. S_BQ:
  3675. begin
  3676. convert_mov_value(A_MOVSX, $FF);
  3677. setsubreg(taicpu(p).oper[1]^.reg, R_SUBQ);
  3678. taicpu(p).opsize := S_Q;
  3679. end;
  3680. S_WQ:
  3681. begin
  3682. convert_mov_value(A_MOVSX, $FFFF);
  3683. setsubreg(taicpu(p).oper[1]^.reg, R_SUBQ);
  3684. taicpu(p).opsize := S_Q;
  3685. end;
  3686. S_LQ:
  3687. begin
  3688. convert_mov_value(A_MOVSXD, $FFFFFFFF); { Note it's MOVSXD, not MOVSX }
  3689. setsubreg(taicpu(p).oper[1]^.reg, R_SUBQ);
  3690. taicpu(p).opsize := S_Q;
  3691. end;
  3692. {$endif x86_64}
  3693. else
  3694. { If hp1 was a MOV instruction, it should have been
  3695. optimised already }
  3696. InternalError(2020021001);
  3697. end;
  3698. DebugMsg(SPeepholeOptimization + 'MovMovXX2MovXX 2 done',p);
  3699. RemoveInstruction(hp1);
  3700. Result := True;
  3701. Exit;
  3702. end;
  3703. top_ref:
  3704. begin
  3705. { We have something like:
  3706. movb mem, %regb
  3707. movzbl %regb,%regd
  3708. Change to:
  3709. movzbl mem, %regd
  3710. }
  3711. if (taicpu(p).oper[0]^.ref^.refaddr<>addr_full) and (IsMOVZXAcceptable or (taicpu(hp1).opcode<>A_MOVZX)) then
  3712. begin
  3713. DebugMsg(SPeepholeOptimization + 'MovMovXX2MovXX 1 done',p);
  3714. taicpu(p).opcode := taicpu(hp1).opcode;
  3715. taicpu(p).opsize := taicpu(hp1).opsize;
  3716. taicpu(p).oper[1]^.reg := taicpu(hp1).oper[1]^.reg;
  3717. RemoveInstruction(hp1);
  3718. Result := True;
  3719. Exit;
  3720. end;
  3721. end;
  3722. else
  3723. if (taicpu(hp1).opcode <> A_MOV) and (taicpu(hp1).opcode <> A_LEA) then
  3724. { Just to make a saving, since there are no more optimisations with MOVZX and MOVSX/D }
  3725. Exit;
  3726. end;
  3727. end
  3728. { The RegInOp check makes sure that movl r/m,%reg1l; movzbl (%reg1l),%reg1l"
  3729. and "movl r/m,%reg1; leal $1(%reg1,%reg2),%reg1" etc. are not incorrectly
  3730. optimised }
  3731. else
  3732. begin
  3733. DebugMsg(SPeepholeOptimization + 'Mov2Nop 5 done',p);
  3734. RemoveCurrentP(p);
  3735. Result := True;
  3736. Exit;
  3737. end;
  3738. end;
  3739. if (taicpu(hp1).opcode = A_MOV) and
  3740. (
  3741. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^)
  3742. {$ifdef x86_64}
  3743. or (
  3744. { Permit zero extension from 32- to 64-bit when writing
  3745. a constant (it will be checked to see if it fits into
  3746. a signed 32-bit integer) }
  3747. (taicpu(p).opsize=S_L) and (taicpu(hp1).opsize=S_Q) and
  3748. (
  3749. { Valid situations... writing an unsigned 32-bit
  3750. immediate, or the destination is a 64-bit register }
  3751. (taicpu(p).oper[0]^.typ = top_const) or
  3752. (taicpu(hp1).oper[1]^.typ = top_reg)
  3753. ) and
  3754. (taicpu(hp1).oper[0]^.typ = top_reg) and
  3755. SuperRegistersEqual(p_TargetReg, taicpu(hp1).oper[0]^.reg)
  3756. )
  3757. {$endif x86_64}
  3758. ) then
  3759. begin
  3760. { Remember that p_TargetReg contains taicpu(p).oper[1]^.reg }
  3761. TransferUsedRegs(TmpUsedRegs);
  3762. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  3763. { we have
  3764. mov x, %treg
  3765. mov %treg, y
  3766. }
  3767. if not(RegInOp(p_TargetReg, taicpu(hp1).oper[1]^)) then
  3768. if not(RegUsedAfterInstruction(p_TargetReg, hp1, TmpUsedRegs)) then
  3769. begin
  3770. { we've got
  3771. mov x, %treg
  3772. mov %treg, y
  3773. with %treg is not used after }
  3774. case taicpu(p).oper[0]^.typ Of
  3775. { top_reg is covered by DeepMOVOpt }
  3776. top_const:
  3777. begin
  3778. { change
  3779. mov const, %treg
  3780. mov %treg, y
  3781. to
  3782. mov const, y
  3783. }
  3784. {$ifdef x86_64}
  3785. if (taicpu(hp1).oper[1]^.typ=top_reg) or
  3786. (
  3787. { For 32-to-64-bit zero-extension, the immediate
  3788. must be between 0 and 2^31 - 1}
  3789. (taicpu(p).opsize=S_L) and (taicpu(hp1).opsize=S_Q) and
  3790. ((taicpu(p).oper[0]^.val>=0) and (taicpu(p).oper[0]^.val<=high(longint)))
  3791. ) or
  3792. (
  3793. not ((taicpu(p).opsize=S_L) and (taicpu(hp1).opsize=S_Q)) and
  3794. (
  3795. (taicpu(p).oper[0]^.val>=low(longint)) and (taicpu(p).oper[0]^.val<=high(longint))
  3796. )
  3797. ) then
  3798. {$endif x86_64}
  3799. begin
  3800. taicpu(hp1).loadconst(0, taicpu(p).oper[0]^.val);
  3801. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 5 done', hp1);
  3802. RemoveCurrentP(p);
  3803. Result := True;
  3804. Exit;
  3805. end;
  3806. end;
  3807. top_ref:
  3808. case taicpu(hp1).oper[1]^.typ of
  3809. top_reg:
  3810. { change
  3811. mov mem, %treg
  3812. mov %treg, %reg
  3813. to
  3814. mov mem, %reg"
  3815. }
  3816. if not RegUsedBetween(taicpu(hp1).oper[1]^.reg, p, hp1) then
  3817. begin
  3818. {$ifdef x86_64}
  3819. { If zero extending from 32-bit to 64-bit,
  3820. we have to make sure the replaced
  3821. register is the right size }
  3822. taicpu(p).loadreg(1, newreg(R_INTREGISTER,getsupreg(taicpu(hp1).oper[1]^.reg),getsubreg(p_TargetReg)));
  3823. {$else}
  3824. taicpu(p).loadreg(1, taicpu(hp1).oper[1]^.reg);
  3825. {$endif x86_64}
  3826. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 3a done', p);
  3827. AllocRegBetween(taicpu(hp1).oper[1]^.reg, p, hp1, UsedRegs);
  3828. RemoveInstruction(hp1);
  3829. Result := True;
  3830. Exit;
  3831. end
  3832. else if
  3833. { Make sure that if a reference is used, its
  3834. registers are not modified in between }
  3835. not RefModifiedBetween(taicpu(p).oper[0]^.ref^, topsize2memsize[taicpu(p).opsize] shr 3, p, hp1) then
  3836. begin
  3837. if (taicpu(p).oper[0]^.ref^.base <> NR_NO){$ifdef x86_64} and (taicpu(p).oper[0]^.ref^.base <> NR_RIP){$endif x86_64} then
  3838. AllocRegBetween(taicpu(p).oper[0]^.ref^.base, p, hp1, UsedRegs);
  3839. if (taicpu(p).oper[0]^.ref^.index <> NR_NO) and (taicpu(p).oper[0]^.ref^.index <> taicpu(p).oper[0]^.ref^.base) then
  3840. AllocRegBetween(taicpu(p).oper[0]^.ref^.index, p, hp1, UsedRegs);
  3841. taicpu(hp1).loadref(0, taicpu(p).oper[0]^.ref^);
  3842. if Assigned(taicpu(p).oper[0]^.ref^.symbol) then
  3843. taicpu(p).oper[0]^.ref^.symbol.decrefs;
  3844. if Assigned(taicpu(p).oper[0]^.ref^.relsymbol) then
  3845. taicpu(p).oper[0]^.ref^.relsymbol.decrefs;
  3846. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 3 done', hp1);
  3847. RemoveCurrentP(p);
  3848. Result := True;
  3849. Exit;
  3850. end;
  3851. top_ref:
  3852. if not RegInRef(p_TargetReg, taicpu(p).oper[0]^.ref^) then
  3853. begin
  3854. {$ifdef x86_64}
  3855. { Look for the following to simplify:
  3856. mov x(mem1), %reg
  3857. mov %reg, y(mem2)
  3858. mov x+8(mem1), %reg
  3859. mov %reg, y+8(mem2)
  3860. Change to:
  3861. movdqu x(mem1), %xmmreg
  3862. movdqu %xmmreg, y(mem2)
  3863. ...but only as long as the memory blocks don't overlap
  3864. }
  3865. SourceRef := taicpu(p).oper[0]^.ref^;
  3866. TargetRef := taicpu(hp1).oper[1]^.ref^;
  3867. if (taicpu(p).opsize = S_Q) and
  3868. not RegUsedAfterInstruction(p_TargetReg, hp1, TmpUsedRegs) and
  3869. GetNextInstruction(hp1, hp2) and
  3870. MatchInstruction(hp2, A_MOV, [taicpu(p).opsize]) and
  3871. MatchOpType(taicpu(hp2), top_ref, top_reg) then
  3872. begin
  3873. { Delay calling GetNextInstruction(hp2, hp3) for as long as possible }
  3874. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  3875. Inc(SourceRef.offset, 8);
  3876. if UseAVX then
  3877. begin
  3878. MovAligned := A_VMOVDQA;
  3879. MovUnaligned := A_VMOVDQU;
  3880. end
  3881. else
  3882. begin
  3883. MovAligned := A_MOVDQA;
  3884. MovUnaligned := A_MOVDQU;
  3885. end;
  3886. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) and
  3887. not RefsMightOverlap(taicpu(p).oper[0]^.ref^, TargetRef, 16) then
  3888. begin
  3889. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  3890. Inc(TargetRef.offset, 8);
  3891. if GetNextInstruction(hp2, hp3) and
  3892. MatchInstruction(hp3, A_MOV, [taicpu(p).opsize]) and
  3893. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  3894. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  3895. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  3896. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  3897. begin
  3898. NewMMReg := GetMMRegisterBetween(R_SUBMMX, UsedRegs, p, hp3);
  3899. if NewMMReg <> NR_NO then
  3900. begin
  3901. { Remember that the offsets are 8 ahead }
  3902. if ((SourceRef.offset mod 16) = 8) and
  3903. (
  3904. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3905. (SourceRef.base = current_procinfo.framepointer) or
  3906. ((SourceRef.alignment >= 16) and ((SourceRef.alignment mod 16) = 0))
  3907. ) then
  3908. taicpu(p).opcode := MovAligned
  3909. else
  3910. taicpu(p).opcode := MovUnaligned;
  3911. taicpu(p).opsize := S_XMM;
  3912. taicpu(p).oper[1]^.reg := NewMMReg;
  3913. if ((TargetRef.offset mod 16) = 8) and
  3914. (
  3915. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3916. (TargetRef.base = current_procinfo.framepointer) or
  3917. ((TargetRef.alignment >= 16) and ((TargetRef.alignment mod 16) = 0))
  3918. ) then
  3919. taicpu(hp1).opcode := MovAligned
  3920. else
  3921. taicpu(hp1).opcode := MovUnaligned;
  3922. taicpu(hp1).opsize := S_XMM;
  3923. taicpu(hp1).oper[0]^.reg := NewMMReg;
  3924. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(NewMMReg) + ' to merge a pair of memory moves (MovMovMovMov2MovdqMovdq 1)', p);
  3925. RemoveInstruction(hp2);
  3926. RemoveInstruction(hp3);
  3927. Result := True;
  3928. Exit;
  3929. end;
  3930. end;
  3931. end
  3932. else
  3933. begin
  3934. { See if the next references are 8 less rather than 8 greater }
  3935. Dec(SourceRef.offset, 16); { -8 the other way }
  3936. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) then
  3937. begin
  3938. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  3939. Dec(TargetRef.offset, 8); { Only 8, not 16, as it wasn't incremented unlike SourceRef }
  3940. if not RefsMightOverlap(SourceRef, TargetRef, 16) and
  3941. GetNextInstruction(hp2, hp3) and
  3942. MatchInstruction(hp3, A_MOV, [taicpu(p).opsize]) and
  3943. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  3944. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  3945. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  3946. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  3947. begin
  3948. NewMMReg := GetMMRegisterBetween(R_SUBMMX, UsedRegs, p, hp3);
  3949. if NewMMReg <> NR_NO then
  3950. begin
  3951. { hp2 and hp3 are the starting offsets, so mod = 0 this time }
  3952. if ((SourceRef.offset mod 16) = 0) and
  3953. (
  3954. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3955. (SourceRef.base = current_procinfo.framepointer) or
  3956. ((SourceRef.alignment >= 16) and ((SourceRef.alignment mod 16) = 0))
  3957. ) then
  3958. taicpu(hp2).opcode := MovAligned
  3959. else
  3960. taicpu(hp2).opcode := MovUnaligned;
  3961. taicpu(hp2).opsize := S_XMM;
  3962. taicpu(hp2).oper[1]^.reg := NewMMReg;
  3963. if ((TargetRef.offset mod 16) = 0) and
  3964. (
  3965. { Base pointer is always aligned (stack pointer won't be if there's no stack frame) }
  3966. (TargetRef.base = current_procinfo.framepointer) or
  3967. ((TargetRef.alignment >= 16) and ((TargetRef.alignment mod 16) = 0))
  3968. ) then
  3969. taicpu(hp3).opcode := MovAligned
  3970. else
  3971. taicpu(hp3).opcode := MovUnaligned;
  3972. taicpu(hp3).opsize := S_XMM;
  3973. taicpu(hp3).oper[0]^.reg := NewMMReg;
  3974. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(NewMMReg) + ' to merge a pair of memory moves (MovMovMovMov2MovdqMovdq 2)', p);
  3975. RemoveInstruction(hp1);
  3976. RemoveCurrentP(p);
  3977. Result := True;
  3978. Exit;
  3979. end;
  3980. end;
  3981. end;
  3982. end;
  3983. end;
  3984. {$endif x86_64}
  3985. end;
  3986. else
  3987. { The write target should be a reg or a ref }
  3988. InternalError(2021091601);
  3989. end;
  3990. else
  3991. ;
  3992. end;
  3993. end
  3994. else if (taicpu(p).oper[0]^.typ = top_const) and
  3995. { %treg is used afterwards, but all eventualities other
  3996. than the first MOV instruction being a constant are
  3997. covered by DeepMOVOpt, so only check for that }
  3998. (
  3999. { For MOV operations, a size saving is only made if the register/const is byte-sized }
  4000. not (cs_opt_size in current_settings.optimizerswitches) or
  4001. (taicpu(hp1).opsize = S_B)
  4002. ) and
  4003. (
  4004. (taicpu(hp1).oper[1]^.typ=top_reg) or
  4005. (
  4006. { For 32-to-64-bit zero-extension, the immediate
  4007. must be between 0 and 2^31 - 1}
  4008. (taicpu(p).opsize=S_L) and (taicpu(hp1).opsize=S_Q) and
  4009. ((taicpu(p).oper[0]^.val>=0) and (taicpu(p).oper[0]^.val<=high(longint)))
  4010. ) or
  4011. (
  4012. not ((taicpu(p).opsize=S_L) and (taicpu(hp1).opsize=S_Q)) and
  4013. (
  4014. (taicpu(p).oper[0]^.val>=low(longint)) and (taicpu(p).oper[0]^.val<=high(longint))
  4015. )
  4016. )
  4017. ) then
  4018. begin
  4019. DebugMsg(SPeepholeOptimization + debug_operstr(taicpu(hp1).oper[0]^) + ' = $' + debug_tostr(taicpu(p).oper[0]^.val) + '; changed to minimise pipeline stall (MovMov2Mov 6b)',hp1);
  4020. taicpu(hp1).loadconst(0, taicpu(p).oper[0]^.val);
  4021. Include(OptsToCheck, aoc_ForceNewIteration);
  4022. end;
  4023. end;
  4024. Break;
  4025. end;
  4026. end;
  4027. if taicpu(p).oper[0]^.typ = top_reg then
  4028. begin
  4029. { oper[1] is a reference }
  4030. { Saves on a large number of dereferences }
  4031. p_SourceReg := taicpu(p).oper[0]^.reg;
  4032. if NotFirstIteration and (cs_opt_level3 in current_settings.optimizerswitches) then
  4033. GetNextInstruction_p := GetNextInstructionUsingReg(p, hp1, p_SourceReg)
  4034. else
  4035. GetNextInstruction_p := GetNextInstruction(p, hp1);
  4036. if GetNextInstruction_p and (hp1.typ = ait_instruction) then
  4037. begin
  4038. if taicpu(p).oper[1]^.typ = top_reg then
  4039. begin
  4040. p_TargetReg := taicpu(p).oper[1]^.reg;
  4041. { Change:
  4042. movl %reg1,%reg2
  4043. ...
  4044. movl x(%reg1),%reg1 (If something other than %reg1 is written to, DeepMOVOpt would have caught it)
  4045. ...
  4046. movl x(%reg2),%regX (%regX can be %reg2 or something else)
  4047. To:
  4048. movl %reg1,%reg2 (if %regX = %reg2, then remove this instruction)
  4049. ...
  4050. movl x(%reg1),%reg1
  4051. ...
  4052. movl %reg1,%regX
  4053. }
  4054. if MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  4055. (taicpu(hp1).oper[0]^.typ = top_ref) { The other operand will be a register } and
  4056. (taicpu(hp1).oper[1]^.reg = p_SourceReg) and
  4057. RegInRef(p_SourceReg, taicpu(hp1).oper[0]^.ref^) and
  4058. not RegModifiedBetween(p_TargetReg, p, hp1) and
  4059. GetNextInstructionUsingReg(hp1, hp2, p_TargetReg) and
  4060. MatchInstruction(hp2, A_MOV, [taicpu(p).opsize]) and
  4061. (taicpu(hp2).oper[0]^.typ = top_ref) { The other operand will be a register } and
  4062. not RegModifiedBetween(p_SourceReg, hp1, hp2) then
  4063. begin
  4064. SourceRef := taicpu(hp2).oper[0]^.ref^;
  4065. if RegInRef(p_TargetReg, SourceRef) and
  4066. { If %reg1 also appears in the second reference, then it will
  4067. not refer to the same memory block as the first reference }
  4068. not RegInRef(p_SourceReg, SourceRef) then
  4069. begin
  4070. { Check to see if the references match if %reg2 is changed to %reg1 }
  4071. if SourceRef.base = p_TargetReg then
  4072. SourceRef.base := p_SourceReg;
  4073. if SourceRef.index = p_TargetReg then
  4074. SourceRef.index := p_SourceReg;
  4075. { RefsEqual also checks to ensure both references are non-volatile }
  4076. if RefsEqual(taicpu(hp1).oper[0]^.ref^, SourceRef) then
  4077. begin
  4078. taicpu(hp2).loadreg(0, p_SourceReg);
  4079. TransferUsedRegs(TmpUsedRegs);
  4080. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  4081. { Make sure the register is allocated between these instructions
  4082. even though it doesn't change value, since it may cause
  4083. optimisations on a later pass to behave incorrectly. (Fixes #41155) }
  4084. AllocRegBetween(p_SourceReg, hp1, hp2, TmpUsedRegs);
  4085. DebugMsg(SPeepholeOptimization + 'Optimised register duplication and memory read (MovMovMov2MovMovMov)', p);
  4086. Result := True;
  4087. if taicpu(hp2).oper[1]^.reg = p_TargetReg then
  4088. begin
  4089. DebugMsg(SPeepholeOptimization + 'Mov2Nop 5a done', p);
  4090. RemoveCurrentP(p);
  4091. Exit;
  4092. end
  4093. else
  4094. begin
  4095. if not RegUsedAfterInstruction(p_TargetReg, hp2, TmpUsedRegs) then
  4096. begin
  4097. DebugMsg(SPeepholeOptimization + 'Mov2Nop 5b done', p);
  4098. RemoveCurrentP(p);
  4099. Exit;
  4100. end;
  4101. end;
  4102. { If we reach this point, p and hp1 weren't actually modified,
  4103. so we can do a bit more work on this pass }
  4104. end;
  4105. end;
  4106. end;
  4107. end;
  4108. end;
  4109. end;
  4110. GetNextInstruction_p:=GetNextInstruction(p, hp1);
  4111. { All the next optimisations require a next instruction }
  4112. if not GetNextInstruction_p or (hp1.typ <> ait_instruction) then
  4113. Exit;
  4114. { Change:
  4115. movl/q (ref), %reg
  4116. movd/q %reg, %xmm0
  4117. (dealloc %reg)
  4118. To:
  4119. movd/q (ref), %xmm0
  4120. }
  4121. if MatchOpType(taicpu(p),top_ref,top_reg) and
  4122. MatchInstruction(hp1,[A_MOVD,A_VMOVD{$ifdef x86_64},A_MOVQ,A_VMOVQ{$endif x86_64}],[]) and
  4123. MatchOperand(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^.reg) and
  4124. (taicpu(hp1).oper[1]^.typ=top_reg) and
  4125. (GetRegType(taicpu(hp1).oper[1]^.reg)=R_MMREGISTER) then
  4126. begin
  4127. TransferUsedRegs(TmpUsedRegs);
  4128. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  4129. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs) then
  4130. begin
  4131. taicpu(hp1).loadref(0,taicpu(p).oper[0]^.ref^);
  4132. { loadref increases the reference count, so decrement it again }
  4133. if Assigned(taicpu(p).oper[0]^.ref^.symbol) then
  4134. taicpu(p).oper[0]^.ref^.symbol.decrefs;
  4135. if Assigned(taicpu(p).oper[0]^.ref^.relsymbol) then
  4136. taicpu(p).oper[0]^.ref^.relsymbol.decrefs;
  4137. DebugMsg(SPeepholeOptimization+'Merged MOV and (V)MOVD/(V)MOVQ to eliminate intermediate register (MovMovD/Q2MovD/Q)',p);
  4138. RemoveCurrentP(p,hp1);
  4139. Result:=True;
  4140. Exit;
  4141. end;
  4142. end;
  4143. { Next instruction is also a MOV ? }
  4144. if MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) then
  4145. begin
  4146. if MatchOpType(taicpu(p), top_const, top_ref) and
  4147. MatchOpType(taicpu(hp1), top_const, top_ref) and
  4148. TryConstMerge(p, hp1) then
  4149. begin
  4150. Result := True;
  4151. { In case we have four byte writes in a row, check for 2 more
  4152. right now so we don't have to wait for another iteration of
  4153. pass 1
  4154. }
  4155. { If two byte-writes were merged, the opsize is now S_W, not S_B }
  4156. case taicpu(p).opsize of
  4157. S_W:
  4158. begin
  4159. if GetNextInstruction(p, hp1) and
  4160. MatchInstruction(hp1, A_MOV, [S_B]) and
  4161. MatchOpType(taicpu(hp1), top_const, top_ref) and
  4162. GetNextInstruction(hp1, hp2) and
  4163. MatchInstruction(hp2, A_MOV, [S_B]) and
  4164. MatchOpType(taicpu(hp2), top_const, top_ref) and
  4165. { Try to merge the two bytes }
  4166. TryConstMerge(hp1, hp2) then
  4167. { Now try to merge the two words (hp2 will get deleted) }
  4168. TryConstMerge(p, hp1);
  4169. end;
  4170. S_L:
  4171. begin
  4172. { Though this only really benefits x86_64 and not i386, it
  4173. gets a potential optimisation done faster and hence
  4174. reduces the number of times OptPass1MOV is entered }
  4175. if GetNextInstruction(p, hp1) and
  4176. MatchInstruction(hp1, A_MOV, [S_W]) and
  4177. MatchOpType(taicpu(hp1), top_const, top_ref) and
  4178. GetNextInstruction(hp1, hp2) and
  4179. MatchInstruction(hp2, A_MOV, [S_W]) and
  4180. MatchOpType(taicpu(hp2), top_const, top_ref) and
  4181. { Try to merge the two words }
  4182. TryConstMerge(hp1, hp2) then
  4183. { This will always fail on i386, so don't bother
  4184. calling it unless we're doing x86_64 }
  4185. {$ifdef x86_64}
  4186. { Now try to merge the two longwords (hp2 will get deleted) }
  4187. TryConstMerge(p, hp1)
  4188. {$endif x86_64}
  4189. ;
  4190. end;
  4191. else
  4192. ;
  4193. end;
  4194. Exit;
  4195. end;
  4196. if (taicpu(hp1).oper[0]^.typ = taicpu(p).oper[1]^.typ) and
  4197. (taicpu(hp1).oper[1]^.typ = taicpu(p).oper[0]^.typ) then
  4198. { mov reg1, mem1 or mov mem1, reg1
  4199. mov mem2, reg2 mov reg2, mem2}
  4200. begin
  4201. if OpsEqual(taicpu(hp1).oper[1]^,taicpu(p).oper[0]^) then
  4202. { mov reg1, mem1 or mov mem1, reg1
  4203. mov mem2, reg1 mov reg2, mem1}
  4204. begin
  4205. if OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  4206. { Removes the second statement from
  4207. mov reg1, mem1/reg2
  4208. mov mem1/reg2, reg1 }
  4209. begin
  4210. if taicpu(p).oper[0]^.typ=top_reg then
  4211. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,usedregs);
  4212. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 1',p);
  4213. RemoveInstruction(hp1);
  4214. Result:=true;
  4215. if (taicpu(p).oper[1]^.typ = top_reg) then
  4216. begin
  4217. TransferUsedRegs(TmpUsedRegs);
  4218. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, p, TmpUsedRegs) then
  4219. begin
  4220. { reg2 is no longer in use }
  4221. DebugMsg(SPeepholeOptimization + 'Mov2Nop 6 done',p);
  4222. RemoveCurrentP(p);
  4223. end;
  4224. end;
  4225. exit;
  4226. end
  4227. else
  4228. begin
  4229. TransferUsedRegs(TmpUsedRegs);
  4230. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  4231. if (taicpu(p).oper[1]^.typ = top_ref) and
  4232. { mov reg1, mem1
  4233. mov mem2, reg1 }
  4234. (taicpu(hp1).oper[0]^.ref^.refaddr = addr_no) and
  4235. GetNextInstruction(hp1, hp2) and
  4236. MatchInstruction(hp2,A_CMP,[taicpu(p).opsize]) and
  4237. OpsEqual(taicpu(p).oper[1]^,taicpu(hp2).oper[0]^) and
  4238. OpsEqual(taicpu(p).oper[0]^,taicpu(hp2).oper[1]^) and
  4239. not(RegUsedAfterInstruction(taicpu(p).oper[0]^.reg, hp2, TmpUsedRegs)) then
  4240. { change to
  4241. mov reg1, mem1 mov reg1, mem1
  4242. mov mem2, reg1 cmp reg1, mem2
  4243. cmp mem1, reg1
  4244. }
  4245. begin
  4246. RemoveInstruction(hp2);
  4247. taicpu(hp1).opcode := A_CMP;
  4248. taicpu(hp1).loadref(1,taicpu(hp1).oper[0]^.ref^);
  4249. taicpu(hp1).loadreg(0,taicpu(p).oper[0]^.reg);
  4250. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,UsedRegs);
  4251. DebugMsg(SPeepholeOptimization + 'MovMovCmp2MovCmp done',hp1);
  4252. end;
  4253. end;
  4254. end
  4255. else if (taicpu(p).oper[1]^.typ=top_ref) and
  4256. OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  4257. begin
  4258. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,UsedRegs);
  4259. taicpu(hp1).loadreg(0,taicpu(p).oper[0]^.reg);
  4260. DebugMsg(SPeepholeOptimization + 'MovMov2MovMov1 done',p);
  4261. end
  4262. else
  4263. begin
  4264. TransferUsedRegs(TmpUsedRegs);
  4265. if GetNextInstruction(hp1, hp2) and
  4266. MatchOpType(taicpu(p),top_ref,top_reg) and
  4267. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  4268. (taicpu(hp1).oper[1]^.typ = top_ref) and
  4269. MatchInstruction(hp2,A_MOV,[taicpu(p).opsize]) and
  4270. MatchOpType(taicpu(hp2),top_ref,top_reg) and
  4271. RefsEqual(taicpu(hp2).oper[0]^.ref^, taicpu(hp1).oper[1]^.ref^) then
  4272. if not RegInRef(taicpu(hp2).oper[1]^.reg,taicpu(hp2).oper[0]^.ref^) and
  4273. not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,tmpUsedRegs)) then
  4274. { mov mem1, %reg1
  4275. mov %reg1, mem2
  4276. mov mem2, reg2
  4277. to:
  4278. mov mem1, reg2
  4279. mov reg2, mem2}
  4280. begin
  4281. AllocRegBetween(taicpu(hp2).oper[1]^.reg,p,hp2,usedregs);
  4282. DebugMsg(SPeepholeOptimization + 'MovMovMov2MovMov 1 done',p);
  4283. taicpu(p).loadoper(1,taicpu(hp2).oper[1]^);
  4284. taicpu(hp1).loadoper(0,taicpu(hp2).oper[1]^);
  4285. RemoveInstruction(hp2);
  4286. Result := True;
  4287. end
  4288. {$ifdef i386}
  4289. { this is enabled for i386 only, as the rules to create the reg sets below
  4290. are too complicated for x86-64, so this makes this code too error prone
  4291. on x86-64
  4292. }
  4293. else if (taicpu(p).oper[1]^.reg <> taicpu(hp2).oper[1]^.reg) and
  4294. not(RegInRef(taicpu(p).oper[1]^.reg,taicpu(p).oper[0]^.ref^)) and
  4295. not(RegInRef(taicpu(hp2).oper[1]^.reg,taicpu(hp2).oper[0]^.ref^)) then
  4296. { mov mem1, reg1 mov mem1, reg1
  4297. mov reg1, mem2 mov reg1, mem2
  4298. mov mem2, reg2 mov mem2, reg1
  4299. to: to:
  4300. mov mem1, reg1 mov mem1, reg1
  4301. mov mem1, reg2 mov reg1, mem2
  4302. mov reg1, mem2
  4303. or (if mem1 depends on reg1
  4304. and/or if mem2 depends on reg2)
  4305. to:
  4306. mov mem1, reg1
  4307. mov reg1, mem2
  4308. mov reg1, reg2
  4309. }
  4310. begin
  4311. taicpu(hp1).loadRef(0,taicpu(p).oper[0]^.ref^);
  4312. taicpu(hp1).loadReg(1,taicpu(hp2).oper[1]^.reg);
  4313. taicpu(hp2).loadRef(1,taicpu(hp2).oper[0]^.ref^);
  4314. taicpu(hp2).loadReg(0,taicpu(p).oper[1]^.reg);
  4315. AllocRegBetween(taicpu(p).oper[1]^.reg,p,hp2,usedregs);
  4316. if (taicpu(p).oper[0]^.ref^.base <> NR_NO) and
  4317. (getsupreg(taicpu(p).oper[0]^.ref^.base) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX,RS_ESI,RS_EDI]) then
  4318. AllocRegBetween(taicpu(p).oper[0]^.ref^.base,p,hp2,usedregs);
  4319. if (taicpu(p).oper[0]^.ref^.index <> NR_NO) and
  4320. (getsupreg(taicpu(p).oper[0]^.ref^.index) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX,RS_ESI,RS_EDI]) then
  4321. AllocRegBetween(taicpu(p).oper[0]^.ref^.index,p,hp2,usedregs);
  4322. end
  4323. else if (taicpu(hp1).Oper[0]^.reg <> taicpu(hp2).Oper[1]^.reg) then
  4324. begin
  4325. taicpu(hp2).loadReg(0,taicpu(hp1).Oper[0]^.reg);
  4326. AllocRegBetween(taicpu(p).oper[1]^.reg,p,hp2,usedregs);
  4327. end
  4328. else
  4329. begin
  4330. RemoveInstruction(hp2);
  4331. end
  4332. {$endif i386}
  4333. ;
  4334. end;
  4335. end
  4336. { movl [mem1],reg1
  4337. movl [mem1],reg2
  4338. to
  4339. movl [mem1],reg1
  4340. movl reg1,reg2
  4341. }
  4342. else if not CheckMovMov2MovMov2(p, hp1) and
  4343. { movl const1,[mem1]
  4344. movl [mem1],reg1
  4345. to
  4346. movl const1,reg1
  4347. movl reg1,[mem1]
  4348. }
  4349. MatchOpType(Taicpu(p),top_const,top_ref) and
  4350. MatchOpType(Taicpu(hp1),top_ref,top_reg) and
  4351. (taicpu(p).opsize = taicpu(hp1).opsize) and
  4352. RefsEqual(taicpu(hp1).oper[0]^.ref^,taicpu(p).oper[1]^.ref^) and
  4353. not(RegInRef(taicpu(hp1).oper[1]^.reg,taicpu(hp1).oper[0]^.ref^)) then
  4354. begin
  4355. AllocRegBetween(taicpu(hp1).oper[1]^.reg,p,hp1,usedregs);
  4356. taicpu(hp1).loadReg(0,taicpu(hp1).oper[1]^.reg);
  4357. taicpu(hp1).loadRef(1,taicpu(p).oper[1]^.ref^);
  4358. taicpu(p).loadReg(1,taicpu(hp1).oper[0]^.reg);
  4359. taicpu(hp1).fileinfo := taicpu(p).fileinfo;
  4360. DebugMsg(SPeepholeOptimization + 'MovMov2MovMov 1',p);
  4361. Result:=true;
  4362. exit;
  4363. end;
  4364. { mov x,reg1; mov y,reg1 -> mov y,reg1 is handled by the Mov2Nop 5 optimisation }
  4365. end;
  4366. { search further than the next instruction for a mov (as long as it's not a jump) }
  4367. if not is_calljmpuncondret(taicpu(hp1).opcode) and
  4368. { check as much as possible before the expensive GetNextInstructionUsingRegCond call }
  4369. (taicpu(p).oper[1]^.typ = top_reg) and
  4370. (taicpu(p).oper[0]^.typ in [top_reg,top_const]) and
  4371. not RegModifiedByInstruction(taicpu(p).oper[1]^.reg, hp1) then
  4372. begin
  4373. { we work with hp2 here, so hp1 can be still used later on when
  4374. checking for GetNextInstruction_p }
  4375. hp3 := hp1;
  4376. { Initialise CrossJump (if it becomes True at any point, it will remain True) }
  4377. CrossJump := (taicpu(hp1).opcode = A_Jcc);
  4378. { Remember that p_TargetReg contains taicpu(p).oper[1]^.reg }
  4379. TransferUsedRegs(TmpUsedRegs);
  4380. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  4381. if NotFirstIteration then
  4382. JumpTracking := TLinkedList.Create
  4383. else
  4384. JumpTracking := nil;
  4385. while GetNextInstructionUsingRegCond(hp3,hp2,p_TargetReg,JumpTracking,CrossJump) and
  4386. { GetNextInstructionUsingRegCond only searches one instruction ahead unless -O3 is specified }
  4387. (hp2.typ=ait_instruction) do
  4388. begin
  4389. case taicpu(hp2).opcode of
  4390. A_POP:
  4391. if MatchOperand(taicpu(hp2).oper[0]^,p_TargetReg) then
  4392. begin
  4393. if not CrossJump and
  4394. not RegUsedBetween(p_TargetReg, p, hp2) then
  4395. begin
  4396. { We can remove the original MOV since the register
  4397. wasn't used between it and its popping from the stack }
  4398. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3c done',p);
  4399. RemoveCurrentp(p, hp1);
  4400. Result := True;
  4401. JumpTracking.Free;
  4402. Exit;
  4403. end;
  4404. { Can't go any further }
  4405. Break;
  4406. end;
  4407. A_MOV:
  4408. if MatchOperand(taicpu(hp2).oper[0]^,p_TargetReg) and
  4409. ((taicpu(p).oper[0]^.typ=top_const) or
  4410. ((taicpu(p).oper[0]^.typ=top_reg) and
  4411. not(RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp2))
  4412. )
  4413. ) then
  4414. begin
  4415. { we have
  4416. mov x, %treg
  4417. mov %treg, y
  4418. }
  4419. { We don't need to call UpdateUsedRegs for every instruction between
  4420. p and hp2 because the register we're concerned about will not
  4421. become deallocated (otherwise GetNextInstructionUsingReg would
  4422. have stopped at an earlier instruction). [Kit] }
  4423. TempRegUsed :=
  4424. CrossJump { Assume the register is in use if it crossed a conditional jump } or
  4425. RegReadByInstruction(p_TargetReg, hp3) or
  4426. RegUsedAfterInstruction(p_TargetReg, hp2, TmpUsedRegs);
  4427. case taicpu(p).oper[0]^.typ Of
  4428. top_reg:
  4429. begin
  4430. { change
  4431. mov %reg, %treg
  4432. mov %treg, y
  4433. to
  4434. mov %reg, y
  4435. }
  4436. p_SourceReg := taicpu(p).oper[0]^.reg; { Saves on a handful of pointer dereferences }
  4437. RegName1 := debug_regname(taicpu(hp2).oper[0]^.reg);
  4438. if MatchOperand(taicpu(hp2).oper[1]^, p_SourceReg) then
  4439. begin
  4440. { %reg = y - remove hp2 completely (doing it here instead of relying on
  4441. the "mov %reg,%reg" optimisation might cut down on a pass iteration) }
  4442. if TempRegUsed then
  4443. begin
  4444. DebugMsg(SPeepholeOptimization + debug_regname(p_SourceReg) + ' = ' + RegName1 + '; removed unnecessary instruction (MovMov2MovNop 6b}',hp2);
  4445. AllocRegBetween(p_SourceReg, p, hp2, UsedRegs);
  4446. { Set the start of the next GetNextInstructionUsingRegCond search
  4447. to start at the entry right before hp2 (which is about to be removed) }
  4448. hp3 := tai(hp2.Previous);
  4449. RemoveInstruction(hp2);
  4450. Include(OptsToCheck, aoc_ForceNewIteration);
  4451. { See if there's more we can optimise }
  4452. Continue;
  4453. end
  4454. else
  4455. begin
  4456. RemoveInstruction(hp2);
  4457. { We can remove the original MOV too }
  4458. DebugMsg(SPeepholeOptimization + 'MovMov2NopNop 6b done',p);
  4459. RemoveCurrentP(p, hp1);
  4460. Result:=true;
  4461. JumpTracking.Free;
  4462. Exit;
  4463. end;
  4464. end
  4465. else
  4466. begin
  4467. AllocRegBetween(p_SourceReg, p, hp2, UsedRegs);
  4468. taicpu(hp2).loadReg(0, p_SourceReg);
  4469. DebugMsg(SPeepholeOptimization + RegName1 + ' = ' + debug_regname(p_SourceReg) + '; changed to minimise pipeline stall (MovMov2Mov 6a}',hp2);
  4470. { Check to see if the register also appears in the reference }
  4471. if (taicpu(hp2).oper[1]^.typ = top_ref) then
  4472. ReplaceRegisterInRef(taicpu(hp2).oper[1]^.ref^, p_TargetReg, p_SourceReg);
  4473. { ReplaceRegisterInRef won't actually replace the register if it's a different size }
  4474. if not RegInOp(p_TargetReg, taicpu(hp2).oper[1]^) then
  4475. begin
  4476. { Don't remove the first instruction if the temporary register is in use }
  4477. if not TempRegUsed then
  4478. begin
  4479. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 6 done',p);
  4480. RemoveCurrentP(p, hp1);
  4481. Result:=true;
  4482. JumpTracking.Free;
  4483. Exit;
  4484. end;
  4485. { No need to set Result to True here. If there's another instruction later
  4486. on that can be optimised, it will be detected when the main Pass 1 loop
  4487. reaches what is now hp2 and passes it through OptPass1MOV. [Kit] }
  4488. hp3 := hp2;
  4489. Continue;
  4490. end;
  4491. end;
  4492. end;
  4493. top_const:
  4494. if not (cs_opt_size in current_settings.optimizerswitches) or (taicpu(hp2).opsize = S_B) then
  4495. begin
  4496. { change
  4497. mov const, %treg
  4498. mov %treg, y
  4499. to
  4500. mov const, y
  4501. }
  4502. if (taicpu(hp2).oper[1]^.typ=top_reg) or
  4503. ((taicpu(p).oper[0]^.val>=low(longint)) and (taicpu(p).oper[0]^.val<=high(longint))) then
  4504. begin
  4505. RegName1 := debug_regname(taicpu(hp2).oper[0]^.reg);
  4506. taicpu(hp2).loadOper(0,taicpu(p).oper[0]^);
  4507. if TempRegUsed then
  4508. begin
  4509. { Don't remove the first instruction if the temporary register is in use }
  4510. DebugMsg(SPeepholeOptimization + RegName1 + ' = ' + debug_tostr(taicpu(p).oper[0]^.val) + '; changed to minimise pipeline stall (MovMov2Mov 7a)',hp2);
  4511. { No need to set Result to True. If there's another instruction later on
  4512. that can be optimised, it will be detected when the main Pass 1 loop
  4513. reaches what is now hp2 and passes it through OptPass1MOV. [Kit] };
  4514. end
  4515. else
  4516. begin
  4517. DebugMsg(SPeepholeOptimization + 'MovMov2Mov 7 done',p);
  4518. RemoveCurrentP(p, hp1);
  4519. Result:=true;
  4520. Exit;
  4521. end;
  4522. end;
  4523. end;
  4524. else
  4525. Internalerror(2019103001);
  4526. end;
  4527. end
  4528. else if MatchOperand(taicpu(hp2).oper[1]^, p_TargetReg) then
  4529. begin
  4530. if not CrossJump and
  4531. not RegUsedBetween(p_TargetReg, p, hp2) and
  4532. not RegReadByInstruction(p_TargetReg, hp2) then
  4533. begin
  4534. { Register is not used before it is overwritten }
  4535. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3a done',p);
  4536. RemoveCurrentp(p, hp1);
  4537. Result := True;
  4538. Exit;
  4539. end;
  4540. if (taicpu(p).oper[0]^.typ = top_const) and
  4541. (taicpu(hp2).oper[0]^.typ = top_const) then
  4542. begin
  4543. if taicpu(p).oper[0]^.val = taicpu(hp2).oper[0]^.val then
  4544. begin
  4545. { Same value - register hasn't changed }
  4546. DebugMsg(SPeepholeOptimization + 'Mov2Nop 2 done', hp2);
  4547. RemoveInstruction(hp2);
  4548. Include(OptsToCheck, aoc_ForceNewIteration);
  4549. { See if there's more we can optimise }
  4550. Continue;
  4551. end;
  4552. end;
  4553. {$ifdef x86_64}
  4554. end
  4555. { Change:
  4556. movl %reg1l,%reg2l
  4557. ...
  4558. movq %reg2q,%reg3q (%reg1 <> %reg3)
  4559. To:
  4560. movl %reg1l,%reg2l
  4561. ...
  4562. movl %reg1l,%reg3l (Upper 32 bits of %reg3q will be zero)
  4563. If %reg1 = %reg3, convert to:
  4564. movl %reg1l,%reg2l
  4565. ...
  4566. andl %reg1l,%reg1l
  4567. }
  4568. else if (taicpu(p).opsize = S_L) and MatchInstruction(hp2,A_MOV,[S_Q]) and
  4569. (taicpu(p).oper[0]^.typ = top_reg) and
  4570. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  4571. SuperRegistersEqual(p_TargetReg, taicpu(hp2).oper[0]^.reg) and
  4572. not RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp2) then
  4573. begin
  4574. TempRegUsed :=
  4575. CrossJump { Assume the register is in use if it crossed a conditional jump } or
  4576. RegReadByInstruction(p_TargetReg, hp3) or
  4577. RegUsedAfterInstruction(p_TargetReg, hp2, TmpUsedRegs);
  4578. taicpu(hp2).opsize := S_L;
  4579. taicpu(hp2).loadreg(0, taicpu(p).oper[0]^.reg);
  4580. setsubreg(taicpu(hp2).oper[1]^.reg, R_SUBD);
  4581. AllocRegBetween(taicpu(p).oper[0]^.reg, p, hp2, UsedRegs);
  4582. if (taicpu(p).oper[0]^.reg = taicpu(hp2).oper[1]^.reg) then
  4583. begin
  4584. { %reg1 = %reg3 }
  4585. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlAndl 2)', hp2);
  4586. taicpu(hp2).opcode := A_AND;
  4587. end
  4588. else
  4589. begin
  4590. { %reg1 <> %reg3 }
  4591. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlMovl 2)', hp2);
  4592. end;
  4593. if not TempRegUsed then
  4594. begin
  4595. DebugMsg(SPeepholeOptimization + 'Mov2Nop 8a done', p);
  4596. RemoveCurrentP(p, hp1);
  4597. Result := True;
  4598. Exit;
  4599. end
  4600. else
  4601. begin
  4602. { Initial instruction wasn't actually changed }
  4603. Include(OptsToCheck, aoc_ForceNewIteration);
  4604. { if %reg1 = %reg3, don't do the long-distance lookahead that
  4605. appears below since %reg1 has technically changed }
  4606. if taicpu(hp2).opcode = A_AND then
  4607. Break;
  4608. end;
  4609. {$endif x86_64}
  4610. end
  4611. else if (taicpu(hp2).oper[0]^.typ = top_ref) and
  4612. GetNextInstruction(hp2, hp4) and
  4613. (hp4.typ = ait_instruction) and (taicpu(hp4).opcode = A_MOV) then
  4614. { Optimise the following first:
  4615. movl [mem1],reg1
  4616. movl [mem1],reg2
  4617. to
  4618. movl [mem1],reg1
  4619. movl reg1,reg2
  4620. If [mem1] contains the target register and reg1 is the
  4621. the source register, this optimisation will get missed
  4622. and produce less efficient code later on.
  4623. }
  4624. if CheckMovMov2MovMov2(hp2, hp4) then
  4625. { Initial instruction wasn't actually changed }
  4626. Include(OptsToCheck, aoc_ForceNewIteration);
  4627. A_MOVZX, A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  4628. if MatchOpType(taicpu(hp2), top_reg, top_reg) and
  4629. MatchOperand(taicpu(hp2).oper[0]^, p_TargetReg) and
  4630. SuperRegistersEqual(taicpu(hp2).oper[1]^.reg, p_TargetReg) then
  4631. begin
  4632. {
  4633. Change from:
  4634. mov ###, %reg
  4635. ...
  4636. movs/z %reg,%reg (Same register, just different sizes)
  4637. To:
  4638. movs/z ###, %reg (Longer version)
  4639. ...
  4640. (remove)
  4641. }
  4642. DebugMsg(SPeepholeOptimization + 'MovMovs/z2Mov/s/z done', p);
  4643. taicpu(p).oper[1]^.reg := taicpu(hp2).oper[1]^.reg;
  4644. { Keep the first instruction as mov if ### is a constant }
  4645. if taicpu(p).oper[0]^.typ = top_const then
  4646. taicpu(p).opsize := reg2opsize(taicpu(hp2).oper[1]^.reg)
  4647. else
  4648. begin
  4649. taicpu(p).opcode := taicpu(hp2).opcode;
  4650. taicpu(p).opsize := taicpu(hp2).opsize;
  4651. end;
  4652. DebugMsg(SPeepholeOptimization + 'Removed movs/z instruction and extended earlier write (MovMovs/z2Mov/s/z)', hp2);
  4653. AllocRegBetween(taicpu(hp2).oper[1]^.reg, p, hp2, UsedRegs);
  4654. RemoveInstruction(hp2);
  4655. Result := True;
  4656. JumpTracking.Free;
  4657. Exit;
  4658. end;
  4659. else
  4660. { Move down to the if-block below };
  4661. end;
  4662. { Also catches MOV/S/Z instructions that aren't modified }
  4663. if taicpu(p).oper[0]^.typ = top_reg then
  4664. begin
  4665. p_SourceReg := taicpu(p).oper[0]^.reg;
  4666. if
  4667. not RegModifiedByInstruction(p_SourceReg, hp3) and
  4668. not RegModifiedBetween(p_SourceReg, hp3, hp2) and
  4669. DeepMOVOpt(taicpu(p), taicpu(hp2)) then
  4670. begin
  4671. Result := True;
  4672. { Just in case something didn't get modified (e.g. an
  4673. implicit register). Also, if it does read from this
  4674. register, then there's no longer an advantage to
  4675. changing the register on subsequent instructions.}
  4676. if not RegReadByInstruction(p_TargetReg, hp2) then
  4677. begin
  4678. { If a conditional jump was crossed, do not delete
  4679. the original MOV no matter what }
  4680. if not CrossJump and
  4681. { RegEndOfLife returns True if the register is
  4682. deallocated before the next instruction or has
  4683. been loaded with a new value }
  4684. RegEndOfLife(p_TargetReg, taicpu(hp2)) then
  4685. begin
  4686. { We can remove the original MOV }
  4687. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3b done',p);
  4688. RemoveCurrentp(p, hp1);
  4689. JumpTracking.Free;
  4690. Result := True;
  4691. Exit;
  4692. end;
  4693. if not RegModifiedByInstruction(p_TargetReg, hp2) then
  4694. begin
  4695. { See if there's more we can optimise }
  4696. hp3 := hp2;
  4697. Continue;
  4698. end;
  4699. end;
  4700. end;
  4701. end;
  4702. { Break out of the while loop under normal circumstances }
  4703. Break;
  4704. end;
  4705. JumpTracking.Free;
  4706. end;
  4707. if (aoc_MovAnd2Mov_3 in OptsToCheck) and
  4708. (taicpu(p).oper[1]^.typ = top_reg) and
  4709. (taicpu(p).opsize = S_L) and
  4710. GetNextInstructionUsingRegTrackingUse(p,hp2,taicpu(p).oper[1]^.reg) and
  4711. (hp2.typ = ait_instruction) and
  4712. (taicpu(hp2).opcode = A_AND) and
  4713. (MatchOpType(taicpu(hp2),top_const,top_reg) or
  4714. (MatchOpType(taicpu(hp2),top_reg,top_reg) and
  4715. MatchOperand(taicpu(hp2).oper[0]^,taicpu(hp2).oper[1]^))
  4716. ) then
  4717. begin
  4718. if SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp2).oper[1]^.reg) then
  4719. begin
  4720. if ((taicpu(hp2).oper[0]^.typ=top_const) and (taicpu(hp2).oper[0]^.val = $ffffffff)) or
  4721. ((taicpu(hp2).oper[0]^.typ=top_reg) and (taicpu(hp2).opsize=S_L)) then
  4722. begin
  4723. { Optimize out:
  4724. mov x, %reg
  4725. and ffffffffh, %reg
  4726. }
  4727. DebugMsg(SPeepholeOptimization + 'MovAnd2Mov 3 done',p);
  4728. RemoveInstruction(hp2);
  4729. Result:=true;
  4730. exit;
  4731. end;
  4732. end;
  4733. end;
  4734. { leave out the mov from "mov reg, x(%frame_pointer); leave/ret" (with
  4735. x >= RetOffset) as it doesn't do anything (it writes either to a
  4736. parameter or to the temporary storage room for the function
  4737. result)
  4738. }
  4739. if IsExitCode(hp1) and
  4740. (taicpu(p).oper[1]^.typ = top_ref) and
  4741. (taicpu(p).oper[1]^.ref^.index = NR_NO) and
  4742. (
  4743. (
  4744. (taicpu(p).oper[1]^.ref^.base = current_procinfo.FramePointer) and
  4745. not (
  4746. assigned(current_procinfo.procdef.funcretsym) and
  4747. (taicpu(p).oper[1]^.ref^.offset <= tabstractnormalvarsym(current_procinfo.procdef.funcretsym).localloc.reference.offset)
  4748. )
  4749. ) or
  4750. { Also discard writes to the stack that are below the base pointer,
  4751. as this is temporary storage rather than a function result on the
  4752. stack, say. }
  4753. (
  4754. (taicpu(p).oper[1]^.ref^.base = NR_STACK_POINTER_REG) and
  4755. (taicpu(p).oper[1]^.ref^.offset < current_procinfo.final_localsize)
  4756. )
  4757. ) then
  4758. begin
  4759. RemoveCurrentp(p, hp1);
  4760. DebugMsg(SPeepholeOptimization + 'removed deadstore before leave/ret',p);
  4761. RemoveLastDeallocForFuncRes(p);
  4762. Result:=true;
  4763. exit;
  4764. end;
  4765. if MatchInstruction(hp1,A_CMP,A_TEST,[taicpu(p).opsize]) then
  4766. begin
  4767. if MatchOpType(taicpu(p),top_reg,top_ref) and
  4768. (taicpu(hp1).oper[1]^.typ = top_ref) and
  4769. RefsEqual(taicpu(p).oper[1]^.ref^, taicpu(hp1).oper[1]^.ref^) then
  4770. begin
  4771. { change
  4772. mov reg1, mem1
  4773. test/cmp x, mem1
  4774. to
  4775. mov reg1, mem1
  4776. test/cmp x, reg1
  4777. }
  4778. taicpu(hp1).loadreg(1,taicpu(p).oper[0]^.reg);
  4779. DebugMsg(SPeepholeOptimization + 'MovTestCmp2MovTestCmp 1',hp1);
  4780. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,usedregs);
  4781. Result := True;
  4782. Exit;
  4783. end;
  4784. if DoMovCmpMemOpt(p, hp1) then
  4785. begin
  4786. Result := True;
  4787. Exit;
  4788. end;
  4789. end;
  4790. if (taicpu(p).oper[1]^.typ = top_reg) and
  4791. (hp1.typ = ait_instruction) and
  4792. GetNextInstruction(hp1, hp2) and
  4793. MatchInstruction(hp2,A_MOV,[]) and
  4794. (SuperRegistersEqual(taicpu(hp2).oper[0]^.reg,taicpu(p).oper[1]^.reg)) and
  4795. (topsize2memsize[taicpu(hp1).opsize]>=topsize2memsize[taicpu(hp2).opsize]) and
  4796. (
  4797. IsFoldableArithOp(taicpu(hp1), taicpu(p).oper[1]^.reg)
  4798. {$ifdef x86_64}
  4799. or
  4800. (
  4801. (taicpu(p).opsize=S_L) and (taicpu(hp1).opsize=S_Q) and (taicpu(hp2).opsize=S_L) and
  4802. IsFoldableArithOp(taicpu(hp1), newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[1]^.reg),R_SUBQ))
  4803. )
  4804. {$endif x86_64}
  4805. ) then
  4806. begin
  4807. if OpsEqual(taicpu(hp2).oper[1]^, taicpu(p).oper[0]^) and
  4808. (taicpu(hp2).oper[0]^.typ=top_reg) then
  4809. { change movsX/movzX reg/ref, reg2
  4810. add/sub/or/... reg3/$const, reg2
  4811. mov reg2 reg/ref
  4812. dealloc reg2
  4813. to
  4814. add/sub/or/... reg3/$const, reg/ref }
  4815. begin
  4816. TransferUsedRegs(TmpUsedRegs);
  4817. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  4818. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  4819. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  4820. begin
  4821. { by example:
  4822. movswl %si,%eax movswl %si,%eax p
  4823. decl %eax addl %edx,%eax hp1
  4824. movw %ax,%si movw %ax,%si hp2
  4825. ->
  4826. movswl %si,%eax movswl %si,%eax p
  4827. decw %eax addw %edx,%eax hp1
  4828. movw %ax,%si movw %ax,%si hp2
  4829. }
  4830. DebugMsg(SPeepholeOptimization + 'MovOpMov2Op ('+
  4831. debug_op2str(taicpu(p).opcode)+debug_opsize2str(taicpu(p).opsize)+' '+
  4832. debug_op2str(taicpu(hp1).opcode)+debug_opsize2str(taicpu(hp1).opsize)+' '+
  4833. debug_op2str(taicpu(hp2).opcode)+debug_opsize2str(taicpu(hp2).opsize)+')',p);
  4834. taicpu(hp1).changeopsize(taicpu(hp2).opsize);
  4835. {
  4836. ->
  4837. movswl %si,%eax movswl %si,%eax p
  4838. decw %si addw %dx,%si hp1
  4839. movw %ax,%si movw %ax,%si hp2
  4840. }
  4841. case taicpu(hp1).ops of
  4842. 1:
  4843. begin
  4844. taicpu(hp1).loadoper(0, taicpu(hp2).oper[1]^);
  4845. if taicpu(hp1).oper[0]^.typ=top_reg then
  4846. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4847. end;
  4848. 2:
  4849. begin
  4850. taicpu(hp1).loadoper(1, taicpu(hp2).oper[1]^);
  4851. if (taicpu(hp1).oper[0]^.typ=top_reg) and
  4852. (taicpu(hp1).opcode<>A_SHL) and
  4853. (taicpu(hp1).opcode<>A_SHR) and
  4854. (taicpu(hp1).opcode<>A_SAR) then
  4855. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4856. end;
  4857. else
  4858. internalerror(2008042701);
  4859. end;
  4860. {
  4861. ->
  4862. decw %si addw %dx,%si p
  4863. }
  4864. RemoveInstruction(hp2);
  4865. RemoveCurrentP(p, hp1);
  4866. Result:=True;
  4867. Exit;
  4868. end;
  4869. end;
  4870. if MatchOpType(taicpu(hp2),top_reg,top_reg) and
  4871. not(SuperRegistersEqual(taicpu(hp1).oper[0]^.reg,taicpu(hp2).oper[1]^.reg)) and
  4872. ((topsize2memsize[taicpu(hp1).opsize]<= topsize2memsize[taicpu(hp2).opsize]) or
  4873. { opsize matters for these opcodes, we could probably work around this, but it is not worth the effort }
  4874. ((taicpu(hp1).opcode<>A_SHL) and (taicpu(hp1).opcode<>A_SHR) and (taicpu(hp1).opcode<>A_SAR))
  4875. ) and
  4876. { if ref contains a symbol, we cannot change its size to a smaller size }
  4877. ((taicpu(p).oper[0]^.typ<>top_ref) or (taicpu(p).oper[0]^.ref^.symbol=nil) or
  4878. (topsize2memsize[taicpu(p).opsize]<=topsize2memsize[taicpu(hp2).opsize])
  4879. )
  4880. {$ifdef i386}
  4881. { byte registers of esi, edi, ebp, esp are not available on i386 }
  4882. and ((taicpu(hp2).opsize<>S_B) or not(getsupreg(taicpu(hp1).oper[0]^.reg) in [RS_ESI,RS_EDI,RS_EBP,RS_ESP]))
  4883. and ((taicpu(hp2).opsize<>S_B) or not(getsupreg(taicpu(p).oper[0]^.reg) in [RS_ESI,RS_EDI,RS_EBP,RS_ESP]))
  4884. {$endif i386}
  4885. then
  4886. { change movsX/movzX reg/ref, reg2
  4887. add/sub/or/... regX/$const, reg2
  4888. mov reg2, reg3
  4889. dealloc reg2
  4890. to
  4891. movsX/movzX reg/ref, reg3
  4892. add/sub/or/... reg3/$const, reg3
  4893. }
  4894. begin
  4895. TransferUsedRegs(TmpUsedRegs);
  4896. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  4897. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  4898. If not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs)) then
  4899. begin
  4900. { by example:
  4901. movswl %si,%eax movswl %si,%eax p
  4902. decl %eax addl %edx,%eax hp1
  4903. movw %ax,%si movw %ax,%si hp2
  4904. ->
  4905. movswl %si,%eax movswl %si,%eax p
  4906. decw %eax addw %edx,%eax hp1
  4907. movw %ax,%si movw %ax,%si hp2
  4908. }
  4909. DebugMsg(SPeepholeOptimization + 'MovOpMov2MovOp ('+
  4910. debug_op2str(taicpu(p).opcode)+debug_opsize2str(taicpu(p).opsize)+' '+
  4911. debug_op2str(taicpu(hp1).opcode)+debug_opsize2str(taicpu(hp1).opsize)+' '+
  4912. debug_op2str(taicpu(hp2).opcode)+debug_opsize2str(taicpu(hp2).opsize)+')',p);
  4913. { limit size of constants as well to avoid assembler errors, but
  4914. check opsize to avoid overflow when left shifting the 1 }
  4915. if (taicpu(p).oper[0]^.typ=top_const) and (topsize2memsize[taicpu(hp2).opsize]<=63) then
  4916. taicpu(p).oper[0]^.val:=taicpu(p).oper[0]^.val and ((qword(1) shl topsize2memsize[taicpu(hp2).opsize])-1);
  4917. {$ifdef x86_64}
  4918. { Be careful of, for example:
  4919. movl %reg1,%reg2
  4920. addl %reg3,%reg2
  4921. movq %reg2,%reg4
  4922. This will cause problems if the upper 32-bits of %reg3 or %reg4 are non-zero
  4923. }
  4924. if (taicpu(hp1).opsize = S_L) and (taicpu(hp2).opsize = S_Q) then
  4925. begin
  4926. taicpu(hp2).changeopsize(S_L);
  4927. setsubreg(taicpu(hp2).oper[0]^.reg, R_SUBD);
  4928. setsubreg(taicpu(hp2).oper[1]^.reg, R_SUBD);
  4929. end;
  4930. {$endif x86_64}
  4931. taicpu(hp1).changeopsize(taicpu(hp2).opsize);
  4932. taicpu(p).changeopsize(taicpu(hp2).opsize);
  4933. if taicpu(p).oper[0]^.typ=top_reg then
  4934. setsubreg(taicpu(p).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4935. taicpu(p).loadoper(1, taicpu(hp2).oper[1]^);
  4936. AllocRegBetween(taicpu(p).oper[1]^.reg,p,hp1,usedregs);
  4937. {
  4938. ->
  4939. movswl %si,%eax movswl %si,%eax p
  4940. decw %si addw %dx,%si hp1
  4941. movw %ax,%si movw %ax,%si hp2
  4942. }
  4943. case taicpu(hp1).ops of
  4944. 1:
  4945. begin
  4946. taicpu(hp1).loadoper(0, taicpu(hp2).oper[1]^);
  4947. if taicpu(hp1).oper[0]^.typ=top_reg then
  4948. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4949. end;
  4950. 2:
  4951. begin
  4952. taicpu(hp1).loadoper(1, taicpu(hp2).oper[1]^);
  4953. if (taicpu(hp1).oper[0]^.typ=top_reg) and
  4954. (taicpu(hp1).opcode<>A_SHL) and
  4955. (taicpu(hp1).opcode<>A_SHR) and
  4956. (taicpu(hp1).opcode<>A_SAR) then
  4957. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  4958. end;
  4959. else
  4960. internalerror(2018111801);
  4961. end;
  4962. {
  4963. ->
  4964. decw %si addw %dx,%si p
  4965. }
  4966. RemoveInstruction(hp2);
  4967. end;
  4968. end;
  4969. end;
  4970. if MatchInstruction(hp1,A_BTS,A_BTR,[Taicpu(p).opsize]) and
  4971. GetNextInstruction(hp1, hp2) and
  4972. MatchInstruction(hp2,A_OR,[Taicpu(p).opsize]) and
  4973. MatchOperand(Taicpu(p).oper[0]^,0) and
  4974. (Taicpu(p).oper[1]^.typ = top_reg) and
  4975. MatchOperand(Taicpu(p).oper[1]^,Taicpu(hp1).oper[1]^) and
  4976. MatchOperand(Taicpu(p).oper[1]^,Taicpu(hp2).oper[1]^) then
  4977. { mov reg1,0
  4978. bts reg1,operand1 --> mov reg1,operand2
  4979. or reg1,operand2 bts reg1,operand1}
  4980. begin
  4981. Taicpu(hp2).opcode:=A_MOV;
  4982. DebugMsg(SPeepholeOptimization + 'MovBtsOr2MovBts done',hp1);
  4983. asml.remove(hp1);
  4984. insertllitem(hp2,hp2.next,hp1);
  4985. RemoveCurrentp(p, hp1);
  4986. Result:=true;
  4987. exit;
  4988. end;
  4989. if MatchInstruction(hp1,A_SUB,[Taicpu(p).opsize]) and
  4990. MatchOperand(Taicpu(p).oper[1]^,Taicpu(hp1).oper[1]^) and
  4991. GetNextInstruction(hp1, hp2) and
  4992. MatchInstruction(hp2,A_CMP,[Taicpu(p).opsize]) and
  4993. MatchOperand(Taicpu(p).oper[0]^,Taicpu(hp2).oper[1]^) and
  4994. MatchOperand(Taicpu(hp1).oper[0]^,Taicpu(hp2).oper[0]^) then
  4995. { change
  4996. mov reg1,reg2
  4997. sub reg3,reg2
  4998. cmp reg3,reg1
  4999. into
  5000. mov reg1,reg2
  5001. sub reg3,reg2
  5002. }
  5003. begin
  5004. DebugMsg(SPeepholeOptimization + 'MovSubCmp2MovSub done',p);
  5005. RemoveInstruction(hp2);
  5006. Result:=true;
  5007. exit;
  5008. end;
  5009. if (taicpu(p).oper[0]^.typ = top_ref) and { Second operand will be a register }
  5010. MatchInstruction(hp1, A_SHR, A_SAR, [taicpu(p).opsize]) and
  5011. MatchOpType(taicpu(hp1), top_const, top_reg) and
  5012. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  5013. begin
  5014. RegName1 := debug_regname(taicpu(hp1).oper[1]^.reg);
  5015. {$ifdef x86_64}
  5016. { Convert:
  5017. movq x(ref),%reg64
  5018. shrq y,%reg64
  5019. To:
  5020. movl x+4(ref),%reg32
  5021. shrl y-32,%reg32 (Remove if y = 32)
  5022. }
  5023. if (taicpu(p).opsize = S_Q) and
  5024. (taicpu(hp1).opcode = A_SHR) and
  5025. (taicpu(hp1).oper[0]^.val >= 32) then
  5026. begin
  5027. PreMessage := 'movq ' + debug_operstr(taicpu(p).oper[0]^) + ',' + RegName1 + '; ' +
  5028. 'shrq $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + RegName1 + ' -> movl ';
  5029. { Convert to 32-bit }
  5030. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  5031. taicpu(p).opsize := S_L;
  5032. Inc(taicpu(p).oper[0]^.ref^.offset, 4);
  5033. PreMessage := PreMessage + debug_operstr(taicpu(p).oper[0]^) + ',' + debug_regname(taicpu(p).oper[1]^.reg);
  5034. if (taicpu(hp1).oper[0]^.val = 32) then
  5035. begin
  5036. DebugMsg(SPeepholeOptimization + PreMessage + ' (MovShr2Mov)', p);
  5037. RemoveInstruction(hp1);
  5038. end
  5039. else
  5040. begin
  5041. { This will potentially open up more arithmetic operations since
  5042. the peephole optimizer now has a big hint that only the lower
  5043. 32 bits are currently in use (and opcodes are smaller in size) }
  5044. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  5045. taicpu(hp1).opsize := S_L;
  5046. Dec(taicpu(hp1).oper[0]^.val, 32);
  5047. DebugMsg(SPeepholeOptimization + PreMessage +
  5048. '; shrl $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' (MovShr2MovShr)', p);
  5049. end;
  5050. Result := True;
  5051. Exit;
  5052. end;
  5053. {$endif x86_64}
  5054. { Convert:
  5055. movl x(ref),%reg
  5056. shrl $24,%reg
  5057. To:
  5058. movzbl x+3(ref),%reg
  5059. Do similar things for movl; shrl $16 -> movzwl and movw; shrw $8 -> movzbw
  5060. Also accept sar instead of shr, but convert to movsx instead of movzx
  5061. }
  5062. if taicpu(hp1).opcode = A_SHR then
  5063. MovUnaligned := A_MOVZX
  5064. else
  5065. MovUnaligned := A_MOVSX;
  5066. NewSize := S_NO;
  5067. NewOffset := 0;
  5068. case taicpu(p).opsize of
  5069. S_B:
  5070. { No valid combinations };
  5071. S_W:
  5072. if (taicpu(hp1).oper[0]^.val = 8) then
  5073. begin
  5074. NewSize := S_BW;
  5075. NewOffset := 1;
  5076. end;
  5077. S_L:
  5078. case taicpu(hp1).oper[0]^.val of
  5079. 16:
  5080. begin
  5081. NewSize := S_WL;
  5082. NewOffset := 2;
  5083. end;
  5084. 24:
  5085. begin
  5086. NewSize := S_BL;
  5087. NewOffset := 3;
  5088. end;
  5089. else
  5090. ;
  5091. end;
  5092. {$ifdef x86_64}
  5093. S_Q:
  5094. case taicpu(hp1).oper[0]^.val of
  5095. 32:
  5096. begin
  5097. if taicpu(hp1).opcode = A_SAR then
  5098. begin
  5099. { 32-bit to 64-bit is a distinct instruction }
  5100. MovUnaligned := A_MOVSXD;
  5101. NewSize := S_LQ;
  5102. NewOffset := 4;
  5103. end
  5104. else
  5105. { Should have been handled by MovShr2Mov above }
  5106. InternalError(2022081811);
  5107. end;
  5108. 48:
  5109. begin
  5110. NewSize := S_WQ;
  5111. NewOffset := 6;
  5112. end;
  5113. 56:
  5114. begin
  5115. NewSize := S_BQ;
  5116. NewOffset := 7;
  5117. end;
  5118. else
  5119. ;
  5120. end;
  5121. {$endif x86_64}
  5122. else
  5123. InternalError(2022081810);
  5124. end;
  5125. if (NewSize <> S_NO) and
  5126. (taicpu(p).oper[0]^.ref^.offset <= $7FFFFFFF - NewOffset) then
  5127. begin
  5128. PreMessage := 'mov' + debug_opsize2str(taicpu(p).opsize) + ' ' + debug_operstr(taicpu(p).oper[0]^) + ',' + RegName1 + '; ' +
  5129. 'shr' + debug_opsize2str(taicpu(p).opsize) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + RegName1 + ' -> ' +
  5130. debug_op2str(MovUnaligned);
  5131. {$ifdef x86_64}
  5132. if MovUnaligned <> A_MOVSXD then
  5133. { Don't add size suffix for MOVSXD }
  5134. {$endif x86_64}
  5135. PreMessage := PreMessage + debug_opsize2str(NewSize);
  5136. Inc(taicpu(p).oper[0]^.ref^.offset, NewOffset);
  5137. taicpu(p).opcode := MovUnaligned;
  5138. taicpu(p).opsize := NewSize;
  5139. DebugMsg(SPeepholeOptimization + PreMessage + ' ' +
  5140. debug_operstr(taicpu(p).oper[0]^) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' (MovShr/Sar2Movx)', p);
  5141. RemoveInstruction(hp1);
  5142. Result := True;
  5143. Exit;
  5144. end;
  5145. end;
  5146. { Backward optimisation shared with OptPass2MOV }
  5147. if FuncMov2Func(p, hp1) then
  5148. begin
  5149. Result := True;
  5150. Exit;
  5151. end;
  5152. end;
  5153. function TX86AsmOptimizer.OptPass1MOVD(var p : tai) : boolean;
  5154. { This function also handles the 64-bit version, MOVQ }
  5155. var
  5156. hp1: tai;
  5157. begin
  5158. Result:=false;
  5159. { Change:
  5160. movd/q %xmm0, %reg
  5161. ...
  5162. movl/q %reg, (ref)
  5163. (dealloc %reg)
  5164. To:
  5165. movd/q %xmm0, (ref)
  5166. }
  5167. if MatchOpType(taicpu(p),top_reg,top_reg) and
  5168. (GetRegType(taicpu(p).oper[0]^.reg)=R_MMREGISTER) and
  5169. (GetRegType(taicpu(p).oper[1]^.reg)=R_INTREGISTER) and
  5170. GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg) and
  5171. MatchInstruction(hp1, A_MOV, []) and
  5172. MatchOperand(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^.reg) and
  5173. (taicpu(hp1).oper[1]^.typ=top_ref) and
  5174. not RegInRef(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^.ref^) then
  5175. begin
  5176. TransferUsedRegs(TmpUsedRegs);
  5177. UpdateUsedRegsBetween(TmpUsedRegs,p,hp1);
  5178. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs) then
  5179. begin
  5180. if (
  5181. { Instructions are always adjacent under -O2 and under }
  5182. not(cs_opt_level3 in current_settings.optimizerswitches) or
  5183. (
  5184. (
  5185. (taicpu(hp1).oper[1]^.ref^.base=NR_NO) or
  5186. not RegModifiedBetween(taicpu(hp1).oper[1]^.ref^.base,p,hp1)
  5187. ) and
  5188. (
  5189. (taicpu(hp1).oper[1]^.ref^.index=NR_NO) or
  5190. not RegModifiedBetween(taicpu(hp1).oper[1]^.ref^.index,p,hp1)
  5191. )
  5192. )
  5193. ) then
  5194. begin
  5195. DebugMsg(SPeepholeOptimization+'Merged (V)MOVD/(V)MOVQ and MOV to eliminate intermediate register (MovD/QMov2MovD/Q 1a)',p);
  5196. taicpu(p).loadref(1,taicpu(hp1).oper[1]^.ref^);
  5197. { loadref increases the reference count, so decrement it again }
  5198. if Assigned(taicpu(hp1).oper[1]^.ref^.symbol) then
  5199. taicpu(hp1).oper[1]^.ref^.symbol.decrefs;
  5200. if Assigned(taicpu(hp1).oper[1]^.ref^.relsymbol) then
  5201. taicpu(hp1).oper[1]^.ref^.relsymbol.decrefs;
  5202. RemoveInstruction(hp1);
  5203. Include(OptsToCheck, aoc_ForceNewIteration);
  5204. end
  5205. else if not RegModifiedBetween(taicpu(p).oper[0]^.reg,p,hp1) then
  5206. begin
  5207. { Still possible to optimise if hp1 is converted instead }
  5208. DebugMsg(SPeepholeOptimization+'Merged (V)MOVD/(V)MOVQ and MOV to eliminate intermediate register (MovD/QMov2MovD/Q 1b)',hp1);
  5209. { Decrement the reference prior to replacing it }
  5210. if Assigned(taicpu(hp1).oper[1]^.ref^.symbol) then
  5211. taicpu(hp1).oper[1]^.ref^.symbol.decrefs;
  5212. if Assigned(taicpu(hp1).oper[1]^.ref^.relsymbol) then
  5213. taicpu(hp1).oper[1]^.ref^.relsymbol.decrefs;
  5214. taicpu(hp1).opcode:=taicpu(p).opcode;
  5215. taicpu(hp1).opsize:=taicpu(p).opsize;
  5216. taicpu(hp1).loadreg(0,taicpu(p).oper[0]^.reg);
  5217. TransferUsedRegs(TmpUsedRegs);
  5218. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,TmpUsedRegs);
  5219. RemoveCurrentP(p);
  5220. Result:=True;
  5221. Exit;
  5222. end;
  5223. end;
  5224. end;
  5225. end;
  5226. function TX86AsmOptimizer.OptPass1MOVXX(var p : tai) : boolean;
  5227. var
  5228. hp1 : tai;
  5229. begin
  5230. Result:=false;
  5231. if taicpu(p).ops <> 2 then
  5232. exit;
  5233. if (MatchOpType(taicpu(p),top_reg,top_reg) and GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg)) or
  5234. GetNextInstruction(p,hp1) then
  5235. begin
  5236. if MatchInstruction(hp1,taicpu(p).opcode,[taicpu(p).opsize]) and
  5237. (taicpu(hp1).ops = 2) then
  5238. begin
  5239. if (taicpu(hp1).oper[0]^.typ = taicpu(p).oper[1]^.typ) and
  5240. (taicpu(hp1).oper[1]^.typ = taicpu(p).oper[0]^.typ) then
  5241. { movXX reg1, mem1 or movXX mem1, reg1
  5242. movXX mem2, reg2 movXX reg2, mem2}
  5243. begin
  5244. if OpsEqual(taicpu(hp1).oper[1]^,taicpu(p).oper[0]^) then
  5245. { movXX reg1, mem1 or movXX mem1, reg1
  5246. movXX mem2, reg1 movXX reg2, mem1}
  5247. begin
  5248. if OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  5249. begin
  5250. { Removes the second statement from
  5251. movXX reg1, mem1/reg2
  5252. movXX mem1/reg2, reg1
  5253. }
  5254. if taicpu(p).oper[0]^.typ=top_reg then
  5255. AllocRegBetween(taicpu(p).oper[0]^.reg,p,hp1,usedregs);
  5256. { Removes the second statement from
  5257. movXX mem1/reg1, reg2
  5258. movXX reg2, mem1/reg1
  5259. }
  5260. if (taicpu(p).oper[1]^.typ=top_reg) and
  5261. not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,UsedRegs)) then
  5262. begin
  5263. DebugMsg(SPeepholeOptimization + 'MovXXMovXX2Nop 1 done',p);
  5264. RemoveInstruction(hp1);
  5265. RemoveCurrentp(p); { p will now be equal to the instruction that follows what was hp1 }
  5266. Result:=true;
  5267. exit;
  5268. end
  5269. else if (taicpu(hp1).oper[1]^.typ<>top_ref) or (not(vol_write in taicpu(hp1).oper[1]^.ref^.volatility)) and
  5270. (taicpu(hp1).oper[0]^.typ<>top_ref) or (not(vol_read in taicpu(hp1).oper[0]^.ref^.volatility)) then
  5271. begin
  5272. DebugMsg(SPeepholeOptimization + 'MovXXMovXX2MoVXX 1 done',p);
  5273. RemoveInstruction(hp1);
  5274. Result:=true;
  5275. exit;
  5276. end;
  5277. end
  5278. end;
  5279. end;
  5280. end;
  5281. end;
  5282. end;
  5283. function TX86AsmOptimizer.OptPass1OP(var p : tai) : boolean;
  5284. var
  5285. hp1 : tai;
  5286. begin
  5287. result:=false;
  5288. { replace
  5289. <Op>X %mreg1,%mreg2 // Op in [ADD,MUL]
  5290. MovX %mreg2,%mreg1
  5291. dealloc %mreg2
  5292. by
  5293. <Op>X %mreg2,%mreg1
  5294. ?
  5295. }
  5296. if GetNextInstruction(p,hp1) and
  5297. { we mix single and double opperations here because we assume that the compiler
  5298. generates vmovapd only after double operations and vmovaps only after single operations }
  5299. MatchInstruction(hp1,A_MOVAPD,A_MOVAPS,[S_NO]) and
  5300. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  5301. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[1]^) and
  5302. (taicpu(p).oper[0]^.typ=top_reg) then
  5303. begin
  5304. TransferUsedRegs(TmpUsedRegs);
  5305. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5306. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  5307. begin
  5308. taicpu(p).loadoper(0,taicpu(hp1).oper[0]^);
  5309. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  5310. DebugMsg(SPeepholeOptimization + 'OpMov2Op done',p);
  5311. RemoveInstruction(hp1);
  5312. result:=true;
  5313. end;
  5314. end;
  5315. end;
  5316. function TX86AsmOptimizer.OptPass1Test(var p: tai) : boolean;
  5317. var
  5318. hp1, p_label, p_dist, hp1_dist, hp1_last: tai;
  5319. JumpLabel, JumpLabel_dist: TAsmLabel;
  5320. FirstValue, SecondValue: TCGInt;
  5321. function OptimizeJump(var InputP: tai): Boolean;
  5322. var
  5323. TempBool: Boolean;
  5324. begin
  5325. Result := False;
  5326. TempBool := True;
  5327. if DoJumpOptimizations(InputP, TempBool) or
  5328. not TempBool then
  5329. begin
  5330. Result := True;
  5331. if Assigned(InputP) then
  5332. begin
  5333. { CollapseZeroDistJump will be set to the label or an align
  5334. before it after the jump if it optimises, whether or not
  5335. the label is live or dead }
  5336. if (InputP.typ = ait_align) or
  5337. (
  5338. (InputP.typ = ait_label) and
  5339. not (tai_label(InputP).labsym.is_used)
  5340. ) then
  5341. GetNextInstruction(InputP, InputP);
  5342. end;
  5343. Exit;
  5344. end;
  5345. end;
  5346. begin
  5347. Result := False;
  5348. if (taicpu(p).oper[0]^.typ = top_const) and
  5349. (taicpu(p).oper[0]^.val <> -1) then
  5350. begin
  5351. { Convert unsigned maximum constants to -1 to aid optimisation }
  5352. case taicpu(p).opsize of
  5353. S_B:
  5354. if (taicpu(p).oper[0]^.val and $FF) = $FF then
  5355. begin
  5356. taicpu(p).oper[0]^.val := -1;
  5357. Result := True;
  5358. Exit;
  5359. end;
  5360. S_W:
  5361. if (taicpu(p).oper[0]^.val and $FFFF) = $FFFF then
  5362. begin
  5363. taicpu(p).oper[0]^.val := -1;
  5364. Result := True;
  5365. Exit;
  5366. end;
  5367. S_L:
  5368. if (taicpu(p).oper[0]^.val and $FFFFFFFF) = $FFFFFFFF then
  5369. begin
  5370. taicpu(p).oper[0]^.val := -1;
  5371. Result := True;
  5372. Exit;
  5373. end;
  5374. {$ifdef x86_64}
  5375. S_Q:
  5376. { Storing anything greater than $7FFFFFFF is not possible so do
  5377. nothing };
  5378. {$endif x86_64}
  5379. else
  5380. InternalError(2021121001);
  5381. end;
  5382. end;
  5383. if GetNextInstruction(p, hp1) and
  5384. TrySwapMovCmp(p, hp1) then
  5385. begin
  5386. Result := True;
  5387. Exit;
  5388. end;
  5389. p_label := nil;
  5390. JumpLabel := nil;
  5391. if MatchInstruction(hp1, A_Jcc, []) then
  5392. begin
  5393. if OptimizeJump(hp1) then
  5394. begin
  5395. Result := True;
  5396. if Assigned(hp1) then
  5397. begin
  5398. { CollapseZeroDistJump will be set to the label or an align
  5399. before it after the jump if it optimises, whether or not
  5400. the label is live or dead }
  5401. if (hp1.typ = ait_align) or
  5402. (
  5403. (hp1.typ = ait_label) and
  5404. not (tai_label(hp1).labsym.is_used)
  5405. ) then
  5406. GetNextInstruction(hp1, hp1);
  5407. end;
  5408. TransferUsedRegs(TmpUsedRegs);
  5409. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  5410. if not Assigned(hp1) or
  5411. (
  5412. not MatchInstruction(hp1, A_Jcc, A_SETcc, A_CMOVcc, []) and
  5413. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)
  5414. ) then
  5415. begin
  5416. { No more conditional jumps; conditional statement is no longer required }
  5417. DebugMsg(SPeepholeOptimization + 'Removed unnecessary condition (Test2Nop)', p);
  5418. RemoveCurrentP(p);
  5419. end;
  5420. Exit;
  5421. end;
  5422. if IsJumpToLabel(taicpu(hp1)) then
  5423. begin
  5424. JumpLabel := TAsmLabel(taicpu(hp1).oper[0]^.ref^.symbol);
  5425. if Assigned(JumpLabel) then
  5426. p_label := getlabelwithsym(JumpLabel);
  5427. end;
  5428. end;
  5429. { Search for:
  5430. test $x,(reg/ref)
  5431. jne @lbl1
  5432. test $y,(reg/ref) (same register or reference)
  5433. jne @lbl1
  5434. Change to:
  5435. test $(x or y),(reg/ref)
  5436. jne @lbl1
  5437. (Note, this doesn't work with je instead of jne)
  5438. Also catch cases where "cmp $0,(reg/ref)" and "test %reg,%reg" are used.
  5439. Also search for:
  5440. test $x,(reg/ref)
  5441. je @lbl1
  5442. ...
  5443. test $y,(reg/ref)
  5444. je/jne @lbl2
  5445. If (x or y) = x, then the second jump is deterministic
  5446. }
  5447. if (
  5448. (
  5449. (taicpu(p).oper[0]^.typ = top_const) or
  5450. (
  5451. { test %reg,%reg can be considered equivalent to test, -1,%reg }
  5452. (taicpu(p).oper[0]^.typ = top_reg) and
  5453. MatchOperand(taicpu(p).oper[1]^, taicpu(p).oper[0]^.reg)
  5454. )
  5455. ) and
  5456. MatchInstruction(hp1, A_JCC, [])
  5457. ) then
  5458. begin
  5459. if (taicpu(p).oper[0]^.typ = top_reg) and
  5460. MatchOperand(taicpu(p).oper[1]^, taicpu(p).oper[0]^.reg) then
  5461. FirstValue := -1
  5462. else
  5463. FirstValue := taicpu(p).oper[0]^.val;
  5464. { If we have several test/jne's in a row, it might be the case that
  5465. the second label doesn't go to the same location, but the one
  5466. after it might (e.g. test; jne @lbl1; test; jne @lbl2; test @lbl1),
  5467. so accommodate for this with a while loop.
  5468. }
  5469. hp1_last := hp1;
  5470. while (
  5471. (
  5472. (taicpu(p).oper[1]^.typ = top_reg) and
  5473. GetNextInstructionUsingReg(hp1_last, p_dist, taicpu(p).oper[1]^.reg)
  5474. ) or GetNextInstruction(hp1_last, p_dist)
  5475. ) and (p_dist.typ = ait_instruction) do
  5476. begin
  5477. if (
  5478. (
  5479. (taicpu(p_dist).opcode = A_TEST) and
  5480. (
  5481. (taicpu(p_dist).oper[0]^.typ = top_const) or
  5482. { test %reg,%reg can be considered equivalent to test, -1,%reg }
  5483. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p_dist).oper[0]^)
  5484. )
  5485. ) or
  5486. (
  5487. { cmp 0,%reg = test %reg,%reg }
  5488. (taicpu(p_dist).opcode = A_CMP) and
  5489. MatchOperand(taicpu(p_dist).oper[0]^, 0)
  5490. )
  5491. ) and
  5492. { Make sure the destination operands are actually the same }
  5493. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p).oper[1]^) and
  5494. GetNextInstruction(p_dist, hp1_dist) and
  5495. MatchInstruction(hp1_dist, A_JCC, []) then
  5496. begin
  5497. if OptimizeJump(hp1_dist) then
  5498. begin
  5499. Result := True;
  5500. Exit;
  5501. end;
  5502. if
  5503. (taicpu(p_dist).opcode = A_CMP) { constant will be zero } or
  5504. (
  5505. (taicpu(p_dist).oper[0]^.typ = top_reg) and
  5506. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p_dist).oper[0]^.reg)
  5507. ) then
  5508. SecondValue := -1
  5509. else
  5510. SecondValue := taicpu(p_dist).oper[0]^.val;
  5511. { If both of the TEST constants are identical, delete the
  5512. second TEST that is unnecessary (be careful though, just
  5513. in case the flags are modified in between) }
  5514. if (FirstValue = SecondValue) then
  5515. begin
  5516. if condition_in(taicpu(hp1_dist).condition, taicpu(hp1).condition) then
  5517. begin
  5518. { Since the second jump's condition is a subset of the first, we
  5519. know it will never branch because the first jump dominates it.
  5520. Get it out of the way now rather than wait for the jump
  5521. optimisations for a speed boost. }
  5522. if IsJumpToLabel(taicpu(hp1_dist)) then
  5523. TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol).DecRefs;
  5524. DebugMsg(SPeepholeOptimization + 'Removed dominated jump (via TEST/Jcc/TEST)', hp1_dist);
  5525. RemoveInstruction(hp1_dist);
  5526. Result := True;
  5527. end
  5528. else if condition_in(inverse_cond(taicpu(hp1).condition), taicpu(hp1_dist).condition) then
  5529. begin
  5530. { If the inverse of the first condition is a subset of the second,
  5531. the second one will definitely branch if the first one doesn't }
  5532. DebugMsg(SPeepholeOptimization + 'Conditional jump will always branch (via TEST/Jcc/TEST)', hp1_dist);
  5533. { We can remove the TEST instruction too }
  5534. DebugMsg(SPeepholeOptimization + 'TEST/Jcc/TEST; removed superfluous TEST', p_dist);
  5535. RemoveInstruction(p_dist);
  5536. MakeUnconditional(taicpu(hp1_dist));
  5537. RemoveDeadCodeAfterJump(hp1_dist);
  5538. { Since the jump is now unconditional, we can't
  5539. continue any further with this particular
  5540. optimisation. The original TEST is still intact
  5541. though, so there might be something else we can
  5542. do }
  5543. Include(OptsToCheck, aoc_ForceNewIteration);
  5544. Break;
  5545. end;
  5546. if Result or
  5547. { If a jump wasn't removed or made unconditional, only
  5548. remove the identical TEST instruction if the flags
  5549. weren't modified }
  5550. not RegModifiedBetween(NR_DEFAULTFLAGS, hp1, p_dist) then
  5551. begin
  5552. DebugMsg(SPeepholeOptimization + 'TEST/Jcc/TEST; removed superfluous TEST', p_dist);
  5553. RemoveInstruction(p_dist);
  5554. { If the jump was removed or made unconditional, we
  5555. don't need to allocate NR_DEFAULTFLAGS over the
  5556. entire range }
  5557. if not Result then
  5558. begin
  5559. { Mark the flags as 'in use' over the entire range }
  5560. AllocRegBetween(NR_DEFAULTFLAGS, hp1, hp1_dist, UsedRegs);
  5561. { Speed gain - continue search from the Jcc instruction }
  5562. hp1_last := hp1_dist;
  5563. { Only the TEST instruction was removed, and the
  5564. original was unchanged, so we can safely do
  5565. another iteration of the while loop }
  5566. Include(OptsToCheck, aoc_ForceNewIteration);
  5567. Continue;
  5568. end;
  5569. Exit;
  5570. end;
  5571. end;
  5572. hp1_last := nil;
  5573. if (taicpu(hp1).condition in [C_NE, C_NZ]) and
  5574. (
  5575. { In this situation, the TEST/JNE pairs must be adjacent (fixes #40366) }
  5576. { Always adjacent under -O2 and under }
  5577. not(cs_opt_level3 in current_settings.optimizerswitches) or
  5578. (
  5579. GetNextInstruction(hp1, hp1_last) and
  5580. (hp1_last = p_dist)
  5581. )
  5582. ) and
  5583. (
  5584. (
  5585. { Test the following variant:
  5586. test $x,(reg/ref)
  5587. jne @lbl1
  5588. test $y,(reg/ref)
  5589. je @lbl2
  5590. @lbl1:
  5591. Becomes:
  5592. test $(x or y),(reg/ref)
  5593. je @lbl2
  5594. @lbl1: (may become a dead label)
  5595. }
  5596. (taicpu(hp1_dist).condition in [C_E, C_Z]) and
  5597. GetNextInstruction(hp1_dist, hp1_last) and
  5598. (hp1_last = p_label)
  5599. ) or
  5600. (
  5601. (taicpu(hp1_dist).condition in [C_NE, C_NZ]) and
  5602. { If the first instruction is test %reg,%reg or test $-1,%reg,
  5603. then the second jump will never branch, so it can also be
  5604. removed regardless of where it goes }
  5605. (
  5606. (FirstValue = -1) or
  5607. (SecondValue = -1) or
  5608. MatchOperand(taicpu(hp1_dist).oper[0]^, taicpu(hp1).oper[0]^)
  5609. )
  5610. )
  5611. ) then
  5612. begin
  5613. { Same jump location... can be a register since nothing's changed }
  5614. { If any of the entries are equivalent to test %reg,%reg, then the
  5615. merged $(x or y) is also test %reg,%reg / test $-1,%reg }
  5616. taicpu(p).loadconst(0, FirstValue or SecondValue);
  5617. if (hp1_last = p_label) then
  5618. begin
  5619. { Variant }
  5620. DebugMsg(SPeepholeOptimization + 'TEST/JNE/TEST/JE/@Lbl merged', p);
  5621. RemoveInstruction(p_dist);
  5622. if Assigned(JumpLabel) then
  5623. JumpLabel.decrefs;
  5624. RemoveInstruction(hp1);
  5625. end
  5626. else
  5627. begin
  5628. { Only remove the second test if no jumps or other conditional instructions follow }
  5629. TransferUsedRegs(TmpUsedRegs);
  5630. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  5631. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  5632. UpdateUsedRegs(TmpUsedRegs, tai(p_dist.Next));
  5633. if not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1_dist, TmpUsedRegs) then
  5634. begin
  5635. DebugMsg(SPeepholeOptimization + 'TEST/JNE/TEST/JNE merged', p);
  5636. RemoveInstruction(p_dist);
  5637. { Remove the first jump, not the second, to keep
  5638. any register deallocations between the second
  5639. TEST/JNE pair in the same place. Aids future
  5640. optimisation. }
  5641. if Assigned(JumpLabel) then
  5642. JumpLabel.decrefs;
  5643. RemoveInstruction(hp1);
  5644. end
  5645. else
  5646. begin
  5647. DebugMsg(SPeepholeOptimization + 'TEST/JNE/TEST/JNE merged (second TEST preserved)', p);
  5648. if IsJumpToLabel(taicpu(hp1_dist)) then
  5649. TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol).DecRefs;
  5650. { Remove second jump in this instance }
  5651. RemoveInstruction(hp1_dist);
  5652. end;
  5653. end;
  5654. Result := True;
  5655. Exit;
  5656. end;
  5657. end;
  5658. if { If -O2 and under, it may stop on any old instruction }
  5659. (cs_opt_level3 in current_settings.optimizerswitches) and
  5660. (taicpu(p).oper[1]^.typ = top_reg) and
  5661. not RegModifiedByInstruction(taicpu(p).oper[1]^.reg, p_dist) then
  5662. begin
  5663. hp1_last := p_dist;
  5664. Continue;
  5665. end;
  5666. Break;
  5667. end;
  5668. end;
  5669. { Search for:
  5670. test %reg,%reg
  5671. j(c1) @lbl1
  5672. ...
  5673. @lbl:
  5674. test %reg,%reg (same register)
  5675. j(c2) @lbl2
  5676. If c2 is a subset of c1, change to:
  5677. test %reg,%reg
  5678. j(c1) @lbl2
  5679. (@lbl1 may become a dead label as a result)
  5680. }
  5681. if (taicpu(p).oper[1]^.typ = top_reg) and
  5682. (taicpu(p).oper[0]^.typ = top_reg) and
  5683. (taicpu(p).oper[0]^.reg = taicpu(p).oper[1]^.reg) and
  5684. { p_label <> nil is a marker that hp1 is a Jcc to a label }
  5685. Assigned(p_label) and
  5686. GetNextInstruction(p_label, p_dist) and
  5687. MatchInstruction(p_dist, A_TEST, []) and
  5688. { It's fine if the second test uses smaller sub-registers }
  5689. (taicpu(p_dist).opsize <= taicpu(p).opsize) and
  5690. MatchOpType(taicpu(p_dist), top_reg, top_reg) and
  5691. SuperRegistersEqual(taicpu(p_dist).oper[0]^.reg, taicpu(p).oper[0]^.reg) and
  5692. SuperRegistersEqual(taicpu(p_dist).oper[1]^.reg, taicpu(p).oper[1]^.reg) and
  5693. GetNextInstruction(p_dist, hp1_dist) and
  5694. MatchInstruction(hp1_dist, A_JCC, []) then { This doesn't have to be an explicit label }
  5695. begin
  5696. JumpLabel_dist := TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol);
  5697. if JumpLabel = JumpLabel_dist then
  5698. { This is an infinite loop }
  5699. Exit;
  5700. { Best optimisation when the first condition is a subset (or equal) of the second }
  5701. if condition_in(taicpu(hp1).condition, taicpu(hp1_dist).condition) then
  5702. begin
  5703. { Any registers used here will already be allocated }
  5704. if Assigned(JumpLabel) then
  5705. JumpLabel.DecRefs;
  5706. DebugMsg(SPeepholeOptimization + 'TEST/Jcc/@Lbl/TEST/Jcc -> TEST/Jcc, redirecting first jump', hp1);
  5707. taicpu(hp1).loadref(0, taicpu(hp1_dist).oper[0]^.ref^); { This also increases the reference count }
  5708. Result := True;
  5709. Exit;
  5710. end;
  5711. end;
  5712. end;
  5713. function TX86AsmOptimizer.OptPass1Add(var p : tai) : boolean;
  5714. var
  5715. hp1, hp2: tai;
  5716. ActiveReg: TRegister;
  5717. OldOffset: asizeint;
  5718. ThisConst: TCGInt;
  5719. function RegDeallocated: Boolean;
  5720. begin
  5721. TransferUsedRegs(TmpUsedRegs);
  5722. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5723. Result := not(RegUsedAfterInstruction(ActiveReg,hp1,TmpUsedRegs))
  5724. end;
  5725. begin
  5726. result:=false;
  5727. hp1 := nil;
  5728. { replace
  5729. addX const,%reg1
  5730. leaX (%reg1,%reg1,Y),%reg2 // Base or index might not be equal to reg1
  5731. dealloc %reg1
  5732. by
  5733. leaX const+const*Y(%reg1,%reg1,Y),%reg2
  5734. }
  5735. if MatchOpType(taicpu(p),top_const,top_reg) then
  5736. begin
  5737. ActiveReg := taicpu(p).oper[1]^.reg;
  5738. { Ensures the entire register was updated }
  5739. if (taicpu(p).opsize >= S_L) and
  5740. GetNextInstructionUsingReg(p,hp1, ActiveReg) and
  5741. MatchInstruction(hp1,A_LEA,[]) and
  5742. (SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.base) or
  5743. SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.index)) and
  5744. (
  5745. { Cover the case where the register in the reference is also the destination register }
  5746. Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, ActiveReg) or
  5747. (
  5748. { Try to avoid the expensive check of RegUsedAfterInstruction if we know it will return False }
  5749. not SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ActiveReg) and
  5750. RegDeallocated
  5751. )
  5752. ) then
  5753. begin
  5754. OldOffset := taicpu(hp1).oper[0]^.ref^.offset;
  5755. {$push}
  5756. {$R-}{$Q-}
  5757. { Explicitly disable overflow checking for these offset calculation
  5758. as those do not matter for the final result }
  5759. if ActiveReg=taicpu(hp1).oper[0]^.ref^.base then
  5760. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val);
  5761. if ActiveReg=taicpu(hp1).oper[0]^.ref^.index then
  5762. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val*max(taicpu(hp1).oper[0]^.ref^.scalefactor,1));
  5763. {$pop}
  5764. {$ifdef x86_64}
  5765. if (taicpu(hp1).oper[0]^.ref^.offset > $7FFFFFFF) or (taicpu(hp1).oper[0]^.ref^.offset < -2147483648) then
  5766. begin
  5767. { Overflow; abort }
  5768. taicpu(hp1).oper[0]^.ref^.offset := OldOffset;
  5769. end
  5770. else
  5771. {$endif x86_64}
  5772. begin
  5773. DebugMsg(SPeepholeOptimization + 'AddLea2Lea done',p);
  5774. if not (cs_opt_level3 in current_settings.optimizerswitches) then
  5775. { hp1 is the immediate next instruction for sure - good for a quick speed boost }
  5776. RemoveCurrentP(p, hp1)
  5777. else
  5778. RemoveCurrentP(p);
  5779. result:=true;
  5780. Exit;
  5781. end;
  5782. end;
  5783. if (
  5784. { Save calling GetNextInstructionUsingReg again }
  5785. Assigned(hp1) or
  5786. GetNextInstructionUsingReg(p,hp1, ActiveReg)
  5787. ) and
  5788. MatchInstruction(hp1,A_ADD,A_SUB,[taicpu(p).opsize]) and
  5789. (taicpu(hp1).oper[1]^.reg = ActiveReg) then
  5790. begin
  5791. { Make sure the flags aren't in use by the second operation }
  5792. TransferUsedRegs(TmpUsedRegs);
  5793. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.next), hp1);
  5794. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  5795. begin
  5796. if taicpu(hp1).oper[0]^.typ = top_const then
  5797. begin
  5798. { Merge add const1,%reg; add/sub const2,%reg to add const1+/-const2,%reg }
  5799. if taicpu(hp1).opcode = A_ADD then
  5800. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val
  5801. else
  5802. ThisConst := taicpu(p).oper[0]^.val - taicpu(hp1).oper[0]^.val;
  5803. Result := True;
  5804. { Handle any overflows }
  5805. case taicpu(p).opsize of
  5806. S_B:
  5807. taicpu(p).oper[0]^.val := ThisConst and $FF;
  5808. S_W:
  5809. taicpu(p).oper[0]^.val := ThisConst and $FFFF;
  5810. S_L:
  5811. taicpu(p).oper[0]^.val := ThisConst and $FFFFFFFF;
  5812. {$ifdef x86_64}
  5813. S_Q:
  5814. if (ThisConst > $7FFFFFFF) or (ThisConst < -2147483648) then
  5815. { Overflow; abort }
  5816. Result := False
  5817. else
  5818. taicpu(p).oper[0]^.val := ThisConst;
  5819. {$endif x86_64}
  5820. else
  5821. InternalError(2021102610);
  5822. end;
  5823. { Result may get set to False again if the combined immediate overflows for S_Q sizes }
  5824. if Result then
  5825. begin
  5826. if (taicpu(p).oper[0]^.val < 0) and
  5827. (
  5828. ((taicpu(p).opsize = S_B) and (taicpu(p).oper[0]^.val <> -128)) or
  5829. ((taicpu(p).opsize = S_W) and (taicpu(p).oper[0]^.val <> -32768)) or
  5830. ((taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) and (taicpu(p).oper[0]^.val <> -2147483648))
  5831. ) then
  5832. begin
  5833. DebugMsg(SPeepholeOptimization + 'ADD; ADD/SUB -> SUB',p);
  5834. taicpu(p).opcode := A_SUB;
  5835. taicpu(p).oper[0]^.val := -taicpu(p).oper[0]^.val;
  5836. end
  5837. else
  5838. DebugMsg(SPeepholeOptimization + 'ADD; ADD/SUB -> ADD',p);
  5839. RemoveInstruction(hp1);
  5840. end;
  5841. end
  5842. else
  5843. begin
  5844. { Move the constant addition to after the reg/ref addition to improve optimisation }
  5845. DebugMsg(SPeepholeOptimization + 'Add/sub swap 1a done',p);
  5846. Asml.Remove(p);
  5847. Asml.InsertAfter(p, hp1);
  5848. p := hp1;
  5849. Result := True;
  5850. Exit;
  5851. end;
  5852. end;
  5853. end;
  5854. if DoArithCombineOpt(p) then
  5855. Result:=true;
  5856. end;
  5857. end;
  5858. function TX86AsmOptimizer.OptPass1LEA(var p : tai) : boolean;
  5859. var
  5860. hp1, hp2: tai;
  5861. ref: Integer;
  5862. saveref: treference;
  5863. offsetcalc: Int64;
  5864. TempReg: TRegister;
  5865. Multiple: TCGInt;
  5866. Adjacent, IntermediateRegDiscarded: Boolean;
  5867. begin
  5868. Result:=false;
  5869. { play save and throw an error if LEA uses a seg register prefix,
  5870. this is most likely an error somewhere else }
  5871. if taicpu(p).oper[0]^.ref^.Segment<>NR_NO then
  5872. internalerror(2022022001);
  5873. { changes "lea (%reg1), %reg2" into "mov %reg1, %reg2" }
  5874. if (taicpu(p).oper[0]^.ref^.base <> NR_NO) and
  5875. (taicpu(p).oper[0]^.ref^.index = NR_NO) and
  5876. (
  5877. { do not mess with leas accessing the stack pointer
  5878. unless it's a null operation }
  5879. (taicpu(p).oper[1]^.reg <> NR_STACK_POINTER_REG) or
  5880. (
  5881. (taicpu(p).oper[0]^.ref^.base = NR_STACK_POINTER_REG) and
  5882. (taicpu(p).oper[0]^.ref^.offset = 0)
  5883. )
  5884. ) and
  5885. (not(Assigned(taicpu(p).oper[0]^.ref^.Symbol))) then
  5886. begin
  5887. if (taicpu(p).oper[0]^.ref^.offset = 0) then
  5888. begin
  5889. if (taicpu(p).oper[0]^.ref^.base <> taicpu(p).oper[1]^.reg) then
  5890. begin
  5891. taicpu(p).opcode := A_MOV;
  5892. taicpu(p).loadreg(0, taicpu(p).oper[0]^.ref^.base);
  5893. DebugMsg(SPeepholeOptimization + 'Lea2Mov done',p);
  5894. end
  5895. else
  5896. begin
  5897. DebugMsg(SPeepholeOptimization + 'Lea2Nop done',p);
  5898. RemoveCurrentP(p);
  5899. end;
  5900. Result:=true;
  5901. exit;
  5902. end
  5903. else if (
  5904. { continue to use lea to adjust the stack pointer,
  5905. it is the recommended way, but only if not optimizing for size }
  5906. (taicpu(p).oper[1]^.reg<>NR_STACK_POINTER_REG) or
  5907. (cs_opt_size in current_settings.optimizerswitches)
  5908. ) and
  5909. { If the flags register is in use, don't change the instruction
  5910. to an ADD otherwise this will scramble the flags. [Kit] }
  5911. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  5912. ConvertLEA(taicpu(p)) then
  5913. begin
  5914. Result:=true;
  5915. exit;
  5916. end;
  5917. end;
  5918. { Don't optimise if the stack or frame pointer is the destination register }
  5919. if (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG) or (taicpu(p).oper[1]^.reg=current_procinfo.framepointer) then
  5920. Exit;
  5921. if GetNextInstruction(p,hp1) and
  5922. (hp1.typ=ait_instruction) then
  5923. begin
  5924. if MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) and
  5925. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  5926. MatchOpType(Taicpu(hp1),top_reg,top_reg) then
  5927. begin
  5928. TransferUsedRegs(TmpUsedRegs);
  5929. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5930. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  5931. begin
  5932. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  5933. DebugMsg(SPeepholeOptimization + 'LeaMov2Lea done',p);
  5934. RemoveInstruction(hp1);
  5935. result:=true;
  5936. exit;
  5937. end;
  5938. end;
  5939. { changes
  5940. lea <ref1>, reg1
  5941. <op> ...,<ref. with reg1>,...
  5942. to
  5943. <op> ...,<ref1>,... }
  5944. { find a reference which uses reg1 }
  5945. if (taicpu(hp1).ops>=1) and (taicpu(hp1).oper[0]^.typ=top_ref) and RegInOp(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^) then
  5946. ref:=0
  5947. else if (taicpu(hp1).ops>=2) and (taicpu(hp1).oper[1]^.typ=top_ref) and RegInOp(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^) then
  5948. ref:=1
  5949. else
  5950. ref:=-1;
  5951. if (ref<>-1) and
  5952. { reg1 must be either the base or the index }
  5953. ((taicpu(hp1).oper[ref]^.ref^.base=taicpu(p).oper[1]^.reg) xor (taicpu(hp1).oper[ref]^.ref^.index=taicpu(p).oper[1]^.reg)) then
  5954. begin
  5955. { reg1 can be removed from the reference }
  5956. saveref:=taicpu(hp1).oper[ref]^.ref^;
  5957. if taicpu(hp1).oper[ref]^.ref^.base=taicpu(p).oper[1]^.reg then
  5958. taicpu(hp1).oper[ref]^.ref^.base:=NR_NO
  5959. else if taicpu(hp1).oper[ref]^.ref^.index=taicpu(p).oper[1]^.reg then
  5960. taicpu(hp1).oper[ref]^.ref^.index:=NR_NO
  5961. else
  5962. Internalerror(2019111201);
  5963. { check if the can insert all data of the lea into the second instruction }
  5964. if ((taicpu(hp1).oper[ref]^.ref^.base=taicpu(p).oper[1]^.reg) or (taicpu(hp1).oper[ref]^.ref^.scalefactor <= 1)) and
  5965. ((taicpu(p).oper[0]^.ref^.base=NR_NO) or (taicpu(hp1).oper[ref]^.ref^.base=NR_NO)) and
  5966. ((taicpu(p).oper[0]^.ref^.index=NR_NO) or (taicpu(hp1).oper[ref]^.ref^.index=NR_NO)) and
  5967. ((taicpu(p).oper[0]^.ref^.symbol=nil) or (taicpu(hp1).oper[ref]^.ref^.symbol=nil)) and
  5968. ((taicpu(p).oper[0]^.ref^.relsymbol=nil) or (taicpu(hp1).oper[ref]^.ref^.relsymbol=nil)) and
  5969. ((taicpu(p).oper[0]^.ref^.scalefactor <= 1) or (taicpu(hp1).oper[ref]^.ref^.scalefactor <= 1)) and
  5970. (taicpu(p).oper[0]^.ref^.segment=NR_NO) and (taicpu(hp1).oper[ref]^.ref^.segment=NR_NO)
  5971. {$ifdef x86_64}
  5972. and (abs(taicpu(hp1).oper[ref]^.ref^.offset+taicpu(p).oper[0]^.ref^.offset)<=$7fffffff)
  5973. and (((taicpu(p).oper[0]^.ref^.base<>NR_RIP) and (taicpu(p).oper[0]^.ref^.index<>NR_RIP)) or
  5974. ((taicpu(hp1).oper[ref]^.ref^.base=NR_NO) and (taicpu(hp1).oper[ref]^.ref^.index=NR_NO))
  5975. )
  5976. {$endif x86_64}
  5977. then
  5978. begin
  5979. { reg1 might not used by the second instruction after it is remove from the reference }
  5980. if not(RegInInstruction(taicpu(p).oper[1]^.reg,taicpu(hp1))) then
  5981. begin
  5982. TransferUsedRegs(TmpUsedRegs);
  5983. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  5984. { reg1 is not updated so it might not be used afterwards }
  5985. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  5986. begin
  5987. DebugMsg(SPeepholeOptimization + 'LeaOp2Op done',p);
  5988. if taicpu(p).oper[0]^.ref^.base<>NR_NO then
  5989. taicpu(hp1).oper[ref]^.ref^.base:=taicpu(p).oper[0]^.ref^.base;
  5990. if taicpu(p).oper[0]^.ref^.index<>NR_NO then
  5991. taicpu(hp1).oper[ref]^.ref^.index:=taicpu(p).oper[0]^.ref^.index;
  5992. if taicpu(p).oper[0]^.ref^.symbol<>nil then
  5993. taicpu(hp1).oper[ref]^.ref^.symbol:=taicpu(p).oper[0]^.ref^.symbol;
  5994. if taicpu(p).oper[0]^.ref^.relsymbol<>nil then
  5995. taicpu(hp1).oper[ref]^.ref^.relsymbol:=taicpu(p).oper[0]^.ref^.relsymbol;
  5996. if taicpu(p).oper[0]^.ref^.scalefactor > 1 then
  5997. taicpu(hp1).oper[ref]^.ref^.scalefactor:=taicpu(p).oper[0]^.ref^.scalefactor;
  5998. inc(taicpu(hp1).oper[ref]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset);
  5999. RemoveCurrentP(p, hp1);
  6000. result:=true;
  6001. exit;
  6002. end
  6003. end;
  6004. end;
  6005. { recover }
  6006. taicpu(hp1).oper[ref]^.ref^:=saveref;
  6007. end;
  6008. Adjacent := RegInInstruction(taicpu(p).oper[1]^.reg, hp1);
  6009. if Adjacent or
  6010. { Check further ahead (up to 2 instructions ahead for -O2) }
  6011. GetNextInstructionUsingReg(hp1,hp1,taicpu(p).oper[1]^.reg) then
  6012. begin
  6013. { Check common LEA/LEA conditions }
  6014. if MatchInstruction(hp1,A_LEA,[taicpu(p).opsize]) and
  6015. (taicpu(p).oper[0]^.ref^.relsymbol = nil) and
  6016. (taicpu(p).oper[0]^.ref^.segment = NR_NO) and
  6017. (taicpu(p).oper[0]^.ref^.symbol = nil) and
  6018. (taicpu(hp1).oper[0]^.ref^.relsymbol = nil) and
  6019. (taicpu(hp1).oper[0]^.ref^.segment = NR_NO) and
  6020. (taicpu(hp1).oper[0]^.ref^.symbol = nil) and
  6021. (
  6022. { If p and hp1 are adjacent, RegModifiedBetween always returns False, so avoid
  6023. calling it (since it calls GetNextInstruction) }
  6024. Adjacent or
  6025. (
  6026. (
  6027. (taicpu(p).oper[0]^.ref^.base = NR_NO) or { Don't call RegModifiedBetween unnecessarily }
  6028. not(RegModifiedBetween(taicpu(p).oper[0]^.ref^.base,p,hp1))
  6029. ) and (
  6030. (taicpu(p).oper[0]^.ref^.index = taicpu(p).oper[0]^.ref^.base) or { Don't call RegModifiedBetween unnecessarily }
  6031. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  6032. not(RegModifiedBetween(taicpu(p).oper[0]^.ref^.index,p,hp1))
  6033. )
  6034. )
  6035. ) then
  6036. begin
  6037. TransferUsedRegs(TmpUsedRegs);
  6038. hp2 := p;
  6039. repeat
  6040. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  6041. until not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  6042. IntermediateRegDiscarded :=
  6043. (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg) or
  6044. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs);
  6045. { changes
  6046. lea offset1(regX,scale), reg1
  6047. lea offset2(reg1,reg1), reg2
  6048. to
  6049. lea (offset1*scale*2)+offset2(regX,scale*2), reg2
  6050. and
  6051. lea offset1(regX,scale1), reg1
  6052. lea offset2(reg1,scale2), reg2
  6053. to
  6054. lea (offset1*scale1*2)+offset2(regX,scale1*scale2), reg2
  6055. and
  6056. lea offset1(regX,scale1), reg1
  6057. lea offset2(reg3,reg1,scale2), reg2
  6058. to
  6059. lea (offset1*scale*2)+offset2(reg3,regX,scale1*scale2), reg2
  6060. ... so long as the final scale does not exceed 8
  6061. (Similarly, allow the first instruction to be "lea (regX,regX),reg1")
  6062. }
  6063. if (taicpu(p).oper[0]^.ref^.base<>NR_STACK_POINTER_REG) and { lea (%rsp,scale),reg is not a valid encoding }
  6064. (
  6065. { Don't optimise if size is a concern and the intermediate register remains in use }
  6066. IntermediateRegDiscarded or
  6067. (
  6068. not (cs_opt_size in current_settings.optimizerswitches) and
  6069. { If the intermediate register is not discarded, it must not
  6070. appear in the first LEA's reference. (Fixes #41166) }
  6071. not RegInRef(taicpu(p).oper[1]^.reg, taicpu(p).oper[0]^.ref^)
  6072. )
  6073. ) and
  6074. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  6075. (
  6076. (taicpu(p).oper[0]^.ref^.base <> taicpu(p).oper[0]^.ref^.index) or
  6077. (taicpu(p).oper[0]^.ref^.scalefactor <= 1)
  6078. ) and (
  6079. (
  6080. { lea (reg1,scale2), reg2 variant }
  6081. (taicpu(hp1).oper[0]^.ref^.base <> taicpu(p).oper[1]^.reg) and
  6082. (
  6083. Adjacent or
  6084. not RegModifiedBetween(taicpu(hp1).oper[0]^.ref^.base, p, hp1)
  6085. ) and
  6086. (
  6087. (
  6088. (taicpu(p).oper[0]^.ref^.base = NR_NO) and
  6089. (taicpu(hp1).oper[0]^.ref^.scalefactor * taicpu(p).oper[0]^.ref^.scalefactor <= 8)
  6090. ) or (
  6091. { lea (regX,regX), reg1 variant }
  6092. (taicpu(p).oper[0]^.ref^.base = taicpu(p).oper[0]^.ref^.index) and
  6093. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 4)
  6094. )
  6095. )
  6096. ) or (
  6097. { lea (reg1,reg1), reg1 variant }
  6098. (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) and
  6099. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 1)
  6100. )
  6101. ) then
  6102. begin
  6103. { Make everything homogeneous to make calculations easier }
  6104. if (taicpu(p).oper[0]^.ref^.base <> NR_NO) then
  6105. begin
  6106. if taicpu(p).oper[0]^.ref^.index <> NR_NO then
  6107. { Convert lea (regX,regX),reg1 to lea (regX,2),reg1 }
  6108. taicpu(p).oper[0]^.ref^.scalefactor := 2
  6109. else
  6110. taicpu(p).oper[0]^.ref^.index := taicpu(p).oper[0]^.ref^.base;
  6111. taicpu(p).oper[0]^.ref^.base := NR_NO;
  6112. end;
  6113. { Make sure the offset doesn't go out of range (use 64-bit arithmetic)}
  6114. offsetcalc := taicpu(hp1).oper[0]^.ref^.offset;
  6115. Inc(offsetcalc, Int64(taicpu(p).oper[0]^.ref^.offset) * max(taicpu(hp1).oper[0]^.ref^.scalefactor, 1));
  6116. if (offsetcalc <= $7FFFFFFF) and (offsetcalc >= -2147483648) then
  6117. begin
  6118. if (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) and
  6119. (taicpu(hp1).oper[0]^.ref^.index <> taicpu(p).oper[1]^.reg) then
  6120. begin
  6121. { Put the register to change in the index register }
  6122. TempReg := taicpu(hp1).oper[0]^.ref^.index;
  6123. taicpu(hp1).oper[0]^.ref^.index := taicpu(hp1).oper[0]^.ref^.base;
  6124. taicpu(hp1).oper[0]^.ref^.base := TempReg;
  6125. end;
  6126. { Change lea (reg,reg) to lea(,reg,2) }
  6127. if (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) then
  6128. begin
  6129. taicpu(hp1).oper[0]^.ref^.base := NR_NO;
  6130. taicpu(hp1).oper[0]^.ref^.scalefactor := 2;
  6131. end;
  6132. if (taicpu(p).oper[0]^.ref^.offset <> 0) then
  6133. Inc(taicpu(hp1).oper[0]^.ref^.offset, taicpu(p).oper[0]^.ref^.offset * max(taicpu(hp1).oper[0]^.ref^.scalefactor, 1));
  6134. taicpu(hp1).oper[0]^.ref^.index := taicpu(p).oper[0]^.ref^.index;
  6135. { Just to prevent miscalculations }
  6136. if (taicpu(hp1).oper[0]^.ref^.scalefactor = 0) then
  6137. taicpu(hp1).oper[0]^.ref^.scalefactor := taicpu(p).oper[0]^.ref^.scalefactor
  6138. else
  6139. taicpu(hp1).oper[0]^.ref^.scalefactor := taicpu(hp1).oper[0]^.ref^.scalefactor * max(taicpu(p).oper[0]^.ref^.scalefactor, 1);
  6140. { Only remove the first LEA if we don't need the intermediate register's value as is }
  6141. if IntermediateRegDiscarded then
  6142. begin
  6143. DebugMsg(SPeepholeOptimization + 'LeaLea2Lea 2 done',p);
  6144. RemoveCurrentP(p);
  6145. end
  6146. else
  6147. DebugMsg(SPeepholeOptimization + 'LeaLea2LeaLea 2 done (intermediate register still in use)',p);
  6148. result:=true;
  6149. exit;
  6150. end;
  6151. end;
  6152. { changes
  6153. lea offset1(regX), reg1
  6154. lea offset2(reg1), reg2
  6155. to
  6156. lea offset1+offset2(regX), reg2 }
  6157. if (
  6158. { Don't optimise if size is a concern and the intermediate register remains in use }
  6159. IntermediateRegDiscarded or
  6160. (
  6161. not (cs_opt_size in current_settings.optimizerswitches) and
  6162. { If the intermediate register is not discarded, it must not
  6163. appear in the first LEA's reference. (Fixes #41166) }
  6164. not RegInRef(taicpu(p).oper[1]^.reg, taicpu(p).oper[0]^.ref^)
  6165. )
  6166. ) and
  6167. (
  6168. (
  6169. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  6170. (getsupreg(taicpu(p).oper[0]^.ref^.base)<>RS_ESP) and
  6171. (taicpu(p).oper[0]^.ref^.index = NR_NO)
  6172. ) or (
  6173. (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) and
  6174. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 1) and
  6175. (
  6176. (
  6177. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  6178. (taicpu(p).oper[0]^.ref^.base = NR_NO)
  6179. ) or (
  6180. (taicpu(p).oper[0]^.ref^.scalefactor <= 1) and
  6181. (
  6182. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  6183. (
  6184. (taicpu(p).oper[0]^.ref^.index = taicpu(p).oper[0]^.ref^.base) and
  6185. (
  6186. (taicpu(hp1).oper[0]^.ref^.index = NR_NO) or
  6187. (taicpu(hp1).oper[0]^.ref^.base = NR_NO)
  6188. )
  6189. )
  6190. )
  6191. )
  6192. )
  6193. )
  6194. ) then
  6195. begin
  6196. { Make sure the offset doesn't go out of range (use 64-bit arithmetic)}
  6197. offsetcalc := taicpu(hp1).oper[0]^.ref^.offset;
  6198. Inc(offsetcalc, Int64(taicpu(p).oper[0]^.ref^.offset) * max(taicpu(hp1).oper[0]^.ref^.scalefactor, 1));
  6199. if (offsetcalc <= $7FFFFFFF) and (offsetcalc >= -2147483648) then
  6200. begin
  6201. if taicpu(hp1).oper[0]^.ref^.index=taicpu(p).oper[1]^.reg then
  6202. begin
  6203. taicpu(hp1).oper[0]^.ref^.index:=taicpu(p).oper[0]^.ref^.base;
  6204. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset*max(taicpu(hp1).oper[0]^.ref^.scalefactor,1));
  6205. { if the register is used as index and base, we have to increase for base as well
  6206. and adapt base }
  6207. if taicpu(hp1).oper[0]^.ref^.base=taicpu(p).oper[1]^.reg then
  6208. begin
  6209. taicpu(hp1).oper[0]^.ref^.base:=taicpu(p).oper[0]^.ref^.base;
  6210. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset);
  6211. end;
  6212. end
  6213. else
  6214. begin
  6215. inc(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.ref^.offset);
  6216. taicpu(hp1).oper[0]^.ref^.base:=taicpu(p).oper[0]^.ref^.base;
  6217. end;
  6218. if taicpu(p).oper[0]^.ref^.index<>NR_NO then
  6219. begin
  6220. taicpu(hp1).oper[0]^.ref^.base:=taicpu(hp1).oper[0]^.ref^.index;
  6221. taicpu(hp1).oper[0]^.ref^.index:=taicpu(p).oper[0]^.ref^.index;
  6222. if (taicpu(p).oper[0]^.ref^.index = taicpu(p).oper[0]^.ref^.base) then
  6223. { Catch the situation where the base = index
  6224. and treat this as *2. The scalefactor of
  6225. p will be 0 or 1 due to the conditional
  6226. checks above. Fixes i40647 }
  6227. taicpu(hp1).oper[0]^.ref^.scalefactor := 2
  6228. else
  6229. taicpu(hp1).oper[0]^.ref^.scalefactor := taicpu(p).oper[0]^.ref^.scalefactor;
  6230. end;
  6231. { Only remove the first LEA if we don't need the intermediate register's value as is }
  6232. if IntermediateRegDiscarded then
  6233. begin
  6234. DebugMsg(SPeepholeOptimization + 'LeaLea2Lea 1 done',p);
  6235. RemoveCurrentP(p);
  6236. end
  6237. else
  6238. DebugMsg(SPeepholeOptimization + 'LeaLea2LeaLea 1 done (intermediate register still in use)',p);
  6239. result:=true;
  6240. exit;
  6241. end;
  6242. end;
  6243. end;
  6244. { Change:
  6245. leal/q $x(%reg1),%reg2
  6246. ...
  6247. shll/q $y,%reg2
  6248. To:
  6249. leal/q $(x+2^y)(%reg1,2^y),%reg2 (if y <= 3)
  6250. }
  6251. if (taicpu(p).oper[0]^.ref^.base<>NR_STACK_POINTER_REG) and { lea (%rsp,scale),reg is not a valid encoding }
  6252. MatchInstruction(hp1, A_SHL, [taicpu(p).opsize]) and
  6253. MatchOpType(taicpu(hp1), top_const, top_reg) and
  6254. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  6255. (taicpu(hp1).oper[0]^.val <= 3) then
  6256. begin
  6257. Multiple := 1 shl taicpu(hp1).oper[0]^.val;
  6258. TransferUsedRegs(TmpUsedRegs);
  6259. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  6260. if
  6261. { This allows the optimisation in some circumstances even if the lea instruction already has a scale factor
  6262. (this works even if scalefactor is zero) }
  6263. ((Multiple * taicpu(p).oper[0]^.ref^.scalefactor) <= 8) and
  6264. { Ensure offset doesn't go out of bounds }
  6265. (abs(taicpu(p).oper[0]^.ref^.offset * Multiple) <= $7FFFFFFF) and
  6266. not (RegInUsedRegs(NR_DEFAULTFLAGS,TmpUsedRegs)) and
  6267. (
  6268. (
  6269. not SuperRegistersEqual(taicpu(p).oper[0]^.ref^.base, taicpu(p).oper[1]^.reg) and
  6270. (
  6271. (taicpu(p).oper[0]^.ref^.index = NR_NO) or
  6272. (taicpu(p).oper[0]^.ref^.index = NR_INVALID) or
  6273. (
  6274. { Check for lea $x(%reg1,%reg1),%reg2 and treat as it it were lea $x(%reg1,2),%reg2 }
  6275. (taicpu(p).oper[0]^.ref^.index = taicpu(p).oper[0]^.ref^.base) and
  6276. (taicpu(p).oper[0]^.ref^.scalefactor <= 1)
  6277. )
  6278. )
  6279. ) or (
  6280. (
  6281. (taicpu(p).oper[0]^.ref^.base = NR_NO) or
  6282. (taicpu(p).oper[0]^.ref^.base = NR_INVALID)
  6283. ) and
  6284. not SuperRegistersEqual(taicpu(p).oper[0]^.ref^.index, taicpu(p).oper[1]^.reg)
  6285. )
  6286. ) then
  6287. begin
  6288. repeat
  6289. with taicpu(p).oper[0]^.ref^ do
  6290. begin
  6291. { Convert lea $x(%reg1,%reg1),%reg2 to lea $x(%reg1,2),%reg2 }
  6292. if index = base then
  6293. begin
  6294. if Multiple > 4 then
  6295. { Optimisation will no longer work because resultant
  6296. scale factor will exceed 8 }
  6297. Break;
  6298. base := NR_NO;
  6299. scalefactor := 2;
  6300. DebugMsg(SPeepholeOptimization + 'lea $x(%reg1,%reg1),%reg2 -> lea $x(%reg1,2),%reg2 for following optimisation', p);
  6301. end
  6302. else if (base <> NR_NO) and (base <> NR_INVALID) then
  6303. begin
  6304. { Scale factor only works on the index register }
  6305. index := base;
  6306. base := NR_NO;
  6307. end;
  6308. { For safety }
  6309. if scalefactor <= 1 then
  6310. begin
  6311. DebugMsg(SPeepholeOptimization + 'LeaShl2Lea 1', p);
  6312. scalefactor := Multiple;
  6313. end
  6314. else
  6315. begin
  6316. DebugMsg(SPeepholeOptimization + 'LeaShl2Lea 2', p);
  6317. scalefactor := scalefactor * Multiple;
  6318. end;
  6319. offset := offset * Multiple;
  6320. end;
  6321. RemoveInstruction(hp1);
  6322. Result := True;
  6323. Exit;
  6324. { This repeat..until loop exists for the benefit of Break }
  6325. until True;
  6326. end;
  6327. end;
  6328. end;
  6329. end;
  6330. end;
  6331. function TX86AsmOptimizer.DoArithCombineOpt(var p: tai): Boolean;
  6332. var
  6333. hp1 : tai;
  6334. SubInstr: Boolean;
  6335. ThisConst: TCGInt;
  6336. const
  6337. OverflowMin: array[S_B..S_Q] of TCGInt = (-128, -32768, -2147483648, -2147483648);
  6338. { Note: 64-bit-sized arithmetic instructions can only take signed 32-bit immediates }
  6339. OverflowMax: array[S_B..S_Q] of TCGInt = ( 255, 65535, $FFFFFFFF, 2147483647);
  6340. begin
  6341. Result := False;
  6342. if taicpu(p).oper[0]^.typ <> top_const then
  6343. { Should have been confirmed before calling }
  6344. InternalError(2021102601);
  6345. SubInstr := (taicpu(p).opcode = A_SUB);
  6346. if not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  6347. GetLastInstruction(p, hp1) and
  6348. (hp1.typ = ait_instruction) and
  6349. (taicpu(hp1).opsize = taicpu(p).opsize) then
  6350. begin
  6351. if not (taicpu(p).opsize in [S_B, S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  6352. { Bad size }
  6353. InternalError(2022042001);
  6354. case taicpu(hp1).opcode Of
  6355. A_INC:
  6356. if MatchOperand(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  6357. begin
  6358. if SubInstr then
  6359. ThisConst := taicpu(p).oper[0]^.val - 1
  6360. else
  6361. ThisConst := taicpu(p).oper[0]^.val + 1;
  6362. end
  6363. else
  6364. Exit;
  6365. A_DEC:
  6366. if MatchOperand(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) then
  6367. begin
  6368. if SubInstr then
  6369. ThisConst := taicpu(p).oper[0]^.val + 1
  6370. else
  6371. ThisConst := taicpu(p).oper[0]^.val - 1;
  6372. end
  6373. else
  6374. Exit;
  6375. A_SUB:
  6376. if (taicpu(hp1).oper[0]^.typ = top_const) and
  6377. MatchOperand(taicpu(hp1).oper[1]^,taicpu(p).oper[1]^) then
  6378. begin
  6379. if SubInstr then
  6380. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val
  6381. else
  6382. ThisConst := taicpu(p).oper[0]^.val - taicpu(hp1).oper[0]^.val;
  6383. end
  6384. else
  6385. Exit;
  6386. A_ADD:
  6387. if (taicpu(hp1).oper[0]^.typ = top_const) and
  6388. MatchOperand(taicpu(hp1).oper[1]^,taicpu(p).oper[1]^) then
  6389. begin
  6390. if SubInstr then
  6391. ThisConst := taicpu(p).oper[0]^.val - taicpu(hp1).oper[0]^.val
  6392. else
  6393. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val;
  6394. end
  6395. else
  6396. Exit;
  6397. else
  6398. Exit;
  6399. end;
  6400. { Check that the values are in range }
  6401. if (ThisConst < OverflowMin[taicpu(p).opsize]) or (ThisConst > OverflowMax[taicpu(p).opsize]) then
  6402. { Overflow; abort }
  6403. Exit;
  6404. if (ThisConst = 0) then
  6405. begin
  6406. DebugMsg(SPeepholeOptimization + 'Arithmetic combine: ' +
  6407. debug_op2str(taicpu(hp1).opcode) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + debug_operstr(taicpu(hp1).oper[1]^) + '; ' +
  6408. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(taicpu(p).oper[0]^.val) + ',' + debug_operstr(taicpu(p).oper[1]^) + ' cancel out (NOP)', p);
  6409. RemoveInstruction(hp1);
  6410. hp1 := tai(p.next);
  6411. RemoveInstruction(p); { Note, the choice to not use RemoveCurrentp is deliberate }
  6412. if not GetLastInstruction(hp1, p) then
  6413. p := hp1;
  6414. end
  6415. else
  6416. begin
  6417. if taicpu(hp1).opercnt=1 then
  6418. DebugMsg(SPeepholeOptimization + 'Arithmetic combine: ' +
  6419. debug_op2str(taicpu(hp1).opcode) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + '; ' +
  6420. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(taicpu(p).oper[0]^.val) + ',' + debug_operstr(taicpu(p).oper[1]^) + ' -> ' +
  6421. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(ThisConst) + ' ' + debug_operstr(taicpu(p).oper[1]^), p)
  6422. else
  6423. DebugMsg(SPeepholeOptimization + 'Arithmetic combine: ' +
  6424. debug_op2str(taicpu(hp1).opcode) + ' $' + debug_tostr(taicpu(hp1).oper[0]^.val) + ',' + debug_operstr(taicpu(hp1).oper[1]^) + '; ' +
  6425. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(taicpu(p).oper[0]^.val) + ',' + debug_operstr(taicpu(p).oper[1]^) + ' -> ' +
  6426. debug_op2str(taicpu(p).opcode) + ' $' + debug_tostr(ThisConst) + ' ' + debug_operstr(taicpu(p).oper[1]^), p);
  6427. RemoveInstruction(hp1);
  6428. taicpu(p).loadconst(0, ThisConst);
  6429. end;
  6430. Result := True;
  6431. end;
  6432. end;
  6433. function TX86AsmOptimizer.DoMovCmpMemOpt(var p : tai; const hp1: tai) : Boolean;
  6434. begin
  6435. Result := False;
  6436. if MatchOpType(taicpu(p),top_ref,top_reg) and
  6437. { The x86 assemblers have difficulty comparing values against absolute addresses }
  6438. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) and
  6439. (taicpu(hp1).oper[0]^.typ <> top_ref) and
  6440. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  6441. (
  6442. (
  6443. (taicpu(hp1).opcode = A_TEST)
  6444. ) or (
  6445. (taicpu(hp1).opcode = A_CMP) and
  6446. { A sanity check more than anything }
  6447. not MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg)
  6448. )
  6449. ) then
  6450. begin
  6451. { change
  6452. mov mem, %reg
  6453. ...
  6454. cmp/test x, %reg / test %reg,%reg
  6455. (reg deallocated)
  6456. to
  6457. cmp/test x, mem / cmp 0, mem
  6458. }
  6459. TransferUsedRegs(TmpUsedRegs);
  6460. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  6461. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) then
  6462. begin
  6463. { Convert test %reg,%reg or test $-1,%reg to cmp $0,mem }
  6464. if (taicpu(hp1).opcode = A_TEST) and
  6465. (
  6466. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) or
  6467. MatchOperand(taicpu(hp1).oper[0]^, -1)
  6468. ) then
  6469. begin
  6470. taicpu(hp1).opcode := A_CMP;
  6471. taicpu(hp1).loadconst(0, 0);
  6472. end;
  6473. taicpu(hp1).loadref(1, taicpu(p).oper[0]^.ref^);
  6474. DebugMsg(SPeepholeOptimization + 'MOV/CMP -> CMP (memory check)', p);
  6475. RemoveCurrentP(p);
  6476. if (p <> hp1) then
  6477. { Correctly update TmpUsedRegs if p and hp1 aren't adjacent }
  6478. UpdateUsedRegsBetween(TmpUsedRegs, p, hp1);
  6479. { Make sure the flags are allocated across the CMP instruction }
  6480. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  6481. AllocRegBetween(NR_DEFAULTFLAGS, hp1, hp1, TmpUsedRegs);
  6482. Result := True;
  6483. Exit;
  6484. end;
  6485. end;
  6486. end;
  6487. function TX86AsmOptimizer.DoSETccLblRETOpt(var p: tai; const hp_label: tai_label) : Boolean;
  6488. var
  6489. hp_allocstart, hp_pos, hp2, hp3, hp4, hp5, hp6: tai;
  6490. ThisReg, SecondReg: TRegister;
  6491. JumpLoc: TAsmLabel;
  6492. NewSize: TOpSize;
  6493. begin
  6494. Result := False;
  6495. {
  6496. Convert:
  6497. j<c> .L1
  6498. .L2:
  6499. mov 1,reg
  6500. jmp .L3 (or ret, although it might not be a RET yet)
  6501. .L1:
  6502. mov 0,reg
  6503. jmp .L3 (or ret)
  6504. ( As long as .L3 <> .L1 or .L2)
  6505. To:
  6506. mov 0,reg
  6507. set<not(c)> reg
  6508. jmp .L3 (or ret)
  6509. .L2:
  6510. mov 1,reg
  6511. jmp .L3 (or ret)
  6512. .L1:
  6513. mov 0,reg
  6514. jmp .L3 (or ret)
  6515. }
  6516. if JumpTargetOp(taicpu(p))^.ref^.refaddr<>addr_full then
  6517. Exit;
  6518. JumpLoc := TAsmLabel(JumpTargetOp(taicpu(p))^.ref^.symbol);
  6519. if GetNextInstruction(hp_label, hp2) and
  6520. MatchInstruction(hp2,A_MOV,[]) and
  6521. (taicpu(hp2).oper[0]^.typ = top_const) and
  6522. (
  6523. (
  6524. (taicpu(hp2).oper[1]^.typ = top_reg)
  6525. {$ifdef i386}
  6526. { Under i386, ESI, EDI, EBP and ESP
  6527. don't have an 8-bit representation }
  6528. and not (getsupreg(taicpu(hp2).oper[1]^.reg) in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  6529. {$endif i386}
  6530. ) or (
  6531. {$ifdef i386}
  6532. (taicpu(hp2).oper[1]^.typ <> top_reg) and
  6533. {$endif i386}
  6534. (taicpu(hp2).opsize = S_B)
  6535. )
  6536. ) and
  6537. GetNextInstruction(hp2, hp3) and
  6538. MatchInstruction(hp3, A_JMP, A_RET, []) and
  6539. (
  6540. (taicpu(hp3).opcode=A_RET) or
  6541. (
  6542. (taicpu(hp3).oper[0]^.ref^.refaddr=addr_full) and
  6543. (tasmlabel(taicpu(hp3).oper[0]^.ref^.symbol)<>tai_label(hp_label).labsym)
  6544. )
  6545. ) and
  6546. GetNextInstruction(hp3, hp4) and
  6547. FindLabel(JumpLoc, hp4) and
  6548. (
  6549. not (cs_opt_size in current_settings.optimizerswitches) or
  6550. { If the initial jump is the label's only reference, then it will
  6551. become a dead label if the other conditions are met and hence
  6552. remove at least 2 instructions, including a jump }
  6553. (JumpLoc.getrefs = 1)
  6554. ) and
  6555. { Don't check if hp3 jumps to hp4 because this is a zero-distance jump
  6556. that will be optimised out }
  6557. GetNextInstruction(hp4, hp5) and
  6558. MatchInstruction(hp5,A_MOV,[taicpu(hp2).opsize]) and
  6559. (taicpu(hp5).oper[0]^.typ = top_const) and
  6560. (
  6561. ((taicpu(hp2).oper[0]^.val = 0) and (taicpu(hp5).oper[0]^.val = 1)) or
  6562. ((taicpu(hp2).oper[0]^.val = 1) and (taicpu(hp5).oper[0]^.val = 0))
  6563. ) and
  6564. MatchOperand(taicpu(hp2).oper[1]^,taicpu(hp5).oper[1]^) and
  6565. GetNextInstruction(hp5,hp6) and
  6566. (
  6567. not (hp6.typ in [ait_align, ait_label]) or
  6568. SkipLabels(hp6, hp6)
  6569. ) and
  6570. (hp6.typ=ait_instruction) then
  6571. begin
  6572. { First, let's look at the two jumps that are hp3 and hp6 }
  6573. if not
  6574. (
  6575. (taicpu(hp6).opcode=taicpu(hp3).opcode) and { Both RET or both JMP to the same label }
  6576. (
  6577. (taicpu(hp6).opcode=A_RET) or
  6578. MatchOperand(taicpu(hp6).oper[0]^, taicpu(hp3).oper[0]^)
  6579. )
  6580. ) then
  6581. { If condition is False, then the JMP/RET instructions matched conventionally }
  6582. begin
  6583. { See if one of the jumps can be instantly converted into a RET }
  6584. if (taicpu(hp3).opcode=A_JMP) then
  6585. begin
  6586. { Reuse hp5 }
  6587. hp5 := getlabelwithsym(TAsmLabel(JumpTargetOp(taicpu(hp3))^.ref^.symbol));
  6588. { Make sure hp5 doesn't jump back to .L1 (zero distance jump) or .L2 (infinite loop) }
  6589. if not Assigned(hp5) or (hp5 = hp_label) or (hp5 = hp4) or not GetNextInstruction(hp5, hp5) then
  6590. Exit;
  6591. if MatchInstruction(hp5, A_RET, []) then
  6592. begin
  6593. DebugMsg(SPeepholeOptimization + 'Converted JMP to RET as part of SETcc optimisation (1st jump)', hp3);
  6594. ConvertJumpToRET(hp3, hp5);
  6595. Result := True;
  6596. end
  6597. else
  6598. Exit;
  6599. end;
  6600. if (taicpu(hp6).opcode=A_JMP) then
  6601. begin
  6602. { Reuse hp5 }
  6603. hp5 := getlabelwithsym(TAsmLabel(JumpTargetOp(taicpu(hp6))^.ref^.symbol));
  6604. if not Assigned(hp5) or not GetNextInstruction(hp5, hp5) then
  6605. Exit;
  6606. if MatchInstruction(hp5, A_RET, []) then
  6607. begin
  6608. DebugMsg(SPeepholeOptimization + 'Converted JMP to RET as part of SETcc optimisation (2nd jump)', hp6);
  6609. ConvertJumpToRET(hp6, hp5);
  6610. Result := True;
  6611. end
  6612. else
  6613. Exit;
  6614. end;
  6615. if not
  6616. (
  6617. (taicpu(hp6).opcode=taicpu(hp3).opcode) and { Both RET or both JMP to the same label }
  6618. (
  6619. (taicpu(hp6).opcode=A_RET) or
  6620. MatchOperand(taicpu(hp6).oper[0]^, taicpu(hp3).oper[0]^)
  6621. )
  6622. ) then
  6623. { Still doesn't match }
  6624. Exit;
  6625. end;
  6626. if (taicpu(hp2).oper[0]^.val = 1) then
  6627. begin
  6628. taicpu(p).condition := inverse_cond(taicpu(p).condition);
  6629. DebugMsg(SPeepholeOptimization + 'J(c)Mov1Jmp/RetMov0Jmp/Ret -> Set(~c)Jmp/Ret',p)
  6630. end
  6631. else
  6632. DebugMsg(SPeepholeOptimization + 'J(c)Mov0Jmp/RetMov1Jmp/Ret -> Set(c)Jmp/Ret',p);
  6633. if taicpu(hp2).opsize=S_B then
  6634. begin
  6635. if taicpu(hp2).oper[1]^.typ = top_reg then
  6636. begin
  6637. SecondReg := taicpu(hp2).oper[1]^.reg;
  6638. hp4:=taicpu.op_reg(A_SETcc, S_B, SecondReg);
  6639. end
  6640. else
  6641. begin
  6642. hp4:=taicpu.op_ref(A_SETcc, S_B, taicpu(hp2).oper[1]^.ref^);
  6643. SecondReg := NR_NO;
  6644. end;
  6645. hp_pos := p;
  6646. hp_allocstart := hp4;
  6647. end
  6648. else
  6649. begin
  6650. { Will be a register because the size can't be S_B otherwise }
  6651. SecondReg:=taicpu(hp2).oper[1]^.reg;
  6652. ThisReg:=newreg(R_INTREGISTER,getsupreg(SecondReg), R_SUBL);
  6653. hp4:=taicpu.op_reg(A_SETcc, S_B, ThisReg);
  6654. if (cs_opt_size in current_settings.optimizerswitches) then
  6655. begin
  6656. { Favour using MOVZX when optimising for size }
  6657. case taicpu(hp2).opsize of
  6658. S_W:
  6659. NewSize := S_BW;
  6660. S_L:
  6661. NewSize := S_BL;
  6662. {$ifdef x86_64}
  6663. S_Q:
  6664. begin
  6665. NewSize := S_BL;
  6666. { Will implicitly zero-extend to 64-bit }
  6667. setsubreg(SecondReg, R_SUBD);
  6668. end;
  6669. {$endif x86_64}
  6670. else
  6671. InternalError(2022101301);
  6672. end;
  6673. hp5:=taicpu.op_reg_reg(A_MOVZX, NewSize, ThisReg, SecondReg);
  6674. { Inserting it right before p will guarantee that the flags are also tracked }
  6675. Asml.InsertBefore(hp5, p);
  6676. { Make sure the SET instruction gets inserted before the MOVZX instruction }
  6677. hp_pos := hp5;
  6678. hp_allocstart := hp4;
  6679. end
  6680. else
  6681. begin
  6682. hp5:=taicpu.op_const_reg(A_MOV, taicpu(hp2).opsize, 0, SecondReg);
  6683. { Inserting it right before p will guarantee that the flags are also tracked }
  6684. Asml.InsertBefore(hp5, p);
  6685. hp_pos := p;
  6686. hp_allocstart := hp5;
  6687. end;
  6688. taicpu(hp5).fileinfo:=taicpu(p).fileinfo;
  6689. end;
  6690. taicpu(hp4).fileinfo := taicpu(p).fileinfo;
  6691. taicpu(hp4).condition := taicpu(p).condition;
  6692. asml.InsertBefore(hp4, hp_pos);
  6693. if taicpu(hp3).is_jmp then
  6694. begin
  6695. JumpLoc.decrefs;
  6696. MakeUnconditional(taicpu(p));
  6697. { This also increases the reference count }
  6698. taicpu(p).loadref(0, JumpTargetOp(taicpu(hp3))^.ref^);
  6699. end
  6700. else
  6701. ConvertJumpToRET(p, hp3);
  6702. if SecondReg <> NR_NO then
  6703. { Ensure the destination register is allocated over this region }
  6704. AllocRegBetween(SecondReg, hp_allocstart, p, UsedRegs);
  6705. if (JumpLoc.getrefs = 0) then
  6706. RemoveDeadCodeAfterJump(hp3);
  6707. Result:=true;
  6708. exit;
  6709. end;
  6710. end;
  6711. function TX86AsmOptimizer.OptPass1Sub(var p : tai) : boolean;
  6712. var
  6713. hp1, hp2: tai;
  6714. ActiveReg: TRegister;
  6715. OldOffset: asizeint;
  6716. ThisConst: TCGInt;
  6717. function RegDeallocated: Boolean;
  6718. begin
  6719. TransferUsedRegs(TmpUsedRegs);
  6720. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  6721. Result := not(RegUsedAfterInstruction(ActiveReg,hp1,TmpUsedRegs))
  6722. end;
  6723. begin
  6724. Result:=false;
  6725. hp1 := nil;
  6726. { replace
  6727. subX const,%reg1
  6728. leaX (%reg1,%reg1,Y),%reg2 // Base or index might not be equal to reg1
  6729. dealloc %reg1
  6730. by
  6731. leaX -const-const*Y(%reg1,%reg1,Y),%reg2
  6732. }
  6733. if MatchOpType(taicpu(p),top_const,top_reg) then
  6734. begin
  6735. ActiveReg := taicpu(p).oper[1]^.reg;
  6736. { Ensures the entire register was updated }
  6737. if (taicpu(p).opsize >= S_L) and
  6738. GetNextInstructionUsingReg(p,hp1, ActiveReg) and
  6739. MatchInstruction(hp1,A_LEA,[]) and
  6740. (SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.base) or
  6741. SuperRegistersEqual(ActiveReg, taicpu(hp1).oper[0]^.ref^.index)) and
  6742. (
  6743. { Cover the case where the register in the reference is also the destination register }
  6744. Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, ActiveReg) or
  6745. (
  6746. { Try to avoid the expensive check of RegUsedAfterInstruction if we know it will return False }
  6747. not SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ActiveReg) and
  6748. RegDeallocated
  6749. )
  6750. ) then
  6751. begin
  6752. OldOffset := taicpu(hp1).oper[0]^.ref^.offset;
  6753. if SuperRegistersEqual(ActiveReg,taicpu(hp1).oper[0]^.ref^.base) then
  6754. Dec(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val);
  6755. if SuperRegistersEqual(ActiveReg,taicpu(hp1).oper[0]^.ref^.index) then
  6756. Dec(taicpu(hp1).oper[0]^.ref^.offset,taicpu(p).oper[0]^.val*max(taicpu(hp1).oper[0]^.ref^.scalefactor,1));
  6757. {$ifdef x86_64}
  6758. if (taicpu(hp1).oper[0]^.ref^.offset > $7FFFFFFF) or (taicpu(hp1).oper[0]^.ref^.offset < -2147483648) then
  6759. begin
  6760. { Overflow; abort }
  6761. taicpu(hp1).oper[0]^.ref^.offset := OldOffset;
  6762. end
  6763. else
  6764. {$endif x86_64}
  6765. begin
  6766. DebugMsg(SPeepholeOptimization + 'SubLea2Lea done',p);
  6767. if not (cs_opt_level3 in current_settings.optimizerswitches) then
  6768. { hp1 is the immediate next instruction for sure - good for a quick speed boost }
  6769. RemoveCurrentP(p, hp1)
  6770. else
  6771. RemoveCurrentP(p);
  6772. result:=true;
  6773. Exit;
  6774. end;
  6775. end;
  6776. if (
  6777. { Save calling GetNextInstructionUsingReg again }
  6778. Assigned(hp1) or
  6779. GetNextInstructionUsingReg(p,hp1, ActiveReg)
  6780. ) and
  6781. MatchInstruction(hp1,A_SUB,[taicpu(p).opsize]) and
  6782. (taicpu(hp1).oper[1]^.reg = ActiveReg) then
  6783. begin
  6784. { Make sure the flags aren't in use by the second operation }
  6785. TransferUsedRegs(TmpUsedRegs);
  6786. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.next), hp1);
  6787. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  6788. begin
  6789. if (taicpu(hp1).oper[0]^.typ = top_const) then
  6790. begin
  6791. { Merge add const1,%reg; add const2,%reg to add const1+const2,%reg }
  6792. ThisConst := taicpu(p).oper[0]^.val + taicpu(hp1).oper[0]^.val;
  6793. Result := True;
  6794. { Handle any overflows }
  6795. case taicpu(p).opsize of
  6796. S_B:
  6797. taicpu(p).oper[0]^.val := ThisConst and $FF;
  6798. S_W:
  6799. taicpu(p).oper[0]^.val := ThisConst and $FFFF;
  6800. S_L:
  6801. taicpu(p).oper[0]^.val := ThisConst and $FFFFFFFF;
  6802. {$ifdef x86_64}
  6803. S_Q:
  6804. if (ThisConst > $7FFFFFFF) or (ThisConst < -2147483648) then
  6805. { Overflow; abort }
  6806. Result := False
  6807. else
  6808. taicpu(p).oper[0]^.val := ThisConst;
  6809. {$endif x86_64}
  6810. else
  6811. InternalError(2021102611);
  6812. end;
  6813. { Result may get set to False again if the combined immediate overflows for S_Q sizes }
  6814. if Result then
  6815. begin
  6816. if (taicpu(p).oper[0]^.val < 0) and
  6817. (
  6818. ((taicpu(p).opsize = S_B) and (taicpu(p).oper[0]^.val <> -128)) or
  6819. ((taicpu(p).opsize = S_W) and (taicpu(p).oper[0]^.val <> -32768)) or
  6820. ((taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) and (taicpu(p).oper[0]^.val <> -2147483648))
  6821. ) then
  6822. begin
  6823. DebugMsg(SPeepholeOptimization + 'SUB; ADD/SUB -> ADD',p);
  6824. taicpu(p).opcode := A_SUB;
  6825. taicpu(p).oper[0]^.val := -taicpu(p).oper[0]^.val;
  6826. end
  6827. else
  6828. DebugMsg(SPeepholeOptimization + 'SUB; ADD/SUB -> SUB',p);
  6829. RemoveInstruction(hp1);
  6830. end;
  6831. end
  6832. else
  6833. begin
  6834. { Move the constant subtraction to after the reg/ref addition to improve optimisation }
  6835. DebugMsg(SPeepholeOptimization + 'Add/sub swap 1b done',p);
  6836. Asml.Remove(p);
  6837. Asml.InsertAfter(p, hp1);
  6838. p := hp1;
  6839. Result := True;
  6840. Exit;
  6841. end;
  6842. end;
  6843. end;
  6844. { * change "subl $2, %esp; pushw x" to "pushl x"}
  6845. { * change "sub/add const1, reg" or "dec reg" followed by
  6846. "sub const2, reg" to one "sub ..., reg" }
  6847. {$ifdef i386}
  6848. if (taicpu(p).oper[0]^.val = 2) and
  6849. (ActiveReg = NR_ESP) and
  6850. { Don't do the sub/push optimization if the sub }
  6851. { comes from setting up the stack frame (JM) }
  6852. (not(GetLastInstruction(p,hp1)) or
  6853. not(MatchInstruction(hp1,A_MOV,[S_L]) and
  6854. MatchOperand(taicpu(hp1).oper[0]^,NR_ESP) and
  6855. MatchOperand(taicpu(hp1).oper[0]^,NR_EBP))) then
  6856. begin
  6857. hp1 := tai(p.next);
  6858. while Assigned(hp1) and
  6859. (tai(hp1).typ in [ait_instruction]+SkipInstr) and
  6860. not RegReadByInstruction(NR_ESP,hp1) and
  6861. not RegModifiedByInstruction(NR_ESP,hp1) do
  6862. hp1 := tai(hp1.next);
  6863. if Assigned(hp1) and
  6864. MatchInstruction(hp1,A_PUSH,[S_W]) then
  6865. begin
  6866. taicpu(hp1).changeopsize(S_L);
  6867. if taicpu(hp1).oper[0]^.typ=top_reg then
  6868. setsubreg(taicpu(hp1).oper[0]^.reg,R_SUBWHOLE);
  6869. hp1 := tai(p.next);
  6870. RemoveCurrentp(p, hp1);
  6871. Result:=true;
  6872. exit;
  6873. end;
  6874. end;
  6875. {$endif i386}
  6876. if DoArithCombineOpt(p) then
  6877. Result:=true;
  6878. end;
  6879. end;
  6880. function TX86AsmOptimizer.OptPass1SHLSAL(var p : tai) : boolean;
  6881. var
  6882. TmpBool1,TmpBool2 : Boolean;
  6883. tmpref : treference;
  6884. hp1,hp2: tai;
  6885. mask, shiftval: tcgint;
  6886. begin
  6887. Result:=false;
  6888. { All these optimisations work on "shl/sal const,%reg" }
  6889. if not MatchOpType(taicpu(p),top_const,top_reg) then
  6890. Exit;
  6891. if (taicpu(p).opsize in [S_L{$ifdef x86_64},S_Q{$endif x86_64}]) and
  6892. (taicpu(p).oper[0]^.val <= 3) then
  6893. { Changes "shl const, %reg32; add const/reg, %reg32" to one lea statement }
  6894. begin
  6895. { should we check the next instruction? }
  6896. TmpBool1 := True;
  6897. { have we found an add/sub which could be
  6898. integrated in the lea? }
  6899. TmpBool2 := False;
  6900. reference_reset(tmpref,2,[]);
  6901. TmpRef.index := taicpu(p).oper[1]^.reg;
  6902. TmpRef.scalefactor := 1 shl taicpu(p).oper[0]^.val;
  6903. while TmpBool1 and
  6904. GetNextInstruction(p, hp1) and
  6905. (tai(hp1).typ = ait_instruction) and
  6906. ((((taicpu(hp1).opcode = A_ADD) or
  6907. (taicpu(hp1).opcode = A_SUB)) and
  6908. (taicpu(hp1).oper[1]^.typ = Top_Reg) and
  6909. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg)) or
  6910. (((taicpu(hp1).opcode = A_INC) or
  6911. (taicpu(hp1).opcode = A_DEC)) and
  6912. (taicpu(hp1).oper[0]^.typ = Top_Reg) and
  6913. (taicpu(hp1).oper[0]^.reg = taicpu(p).oper[1]^.reg)) or
  6914. ((taicpu(hp1).opcode = A_LEA) and
  6915. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  6916. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg))) and
  6917. (not GetNextInstruction(hp1,hp2) or
  6918. not instrReadsFlags(hp2)) Do
  6919. begin
  6920. TmpBool1 := False;
  6921. if taicpu(hp1).opcode=A_LEA then
  6922. begin
  6923. if (TmpRef.base = NR_NO) and
  6924. (taicpu(hp1).oper[0]^.ref^.symbol=nil) and
  6925. (taicpu(hp1).oper[0]^.ref^.relsymbol=nil) and
  6926. { Segment register isn't a concern here }
  6927. ((taicpu(hp1).oper[0]^.ref^.scalefactor=0) or
  6928. (taicpu(hp1).oper[0]^.ref^.scalefactor*tmpref.scalefactor<=8)) then
  6929. begin
  6930. TmpBool1 := True;
  6931. TmpBool2 := True;
  6932. inc(TmpRef.offset, taicpu(hp1).oper[0]^.ref^.offset);
  6933. if taicpu(hp1).oper[0]^.ref^.scalefactor<>0 then
  6934. tmpref.scalefactor:=tmpref.scalefactor*taicpu(hp1).oper[0]^.ref^.scalefactor;
  6935. TmpRef.base := taicpu(hp1).oper[0]^.ref^.base;
  6936. RemoveInstruction(hp1);
  6937. end
  6938. end
  6939. else if (taicpu(hp1).oper[0]^.typ = Top_Const) then
  6940. begin
  6941. TmpBool1 := True;
  6942. TmpBool2 := True;
  6943. case taicpu(hp1).opcode of
  6944. A_ADD:
  6945. inc(TmpRef.offset, longint(taicpu(hp1).oper[0]^.val));
  6946. A_SUB:
  6947. dec(TmpRef.offset, longint(taicpu(hp1).oper[0]^.val));
  6948. else
  6949. internalerror(2019050536);
  6950. end;
  6951. RemoveInstruction(hp1);
  6952. end
  6953. else
  6954. if (taicpu(hp1).oper[0]^.typ = Top_Reg) and
  6955. (((taicpu(hp1).opcode = A_ADD) and
  6956. (TmpRef.base = NR_NO)) or
  6957. (taicpu(hp1).opcode = A_INC) or
  6958. (taicpu(hp1).opcode = A_DEC)) then
  6959. begin
  6960. TmpBool1 := True;
  6961. TmpBool2 := True;
  6962. case taicpu(hp1).opcode of
  6963. A_ADD:
  6964. TmpRef.base := taicpu(hp1).oper[0]^.reg;
  6965. A_INC:
  6966. inc(TmpRef.offset);
  6967. A_DEC:
  6968. dec(TmpRef.offset);
  6969. else
  6970. internalerror(2019050535);
  6971. end;
  6972. RemoveInstruction(hp1);
  6973. end;
  6974. end;
  6975. if TmpBool2
  6976. {$ifndef x86_64}
  6977. or
  6978. ((current_settings.optimizecputype < cpu_Pentium2) and
  6979. (taicpu(p).oper[0]^.val <= 3) and
  6980. not(cs_opt_size in current_settings.optimizerswitches))
  6981. {$endif x86_64}
  6982. then
  6983. begin
  6984. if not(TmpBool2) and
  6985. (taicpu(p).oper[0]^.val=1) then
  6986. begin
  6987. taicpu(p).opcode := A_ADD;
  6988. taicpu(p).loadreg(0, taicpu(p).oper[1]^.reg);
  6989. end
  6990. else
  6991. begin
  6992. taicpu(p).opcode := A_LEA;
  6993. taicpu(p).loadref(0, TmpRef);
  6994. end;
  6995. DebugMsg(SPeepholeOptimization + 'ShlAddLeaSubIncDec2Lea',p);
  6996. Result := True;
  6997. end;
  6998. end
  6999. {$ifndef x86_64}
  7000. else if (current_settings.optimizecputype < cpu_Pentium2) then
  7001. begin
  7002. { changes "shl $1, %reg" to "add %reg, %reg", which is the same on a 386,
  7003. but faster on a 486, and Tairable in both U and V pipes on the Pentium
  7004. (unlike shl, which is only Tairable in the U pipe) }
  7005. if taicpu(p).oper[0]^.val=1 then
  7006. begin
  7007. taicpu(p).opcode := A_ADD;
  7008. taicpu(p).loadreg(0, taicpu(p).oper[1]^.reg);
  7009. Result := True;
  7010. end
  7011. { changes "shl $2, %reg" to "lea (,%reg,4), %reg"
  7012. "shl $3, %reg" to "lea (,%reg,8), %reg }
  7013. else if (taicpu(p).opsize = S_L) and
  7014. (taicpu(p).oper[0]^.val<= 3) then
  7015. begin
  7016. reference_reset(tmpref,2,[]);
  7017. TmpRef.index := taicpu(p).oper[1]^.reg;
  7018. TmpRef.scalefactor := 1 shl taicpu(p).oper[0]^.val;
  7019. taicpu(p).opcode := A_LEA;
  7020. taicpu(p).loadref(0, TmpRef);
  7021. Result := True;
  7022. end;
  7023. end
  7024. {$endif x86_64}
  7025. else if
  7026. GetNextInstruction(p, hp1) and (hp1.typ = ait_instruction) and MatchOpType(taicpu(hp1), top_const, top_reg) and
  7027. (
  7028. (
  7029. MatchInstruction(hp1, A_AND, [taicpu(p).opsize]) and
  7030. SetAndTest(hp1, hp2)
  7031. {$ifdef x86_64}
  7032. ) or
  7033. (
  7034. MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  7035. GetNextInstruction(hp1, hp2) and
  7036. MatchInstruction(hp2, A_AND, [taicpu(p).opsize]) and
  7037. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  7038. (taicpu(hp1).oper[1]^.reg = taicpu(hp2).oper[0]^.reg)
  7039. {$endif x86_64}
  7040. )
  7041. ) and
  7042. (taicpu(p).oper[1]^.reg = taicpu(hp2).oper[1]^.reg) then
  7043. begin
  7044. { Change:
  7045. shl x, %reg1
  7046. mov -(1<<x), %reg2
  7047. and %reg2, %reg1
  7048. Or:
  7049. shl x, %reg1
  7050. and -(1<<x), %reg1
  7051. To just:
  7052. shl x, %reg1
  7053. Since the and operation only zeroes bits that are already zero from the shl operation
  7054. }
  7055. case taicpu(p).oper[0]^.val of
  7056. 8:
  7057. mask:=$FFFFFFFFFFFFFF00;
  7058. 16:
  7059. mask:=$FFFFFFFFFFFF0000;
  7060. 32:
  7061. mask:=$FFFFFFFF00000000;
  7062. 63:
  7063. { Constant pre-calculated to prevent overflow errors with Int64 }
  7064. mask:=$8000000000000000;
  7065. else
  7066. begin
  7067. if taicpu(p).oper[0]^.val >= 64 then
  7068. { Shouldn't happen realistically, since the register
  7069. is guaranteed to be set to zero at this point }
  7070. mask := 0
  7071. else
  7072. mask := -(Int64(1 shl taicpu(p).oper[0]^.val));
  7073. end;
  7074. end;
  7075. if taicpu(hp1).oper[0]^.val = mask then
  7076. begin
  7077. { Everything checks out, perform the optimisation, as long as
  7078. the FLAGS register isn't being used}
  7079. TransferUsedRegs(TmpUsedRegs);
  7080. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  7081. {$ifdef x86_64}
  7082. if (hp1 <> hp2) then
  7083. begin
  7084. { "shl/mov/and" version }
  7085. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  7086. { Don't do the optimisation if the FLAGS register is in use }
  7087. if not(RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp2, TmpUsedRegs)) then
  7088. begin
  7089. DebugMsg(SPeepholeOptimization + 'ShlMovAnd2Shl', p);
  7090. { Don't remove the 'mov' instruction if its register is used elsewhere }
  7091. if not(RegUsedAfterInstruction(taicpu(hp1).oper[1]^.reg, hp2, TmpUsedRegs)) then
  7092. begin
  7093. RemoveInstruction(hp1);
  7094. Result := True;
  7095. end;
  7096. { Only set Result to True if the 'mov' instruction was removed }
  7097. RemoveInstruction(hp2);
  7098. end;
  7099. end
  7100. else
  7101. {$endif x86_64}
  7102. begin
  7103. { "shl/and" version }
  7104. { Don't do the optimisation if the FLAGS register is in use }
  7105. if not(RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)) then
  7106. begin
  7107. DebugMsg(SPeepholeOptimization + 'ShlAnd2Shl', p);
  7108. RemoveInstruction(hp1);
  7109. Result := True;
  7110. end;
  7111. end;
  7112. Exit;
  7113. end
  7114. else {$ifdef x86_64}if (hp1 = hp2) then{$endif x86_64}
  7115. begin
  7116. { Even if the mask doesn't allow for its removal, we might be
  7117. able to optimise the mask for the "shl/and" version, which
  7118. may permit other peephole optimisations }
  7119. {$ifdef DEBUG_AOPTCPU}
  7120. mask := taicpu(hp1).oper[0]^.val and mask;
  7121. if taicpu(hp1).oper[0]^.val <> mask then
  7122. begin
  7123. DebugMsg(
  7124. SPeepholeOptimization +
  7125. 'Changed mask from $' + debug_tostr(taicpu(hp1).oper[0]^.val) +
  7126. ' to $' + debug_tostr(mask) +
  7127. 'based on previous instruction (ShlAnd2ShlAnd)', hp1);
  7128. taicpu(hp1).oper[0]^.val := mask;
  7129. end;
  7130. {$else DEBUG_AOPTCPU}
  7131. { If debugging is off, just set the operand even if it's the same }
  7132. taicpu(hp1).oper[0]^.val := taicpu(hp1).oper[0]^.val and mask;
  7133. {$endif DEBUG_AOPTCPU}
  7134. end;
  7135. end;
  7136. {
  7137. change
  7138. shl/sal const,reg
  7139. <op> ...(...,reg,1),...
  7140. into
  7141. <op> ...(...,reg,1 shl const),...
  7142. if const in 1..3
  7143. }
  7144. if MatchOpType(taicpu(p), top_const, top_reg) and
  7145. (taicpu(p).oper[0]^.val in [1..3]) and
  7146. GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg) and
  7147. ((MatchInstruction(hp1,A_MOV,A_LEA,[]) and
  7148. MatchOpType(taicpu(hp1),top_ref,top_reg)) or
  7149. (MatchInstruction(hp1,A_FST,A_FSTP,A_FLD,[]) and
  7150. MatchOpType(taicpu(hp1),top_ref))
  7151. ) and
  7152. (taicpu(p).oper[1]^.reg=taicpu(hp1).oper[0]^.ref^.index) and
  7153. (taicpu(p).oper[1]^.reg<>taicpu(hp1).oper[0]^.ref^.base) and
  7154. (taicpu(hp1).oper[0]^.ref^.scalefactor in [0,1]) then
  7155. begin
  7156. TransferUsedRegs(TmpUsedRegs);
  7157. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  7158. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs)) then
  7159. begin
  7160. taicpu(hp1).oper[0]^.ref^.scalefactor:=1 shl taicpu(p).oper[0]^.val;
  7161. DebugMsg(SPeepholeOptimization + 'ShlOp2Op', p);
  7162. RemoveCurrentP(p);
  7163. Result:=true;
  7164. exit;
  7165. end;
  7166. end;
  7167. if MatchOpType(taicpu(p), top_const, top_reg) and
  7168. GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[1]^.reg) and
  7169. MatchInstruction(hp1,A_SHL,[taicpu(p).opsize]) and
  7170. MatchOpType(taicpu(hp1),top_const,top_reg) and
  7171. (taicpu(p).oper[1]^.reg=taicpu(hp1).oper[1]^.reg) then
  7172. begin
  7173. shiftval:=taicpu(p).oper[0]^.val+taicpu(hp1).oper[0]^.val;
  7174. if ((taicpu(p).opsize=S_B) and (shiftval>7)) or
  7175. ((taicpu(p).opsize=S_W) and (shiftval>15)) or
  7176. {$ifdef x86_64}
  7177. ((taicpu(p).opsize=S_Q) and (shiftval>63)) or
  7178. {$endif x86_64}
  7179. ((taicpu(p).opsize=S_L) and (shiftval>31)) then
  7180. begin
  7181. DebugMsg(SPeepholeOptimization + 'ShlShl2Mov', p);
  7182. taicpu(hp1).opcode:=A_MOV;
  7183. taicpu(hp1).oper[0]^.val:=0;
  7184. end
  7185. else
  7186. begin
  7187. DebugMsg(SPeepholeOptimization + 'ShlShl2Shl', p);
  7188. taicpu(hp1).oper[0]^.val:=shiftval;
  7189. end;
  7190. RemoveCurrentP(p);
  7191. Result:=true;
  7192. exit;
  7193. end;
  7194. end;
  7195. class function TX86AsmOptimizer.IsShrMovZFoldable(shr_size, movz_size: topsize; Shift: TCGInt): Boolean;
  7196. begin
  7197. case shr_size of
  7198. S_B:
  7199. { No valid combinations }
  7200. Result := False;
  7201. S_W:
  7202. Result := (Shift >= 8) and (movz_size = S_BW);
  7203. S_L:
  7204. Result :=
  7205. (Shift >= 24) { Any opsize is valid for this shift } or
  7206. ((Shift >= 16) and (movz_size = S_WL));
  7207. {$ifdef x86_64}
  7208. S_Q:
  7209. Result :=
  7210. (Shift >= 56) { Any opsize is valid for this shift } or
  7211. ((Shift >= 48) and (movz_size = S_WL));
  7212. {$endif x86_64}
  7213. else
  7214. InternalError(2022081510);
  7215. end;
  7216. end;
  7217. function TX86AsmOptimizer.HandleSHRMerge(var p: tai; const PostPeephole: Boolean): Boolean;
  7218. var
  7219. hp1, hp2: tai;
  7220. IdentityMask, Shift: TCGInt;
  7221. LimitSize: Topsize;
  7222. DoNotMerge: Boolean;
  7223. begin
  7224. if not MatchInstruction(p, A_SHR, []) then
  7225. InternalError(2025040301);
  7226. Result := False;
  7227. DoNotMerge := False;
  7228. Shift := taicpu(p).oper[0]^.val;
  7229. LimitSize := taicpu(p).opsize;
  7230. hp1 := p;
  7231. repeat
  7232. if not GetNextInstructionUsingReg(hp1, hp1, taicpu(p).oper[1]^.reg) or (hp1.typ <> ait_instruction) then
  7233. Exit;
  7234. case taicpu(hp1).opcode of
  7235. A_AND:
  7236. { Detect:
  7237. shr x, %reg
  7238. and y, %reg
  7239. If and y, %reg doesn't actually change the value of %reg (e.g. with
  7240. "shrl $24,%reg; andl $255,%reg", remove the AND instruction.
  7241. (Post-peephole only)
  7242. }
  7243. if PostPeephole and
  7244. (taicpu(hp1).opsize = taicpu(p).opsize) and
  7245. MatchOpType(taicpu(hp1), top_const, top_reg) and
  7246. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  7247. begin
  7248. { Make sure the FLAGS register isn't in use }
  7249. TransferUsedRegs(TmpUsedRegs);
  7250. hp2 := p;
  7251. repeat
  7252. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  7253. until not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  7254. if not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs) then
  7255. begin
  7256. { Generate the identity mask }
  7257. case taicpu(p).opsize of
  7258. S_B:
  7259. IdentityMask := $FF shr Shift;
  7260. S_W:
  7261. IdentityMask := $FFFF shr Shift;
  7262. S_L:
  7263. IdentityMask := $FFFFFFFF shr Shift;
  7264. {$ifdef x86_64}
  7265. S_Q:
  7266. { We need to force the operands to be unsigned 64-bit
  7267. integers otherwise the wrong value is generated }
  7268. IdentityMask := TCGInt(QWord($FFFFFFFFFFFFFFFF) shr QWord(Shift));
  7269. {$endif x86_64}
  7270. else
  7271. InternalError(2022081501);
  7272. end;
  7273. if (taicpu(hp1).oper[0]^.val and IdentityMask) = IdentityMask then
  7274. begin
  7275. DebugMsg(SPeepholeOptimization + 'Removed AND instruction since previous SHR makes this an identity operation (ShrAnd2Shr)', hp1);
  7276. { All the possible 1 bits are covered, so we can remove the AND }
  7277. hp2 := tai(hp1.Previous);
  7278. RemoveInstruction(hp1);
  7279. { p wasn't actually changed, so don't set Result to True,
  7280. but a change was nonetheless made elsewhere }
  7281. Include(OptsToCheck, aoc_ForceNewIteration);
  7282. { Do another pass in case other AND or MOVZX instructions
  7283. follow }
  7284. hp1 := hp2;
  7285. Continue;
  7286. end;
  7287. end;
  7288. end;
  7289. A_TEST, A_CMP:
  7290. { Skip over relevant comparisons, but shift instructions must
  7291. now not be merged since the original value is being read }
  7292. begin
  7293. DoNotMerge := True;
  7294. Continue;
  7295. end;
  7296. A_Jcc:
  7297. { Skip over conditional jumps and relevant comparisons }
  7298. Continue;
  7299. A_MOVZX:
  7300. if MatchOpType(taicpu(hp1), top_reg, top_reg) and
  7301. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(p).oper[1]^.reg) then
  7302. begin
  7303. { Since the original register is being read as is, subsequent
  7304. SHRs must not be merged at this point }
  7305. DoNotMerge := True;
  7306. if IsShrMovZFoldable(taicpu(p).opsize, taicpu(hp1).opsize, Shift) then
  7307. begin
  7308. if SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg) then
  7309. begin
  7310. { If the MOVZX instruction reads and writes the same register,
  7311. defer this to the post-peephole optimisation stage }
  7312. if PostPeephole then
  7313. begin
  7314. DebugMsg(SPeepholeOptimization + 'Removed MOVZX instruction since previous SHR makes it unnecessary (ShrMovz2Shr)', hp1);
  7315. { All the possible 1 bits are covered, so we can remove the MOVZX }
  7316. hp2 := tai(hp1.Previous);
  7317. RemoveInstruction(hp1);
  7318. hp1 := hp2;
  7319. end;
  7320. end
  7321. else { Different register target }
  7322. begin
  7323. DebugMsg(SPeepholeOptimization + 'Converted MOVZX instruction to MOV since previous SHR makes zero-extension unnecessary (ShrMovz2ShrMov 1)', hp1);
  7324. taicpu(hp1).opcode := A_MOV;
  7325. setsubreg(taicpu(hp1).oper[0]^.reg, getsubreg(taicpu(hp1).oper[1]^.reg));
  7326. case taicpu(hp1).opsize of
  7327. S_BW:
  7328. taicpu(hp1).opsize := S_W;
  7329. S_BL, S_WL:
  7330. taicpu(hp1).opsize := S_L;
  7331. else
  7332. InternalError(2022081503);
  7333. end;
  7334. { p itself hasn't changed, so no need to set Result to True }
  7335. Include(OptsToCheck, aoc_ForceNewIteration);
  7336. { See if there's anything afterwards that can be
  7337. optimised, since the input register hasn't changed }
  7338. Continue;
  7339. end;
  7340. Exit;
  7341. end
  7342. else if PostPeephole and
  7343. (Shift > 0) and
  7344. (taicpu(p).opsize = S_W) and
  7345. (taicpu(hp1).opsize = S_WL) and
  7346. (taicpu(hp1).oper[0]^.reg = NR_AX) and
  7347. (taicpu(hp1).oper[1]^.reg = NR_EAX) then
  7348. begin
  7349. { Detect:
  7350. shr x, %ax (x > 0)
  7351. ...
  7352. movzwl %ax,%eax
  7353. -
  7354. Change movzwl %ax,%eax to cwtl (shorter encoding for movswl %ax,%eax)
  7355. But first, check to see if movzwl %ax,%eax can be removed...
  7356. }
  7357. hp2 := tai(hp1.Previous);
  7358. TransferUsedRegs(TmpUsedRegs);
  7359. UpdateUsedRegsBetween(UsedRegs, p, hp1);
  7360. if PostPeepholeOptMovZX(hp1) then
  7361. hp1 := hp2
  7362. else
  7363. begin
  7364. DebugMsg(SPeepholeOptimization + 'Converted movzwl %ax,%eax to cwtl (via ShrMovz2ShrCwtl)', hp1);
  7365. taicpu(hp1).opcode := A_CWDE;
  7366. taicpu(hp1).clearop(0);
  7367. taicpu(hp1).clearop(1);
  7368. taicpu(hp1).ops := 0;
  7369. end;
  7370. RestoreUsedRegs(TmpUsedRegs);
  7371. { Don't need to set aoc_ForceNewIteration if
  7372. PostPeepholeOptMovZX returned True because it's the
  7373. post-peephole stage }
  7374. end;
  7375. { Move onto the next instruction }
  7376. Continue;
  7377. end;
  7378. A_SHL, A_SAL, A_SHR:
  7379. if (taicpu(hp1).opsize <= LimitSize) and
  7380. MatchOpType(taicpu(hp1), top_const, top_reg) and
  7381. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[1]^.reg) then
  7382. begin
  7383. { Make sure the sizes don't exceed the register size limit
  7384. (measured by the shift value falling below the limit) }
  7385. if taicpu(hp1).opsize < LimitSize then
  7386. LimitSize := taicpu(hp1).opsize;
  7387. if taicpu(hp1).opcode = A_SHR then
  7388. Inc(Shift, taicpu(hp1).oper[0]^.val)
  7389. else
  7390. begin
  7391. Dec(Shift, taicpu(hp1).oper[0]^.val);
  7392. DoNotMerge := True;
  7393. end;
  7394. if Shift < topsize2memsize[taicpu(p).opsize] - topsize2memsize[LimitSize] then
  7395. Exit;
  7396. { Since we've established that the combined shift is within
  7397. limits, we can actually combine the adjacent SHR
  7398. instructions even if they're different sizes }
  7399. if not DoNotMerge and (taicpu(hp1).opcode = A_SHR) then
  7400. begin
  7401. hp2 := tai(hp1.Previous);
  7402. DebugMsg(SPeepholeOptimization + 'ShrShr2Shr 1', p);
  7403. Inc(taicpu(p).oper[0]^.val, taicpu(hp1).oper[0]^.val);
  7404. RemoveInstruction(hp1);
  7405. hp1 := hp2;
  7406. { Though p has changed, only the constant has, and its
  7407. effects can still be detected on the next iteration of
  7408. the repeat..until loop }
  7409. Include(OptsToCheck, aoc_ForceNewIteration);
  7410. end;
  7411. { Move onto the next instruction }
  7412. Continue;
  7413. end;
  7414. else
  7415. ;
  7416. end;
  7417. { If the register isn't actually modified, move onto the next instruction,
  7418. but set DoNotMerge to True since the register is being read }
  7419. if (
  7420. { Under -O2 and below, GetNextInstructionUsingReg only returns
  7421. the next instruction, whether or not it contains the register }
  7422. (cs_opt_level3 in current_settings.optimizerswitches) or
  7423. RegReadByInstruction(taicpu(p).oper[1]^.reg, hp1)
  7424. ) and not RegModifiedByInstruction(taicpu(p).oper[1]^.reg, hp1) then
  7425. begin
  7426. DoNotMerge := True;
  7427. Continue;
  7428. end;
  7429. Break;
  7430. until False;
  7431. end;
  7432. function TX86AsmOptimizer.OptPass1SHR(var p : tai) : boolean;
  7433. begin
  7434. Result := False;
  7435. { All these optimisations work on "shr const,%reg" }
  7436. if not MatchOpType(taicpu(p), top_const, top_reg) then
  7437. Exit;
  7438. Result := HandleSHRMerge(p, False);
  7439. end;
  7440. function TX86AsmOptimizer.CheckMemoryWrite(var first_mov, second_mov: taicpu): Boolean;
  7441. var
  7442. CurrentRef: TReference;
  7443. FullReg: TRegister;
  7444. hp1, hp2: tai;
  7445. begin
  7446. Result := False;
  7447. if (first_mov.opsize <> S_B) or (second_mov.opsize <> S_B) then
  7448. Exit;
  7449. { We assume you've checked if the operand is actually a reference by
  7450. this point. If it isn't, you'll most likely get an access violation }
  7451. CurrentRef := first_mov.oper[1]^.ref^;
  7452. { Memory must be aligned }
  7453. if (CurrentRef.offset mod 4) <> 0 then
  7454. Exit;
  7455. Inc(CurrentRef.offset);
  7456. CurrentRef.alignment := 1; { Otherwise references_equal will return False }
  7457. if MatchOperand(second_mov.oper[0]^, 0) and
  7458. references_equal(second_mov.oper[1]^.ref^, CurrentRef) and
  7459. GetNextInstruction(second_mov, hp1) and
  7460. (hp1.typ = ait_instruction) and
  7461. (taicpu(hp1).opcode = A_MOV) and
  7462. MatchOpType(taicpu(hp1), top_const, top_ref) and
  7463. (taicpu(hp1).oper[0]^.val = 0) then
  7464. begin
  7465. Inc(CurrentRef.offset);
  7466. CurrentRef.alignment := taicpu(hp1).oper[1]^.ref^.alignment; { Otherwise references_equal might return False }
  7467. FullReg := newreg(R_INTREGISTER,getsupreg(first_mov.oper[0]^.reg), R_SUBD);
  7468. if references_equal(taicpu(hp1).oper[1]^.ref^, CurrentRef) then
  7469. begin
  7470. case taicpu(hp1).opsize of
  7471. S_B:
  7472. if GetNextInstruction(hp1, hp2) and
  7473. MatchInstruction(taicpu(hp2), A_MOV, [S_B]) and
  7474. MatchOpType(taicpu(hp2), top_const, top_ref) and
  7475. (taicpu(hp2).oper[0]^.val = 0) then
  7476. begin
  7477. Inc(CurrentRef.offset);
  7478. CurrentRef.alignment := 1; { Otherwise references_equal will return False }
  7479. if references_equal(taicpu(hp2).oper[1]^.ref^, CurrentRef) and
  7480. (taicpu(hp2).opsize = S_B) then
  7481. begin
  7482. RemoveInstruction(hp1);
  7483. RemoveInstruction(hp2);
  7484. first_mov.opsize := S_L;
  7485. if first_mov.oper[0]^.typ = top_reg then
  7486. begin
  7487. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVb/MOVb -> MOVZX/MOVl', first_mov);
  7488. { Reuse second_mov as a MOVZX instruction }
  7489. second_mov.opcode := A_MOVZX;
  7490. second_mov.opsize := S_BL;
  7491. second_mov.loadreg(0, first_mov.oper[0]^.reg);
  7492. second_mov.loadreg(1, FullReg);
  7493. first_mov.oper[0]^.reg := FullReg;
  7494. asml.Remove(second_mov);
  7495. asml.InsertBefore(second_mov, first_mov);
  7496. end
  7497. else
  7498. { It's a value }
  7499. begin
  7500. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVb/MOVb -> MOVl', first_mov);
  7501. RemoveInstruction(second_mov);
  7502. end;
  7503. Result := True;
  7504. Exit;
  7505. end;
  7506. end;
  7507. S_W:
  7508. begin
  7509. RemoveInstruction(hp1);
  7510. first_mov.opsize := S_L;
  7511. if first_mov.oper[0]^.typ = top_reg then
  7512. begin
  7513. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVw -> MOVZX/MOVl', first_mov);
  7514. { Reuse second_mov as a MOVZX instruction }
  7515. second_mov.opcode := A_MOVZX;
  7516. second_mov.opsize := S_BL;
  7517. second_mov.loadreg(0, first_mov.oper[0]^.reg);
  7518. second_mov.loadreg(1, FullReg);
  7519. first_mov.oper[0]^.reg := FullReg;
  7520. asml.Remove(second_mov);
  7521. asml.InsertBefore(second_mov, first_mov);
  7522. end
  7523. else
  7524. { It's a value }
  7525. begin
  7526. DebugMsg(SPeepholeOptimization + 'MOVb/MOVb/MOVw -> MOVl', first_mov);
  7527. RemoveInstruction(second_mov);
  7528. end;
  7529. Result := True;
  7530. Exit;
  7531. end;
  7532. else
  7533. ;
  7534. end;
  7535. end;
  7536. end;
  7537. end;
  7538. function TX86AsmOptimizer.OptPass1FSTP(var p: tai): boolean;
  7539. { returns true if a "continue" should be done after this optimization }
  7540. var
  7541. hp1, hp2, hp3: tai;
  7542. begin
  7543. Result := false;
  7544. hp3 := nil;
  7545. if MatchOpType(taicpu(p),top_ref) and
  7546. GetNextInstruction(p, hp1) and
  7547. (hp1.typ = ait_instruction) and
  7548. (((taicpu(hp1).opcode = A_FLD) and
  7549. (taicpu(p).opcode = A_FSTP)) or
  7550. ((taicpu(p).opcode = A_FISTP) and
  7551. (taicpu(hp1).opcode = A_FILD))) and
  7552. MatchOpType(taicpu(hp1),top_ref) and
  7553. (taicpu(hp1).opsize = taicpu(p).opsize) and
  7554. RefsEqual(taicpu(p).oper[0]^.ref^, taicpu(hp1).oper[0]^.ref^) then
  7555. begin
  7556. { replacing fstp f;fld f by fst f is only valid for extended because of rounding or if fastmath is on }
  7557. if ((taicpu(p).opsize=S_FX) or (cs_opt_fastmath in current_settings.optimizerswitches)) and
  7558. GetNextInstruction(hp1, hp2) and
  7559. (((hp2.typ = ait_instruction) and
  7560. IsExitCode(hp2) and
  7561. (taicpu(p).oper[0]^.ref^.base = current_procinfo.FramePointer) and
  7562. not(assigned(current_procinfo.procdef.funcretsym) and
  7563. (taicpu(p).oper[0]^.ref^.offset < tabstractnormalvarsym(current_procinfo.procdef.funcretsym).localloc.reference.offset)) and
  7564. (taicpu(p).oper[0]^.ref^.index = NR_NO)) or
  7565. { fstp <temp>
  7566. fld <temp>
  7567. <dealloc> <temp>
  7568. }
  7569. ((taicpu(p).oper[0]^.ref^.base = current_procinfo.FramePointer) and
  7570. (taicpu(p).oper[0]^.ref^.index = NR_NO) and
  7571. SetAndTest(FindTempDeAlloc(taicpu(p).oper[0]^.ref^.offset,tai(hp1.next)),hp2) and
  7572. (tai_tempalloc(hp2).temppos=taicpu(p).oper[0]^.ref^.offset) and
  7573. (((taicpu(p).opsize=S_FX) and (tai_tempalloc(hp2).tempsize=16)) or
  7574. ((taicpu(p).opsize in [S_IQ,S_FL]) and (tai_tempalloc(hp2).tempsize=8)) or
  7575. ((taicpu(p).opsize=S_FS) and (tai_tempalloc(hp2).tempsize=4))
  7576. )
  7577. )
  7578. ) then
  7579. begin
  7580. DebugMsg(SPeepholeOptimization + 'FstpFld2<Nop>',p);
  7581. RemoveInstruction(hp1);
  7582. RemoveCurrentP(p, hp2);
  7583. { first case: exit code }
  7584. if hp2.typ = ait_instruction then
  7585. RemoveLastDeallocForFuncRes(p);
  7586. Result := true;
  7587. end
  7588. else
  7589. { we can do this only in fast math mode as fstp is rounding ...
  7590. ... still disabled as it breaks the compiler and/or rtl }
  7591. if { (cs_opt_fastmath in current_settings.optimizerswitches) or }
  7592. { ... or if another fstp equal to the first one follows }
  7593. GetNextInstruction(hp1,hp2) and
  7594. (hp2.typ = ait_instruction) and
  7595. (taicpu(p).opcode=taicpu(hp2).opcode) and
  7596. (taicpu(p).opsize=taicpu(hp2).opsize) then
  7597. begin
  7598. if (taicpu(p).oper[0]^.ref^.base = current_procinfo.FramePointer) and
  7599. (taicpu(p).oper[0]^.ref^.index = NR_NO) and
  7600. SetAndTest(FindTempDeAlloc(taicpu(p).oper[0]^.ref^.offset,tai(hp2.next)),hp3) and
  7601. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[0]^) and
  7602. (tai_tempalloc(hp3).temppos=taicpu(p).oper[0]^.ref^.offset) and
  7603. (((taicpu(p).opsize=S_FX) and (tai_tempalloc(hp3).tempsize=16)) or
  7604. ((taicpu(p).opsize in [S_IQ,S_FL]) and (tai_tempalloc(hp3).tempsize=8)) or
  7605. ((taicpu(p).opsize=S_FS) and (tai_tempalloc(hp3).tempsize=4))
  7606. ) then
  7607. begin
  7608. DebugMsg(SPeepholeOptimization + 'FstpFldFstp2Fstp',p);
  7609. RemoveCurrentP(p,hp2);
  7610. RemoveInstruction(hp1);
  7611. Result := true;
  7612. end
  7613. else if { fst can't store an extended/comp value }
  7614. (taicpu(p).opsize <> S_FX) and
  7615. (taicpu(p).opsize <> S_IQ) then
  7616. begin
  7617. if (taicpu(p).opcode = A_FSTP) then
  7618. taicpu(p).opcode := A_FST
  7619. else
  7620. taicpu(p).opcode := A_FIST;
  7621. DebugMsg(SPeepholeOptimization + 'FstpFld2Fst',p);
  7622. RemoveInstruction(hp1);
  7623. Result := true;
  7624. end;
  7625. end;
  7626. end;
  7627. end;
  7628. function TX86AsmOptimizer.OptPass1FLD(var p : tai) : boolean;
  7629. var
  7630. hp1, hp2, hp3: tai;
  7631. begin
  7632. result:=false;
  7633. if MatchOpType(taicpu(p),top_reg) and
  7634. GetNextInstruction(p, hp1) and
  7635. (hp1.typ = Ait_Instruction) and
  7636. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  7637. (taicpu(hp1).oper[0]^.reg = NR_ST) and
  7638. (taicpu(hp1).oper[1]^.reg = NR_ST1) then
  7639. { change to
  7640. fld reg fxxx reg,st
  7641. fxxxp st, st1 (hp1)
  7642. Remark: non commutative operations must be reversed!
  7643. }
  7644. begin
  7645. case taicpu(hp1).opcode Of
  7646. A_FMULP,A_FADDP,
  7647. A_FSUBP,A_FDIVP,A_FSUBRP,A_FDIVRP:
  7648. begin
  7649. case taicpu(hp1).opcode Of
  7650. A_FADDP: taicpu(hp1).opcode := A_FADD;
  7651. A_FMULP: taicpu(hp1).opcode := A_FMUL;
  7652. A_FSUBP: taicpu(hp1).opcode := A_FSUBR;
  7653. A_FSUBRP: taicpu(hp1).opcode := A_FSUB;
  7654. A_FDIVP: taicpu(hp1).opcode := A_FDIVR;
  7655. A_FDIVRP: taicpu(hp1).opcode := A_FDIV;
  7656. else
  7657. internalerror(2019050534);
  7658. end;
  7659. taicpu(hp1).oper[0]^.reg := taicpu(p).oper[0]^.reg;
  7660. taicpu(hp1).oper[1]^.reg := NR_ST;
  7661. DebugMsg(SPeepholeOptimization + 'FldF*p2F*',hp1);
  7662. RemoveCurrentP(p, hp1);
  7663. Result:=true;
  7664. exit;
  7665. end;
  7666. else
  7667. ;
  7668. end;
  7669. end
  7670. else
  7671. if MatchOpType(taicpu(p),top_ref) and
  7672. GetNextInstruction(p, hp2) and
  7673. (hp2.typ = Ait_Instruction) and
  7674. MatchOpType(taicpu(hp2),top_reg,top_reg) and
  7675. (taicpu(p).opsize in [S_FS, S_FL]) and
  7676. (taicpu(hp2).oper[0]^.reg = NR_ST) and
  7677. (taicpu(hp2).oper[1]^.reg = NR_ST1) then
  7678. if GetLastInstruction(p, hp1) and
  7679. MatchInstruction(hp1,A_FLD,A_FST,[taicpu(p).opsize]) and
  7680. MatchOpType(taicpu(hp1),top_ref) and
  7681. RefsEqual(taicpu(p).oper[0]^.ref^, taicpu(hp1).oper[0]^.ref^) then
  7682. if ((taicpu(hp2).opcode = A_FMULP) or
  7683. (taicpu(hp2).opcode = A_FADDP)) then
  7684. { change to
  7685. fld/fst mem1 (hp1) fld/fst mem1
  7686. fld mem1 (p) fadd/
  7687. faddp/ fmul st, st
  7688. fmulp st, st1 (hp2) }
  7689. begin
  7690. DebugMsg(SPeepholeOptimization + 'Fld/FstFldFaddp/Fmulp2Fld/FstFadd/Fmul',hp1);
  7691. RemoveCurrentP(p, hp1);
  7692. if (taicpu(hp2).opcode = A_FADDP) then
  7693. taicpu(hp2).opcode := A_FADD
  7694. else
  7695. taicpu(hp2).opcode := A_FMUL;
  7696. taicpu(hp2).oper[1]^.reg := NR_ST;
  7697. end
  7698. else
  7699. { change to
  7700. fld/fst mem1 (hp1) fld/fst mem1
  7701. fld mem1 (p) fld st
  7702. }
  7703. begin
  7704. DebugMsg(SPeepholeOptimization + 'Fld/Fst<mem>Fld<mem>2Fld/Fst<mem>Fld<reg>',hp1);
  7705. taicpu(p).changeopsize(S_FL);
  7706. taicpu(p).loadreg(0,NR_ST);
  7707. end
  7708. else
  7709. begin
  7710. case taicpu(hp2).opcode Of
  7711. A_FMULP,A_FADDP,A_FSUBP,A_FDIVP,A_FSUBRP,A_FDIVRP:
  7712. { change to
  7713. fld/fst mem1 (hp1) fld/fst mem1
  7714. fld mem2 (p) fxxx mem2
  7715. fxxxp st, st1 (hp2) }
  7716. begin
  7717. case taicpu(hp2).opcode Of
  7718. A_FADDP: taicpu(p).opcode := A_FADD;
  7719. A_FMULP: taicpu(p).opcode := A_FMUL;
  7720. A_FSUBP: taicpu(p).opcode := A_FSUBR;
  7721. A_FSUBRP: taicpu(p).opcode := A_FSUB;
  7722. A_FDIVP: taicpu(p).opcode := A_FDIVR;
  7723. A_FDIVRP: taicpu(p).opcode := A_FDIV;
  7724. else
  7725. internalerror(2019050533);
  7726. end;
  7727. DebugMsg(SPeepholeOptimization + 'Fld/FstFldF*2Fld/FstF*',p);
  7728. RemoveInstruction(hp2);
  7729. end
  7730. else
  7731. ;
  7732. end
  7733. end
  7734. end;
  7735. function IsCmpSubset(cond1, cond2: TAsmCond): Boolean; inline;
  7736. begin
  7737. Result := condition_in(cond1, cond2) or
  7738. { Not strictly subsets due to the actual flags checked, but because we're
  7739. comparing integers, E is a subset of AE and GE and their aliases }
  7740. ((cond1 in [C_E, C_Z]) and (cond2 in [C_AE, C_NB, C_NC, C_GE, C_NL]));
  7741. end;
  7742. function TX86AsmOptimizer.OptPass1Cmp(var p: tai): boolean;
  7743. var
  7744. v: TCGInt;
  7745. true_hp1, hp1, hp2, p_dist, p_jump, hp1_dist, p_label, hp1_label: tai;
  7746. FirstMatch, TempBool: Boolean;
  7747. NewReg: TRegister;
  7748. JumpLabel, JumpLabel_dist, JumpLabel_far: TAsmLabel;
  7749. begin
  7750. Result:=false;
  7751. { All these optimisations need a next instruction }
  7752. if not GetNextInstruction(p, hp1) then
  7753. Exit;
  7754. true_hp1 := hp1;
  7755. { Search for:
  7756. cmp ###,###
  7757. j(c1) @lbl1
  7758. ...
  7759. @lbl:
  7760. cmp ###,### (same comparison as above)
  7761. j(c2) @lbl2
  7762. If c1 is a subset of c2, change to:
  7763. cmp ###,###
  7764. j(c1) @lbl2
  7765. (@lbl1 may become a dead label as a result)
  7766. }
  7767. { Also handle cases where there are multiple jumps in a row }
  7768. p_jump := hp1;
  7769. while Assigned(p_jump) and MatchInstruction(p_jump, A_JCC, []) do
  7770. begin
  7771. Prefetch(p_jump.Next);
  7772. if IsJumpToLabel(taicpu(p_jump)) then
  7773. begin
  7774. { Do jump optimisations first in case the condition becomes
  7775. unnecessary }
  7776. TempBool := True;
  7777. if DoJumpOptimizations(p_jump, TempBool) or
  7778. not TempBool then
  7779. begin
  7780. if Assigned(p_jump) then
  7781. begin
  7782. { CollapseZeroDistJump will be set to the label or an align
  7783. before it after the jump if it optimises, whether or not
  7784. the label is live or dead }
  7785. if (p_jump.typ = ait_align) or
  7786. (
  7787. (p_jump.typ = ait_label) and
  7788. not (tai_label(p_jump).labsym.is_used)
  7789. ) then
  7790. GetNextInstruction(p_jump, p_jump);
  7791. end;
  7792. TransferUsedRegs(TmpUsedRegs);
  7793. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  7794. if not Assigned(p_jump) or
  7795. (
  7796. not MatchInstruction(p_jump, A_Jcc, A_SETcc, A_CMOVcc, []) and
  7797. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, p_jump, TmpUsedRegs)
  7798. ) then
  7799. begin
  7800. { No more conditional jumps; conditional statement is no longer required }
  7801. DebugMsg(SPeepholeOptimization + 'Removed unnecessary condition (Cmp2Nop)', p);
  7802. RemoveCurrentP(p);
  7803. Result := True;
  7804. Exit;
  7805. end;
  7806. hp1 := p_jump;
  7807. Include(OptsToCheck, aoc_ForceNewIteration);
  7808. Continue;
  7809. end;
  7810. JumpLabel := TAsmLabel(taicpu(p_jump).oper[0]^.ref^.symbol);
  7811. if GetNextInstruction(p_jump, hp2) and
  7812. (
  7813. OptimizeConditionalJump(JumpLabel, p_jump, hp2, TempBool) or
  7814. not TempBool
  7815. ) then
  7816. begin
  7817. hp1 := p_jump;
  7818. Include(OptsToCheck, aoc_ForceNewIteration);
  7819. Continue;
  7820. end;
  7821. p_label := nil;
  7822. if Assigned(JumpLabel) then
  7823. p_label := getlabelwithsym(JumpLabel);
  7824. if Assigned(p_label) and
  7825. GetNextInstruction(p_label, p_dist) and
  7826. MatchInstruction(p_dist, A_CMP, []) and
  7827. MatchOperand(taicpu(p_dist).oper[0]^, taicpu(p).oper[0]^) and
  7828. MatchOperand(taicpu(p_dist).oper[1]^, taicpu(p).oper[1]^) and
  7829. GetNextInstruction(p_dist, hp1_dist) and
  7830. MatchInstruction(hp1_dist, A_JCC, []) then { This doesn't have to be an explicit label }
  7831. begin
  7832. JumpLabel_dist := TAsmLabel(taicpu(hp1_dist).oper[0]^.ref^.symbol);
  7833. if JumpLabel = JumpLabel_dist then
  7834. { This is an infinite loop }
  7835. Exit;
  7836. { Best optimisation when the first condition is a subset (or equal) of the second }
  7837. if IsCmpSubset(taicpu(p_jump).condition, taicpu(hp1_dist).condition) then
  7838. begin
  7839. { Any registers used here will already be allocated }
  7840. if Assigned(JumpLabel) then
  7841. JumpLabel.DecRefs;
  7842. DebugMsg(SPeepholeOptimization + 'CMP/Jcc/@Lbl/CMP/Jcc -> CMP/Jcc, redirecting first jump', p_jump);
  7843. taicpu(p_jump).loadref(0, taicpu(hp1_dist).oper[0]^.ref^); { This also increases the reference count }
  7844. Include(OptsToCheck, aoc_ForceNewIteration);
  7845. { Don't exit yet. Since p and p_jump haven't actually been
  7846. removed, we can check for more on this iteration }
  7847. end
  7848. else if IsCmpSubset(taicpu(hp1_dist).condition, inverse_cond(taicpu(p_jump).condition)) and
  7849. GetNextInstruction(hp1_dist, hp1_label) and
  7850. (hp1_label.typ = ait_label) then
  7851. begin
  7852. JumpLabel_far := tai_label(hp1_label).labsym;
  7853. if (JumpLabel_far = JumpLabel_dist) or (JumpLabel_far = JumpLabel) then
  7854. { This is an infinite loop }
  7855. Exit;
  7856. if Assigned(JumpLabel_far) then
  7857. begin
  7858. { In this situation, if the first jump branches, the second one will never,
  7859. branch so change the destination label to after the second jump }
  7860. DebugMsg(SPeepholeOptimization + 'CMP/Jcc/@Lbl/CMP/Jcc/@Lbl -> CMP/Jcc, redirecting first jump to 2nd label', p_jump);
  7861. if Assigned(JumpLabel) then
  7862. JumpLabel.DecRefs;
  7863. JumpLabel_far.IncRefs;
  7864. taicpu(p_jump).oper[0]^.ref^.symbol := JumpLabel_far;
  7865. Result := True;
  7866. { Don't exit yet. Since p and p_jump haven't actually been
  7867. removed, we can check for more on this iteration }
  7868. Continue;
  7869. end;
  7870. end;
  7871. end;
  7872. end;
  7873. { Search for:
  7874. cmp ###,###
  7875. j(c1) @lbl1
  7876. cmp ###,### (same as first)
  7877. Remove second cmp
  7878. }
  7879. if GetNextInstruction(p_jump, hp2) and
  7880. (
  7881. (
  7882. MatchInstruction(hp2, A_CMP, [taicpu(p).opsize]) and
  7883. (
  7884. (
  7885. MatchOpType(taicpu(p), top_const, top_reg) and
  7886. MatchOpType(taicpu(hp2), top_const, top_reg) and
  7887. (taicpu(hp2).oper[0]^.val = taicpu(p).oper[0]^.val) and
  7888. Reg1WriteOverwritesReg2Entirely(taicpu(hp2).oper[1]^.reg, taicpu(p).oper[1]^.reg)
  7889. ) or (
  7890. MatchOperand(taicpu(hp2).oper[0]^, taicpu(p).oper[0]^) and
  7891. MatchOperand(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^)
  7892. )
  7893. )
  7894. ) or (
  7895. { Also match cmp $0,%reg; jcc @lbl; test %reg,%reg }
  7896. MatchOperand(taicpu(p).oper[0]^, 0) and
  7897. (taicpu(p).oper[1]^.typ = top_reg) and
  7898. MatchInstruction(hp2, A_TEST, []) and
  7899. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  7900. (taicpu(hp2).oper[0]^.reg = taicpu(hp2).oper[1]^.reg) and
  7901. Reg1WriteOverwritesReg2Entirely(taicpu(hp2).oper[1]^.reg, taicpu(p).oper[1]^.reg)
  7902. )
  7903. ) then
  7904. begin
  7905. DebugMsg(SPeepholeOptimization + 'CMP/Jcc/CMP; removed superfluous CMP', hp2);
  7906. TransferUsedRegs(TmpUsedRegs);
  7907. AllocRegBetween(NR_DEFAULTFLAGS, p, hp2, TmpUsedRegs);
  7908. RemoveInstruction(hp2);
  7909. Result := True;
  7910. { Continue the while loop in case "Jcc/CMP" follows the second CMP that was just removed }
  7911. end
  7912. else
  7913. begin
  7914. { hp2 is the next instruction, so save time and just set p_jump
  7915. to it instead of calling GetNextInstruction below }
  7916. p_jump := hp2;
  7917. Continue;
  7918. end;
  7919. GetNextInstruction(p_jump, p_jump);
  7920. end;
  7921. if (
  7922. { Don't call GetNextInstruction again if we already have it }
  7923. (true_hp1 = p_jump) or
  7924. GetNextInstruction(p, hp1)
  7925. ) and
  7926. MatchInstruction(hp1, A_Jcc, []) and
  7927. IsJumpToLabel(taicpu(hp1)) and
  7928. (taicpu(hp1).condition in [C_E, C_Z, C_NE, C_NZ]) and
  7929. GetNextInstruction(hp1, hp2) then
  7930. begin
  7931. {
  7932. cmp x, y (or "cmp y, x")
  7933. je @lbl
  7934. mov x, y
  7935. @lbl:
  7936. (x and y can be constants, registers or references)
  7937. Change to:
  7938. mov x, y (x and y will always be equal in the end)
  7939. @lbl: (may beceome a dead label)
  7940. Also:
  7941. cmp x, y (or "cmp y, x")
  7942. jne @lbl
  7943. mov x, y
  7944. @lbl:
  7945. (x and y can be constants, registers or references)
  7946. Change to:
  7947. Absolutely nothing! (Except @lbl if it's still live)
  7948. }
  7949. if MatchInstruction(hp2, A_MOV, [taicpu(p).opsize]) and
  7950. (
  7951. (
  7952. MatchOperand(taicpu(p).oper[0]^, taicpu(hp2).oper[0]^) and
  7953. MatchOperand(taicpu(p).oper[1]^, taicpu(hp2).oper[1]^)
  7954. ) or (
  7955. MatchOperand(taicpu(p).oper[0]^, taicpu(hp2).oper[1]^) and
  7956. MatchOperand(taicpu(p).oper[1]^, taicpu(hp2).oper[0]^)
  7957. )
  7958. ) and
  7959. GetNextInstruction(hp2, hp1_label) and
  7960. (hp1_label.typ = ait_label) and
  7961. (tai_label(hp1_label).labsym = taicpu(hp1).oper[0]^.ref^.symbol) then
  7962. begin
  7963. tai_label(hp1_label).labsym.DecRefs;
  7964. if (taicpu(hp1).condition in [C_NE, C_NZ]) then
  7965. begin
  7966. DebugMsg(SPeepholeOptimization + 'CMP/JNE/MOV/@Lbl -> NOP, since the MOV is only executed if the operands are equal (CmpJneMov2Nop)', p);
  7967. RemoveInstruction(hp2);
  7968. hp2 := hp1_label; { So RemoveCurrentp below can be set to something valid }
  7969. end
  7970. else
  7971. DebugMsg(SPeepholeOptimization + 'CMP/JE/MOV/@Lbl -> MOV, since the MOV is only executed if the operands aren''t equal (CmpJeMov2Mov)', p);
  7972. RemoveInstruction(hp1);
  7973. RemoveCurrentp(p, hp2);
  7974. Result := True;
  7975. Exit;
  7976. end;
  7977. {
  7978. Try to optimise the following:
  7979. cmp $x,### ($x and $y can be registers or constants)
  7980. je @lbl1 (only reference)
  7981. cmp $y,### (### are identical)
  7982. @Lbl:
  7983. sete %reg1
  7984. Change to:
  7985. cmp $x,###
  7986. sete %reg2 (allocate new %reg2)
  7987. cmp $y,###
  7988. sete %reg1
  7989. orb %reg2,%reg1
  7990. (dealloc %reg2)
  7991. This adds an instruction (so don't perform under -Os), but it removes
  7992. a conditional branch.
  7993. }
  7994. if not (cs_opt_size in current_settings.optimizerswitches) and
  7995. MatchInstruction(hp2, A_CMP, A_TEST, [taicpu(p).opsize]) and
  7996. MatchOperand(taicpu(p).oper[1]^, taicpu(hp2).oper[1]^) and
  7997. { The first operand of CMP instructions can only be a register or
  7998. immediate anyway, so no need to check }
  7999. GetNextInstruction(hp2, p_label) and
  8000. (p_label.typ = ait_label) and
  8001. (tai_label(p_label).labsym.getrefs = 1) and
  8002. (JumpTargetOp(taicpu(hp1))^.ref^.symbol = tai_label(p_label).labsym) and
  8003. GetNextInstruction(p_label, p_dist) and
  8004. MatchInstruction(p_dist, A_SETcc, []) and
  8005. (taicpu(p_dist).condition in [C_E, C_Z]) and
  8006. (taicpu(p_dist).oper[0]^.typ = top_reg) then
  8007. begin
  8008. TransferUsedRegs(TmpUsedRegs);
  8009. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  8010. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  8011. UpdateUsedRegs(TmpUsedRegs, tai(p_label.Next));
  8012. UpdateUsedRegs(TmpUsedRegs, tai(p_dist.Next));
  8013. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  8014. { Get the instruction after the SETcc instruction so we can
  8015. allocate a new register over the entire range }
  8016. GetNextInstruction(p_dist, hp1_dist) then
  8017. begin
  8018. { Register can appear in p if it's not used afterwards, so only
  8019. allocate between hp1 and hp1_dist }
  8020. NewReg := GetIntRegisterBetween(R_SUBL, TmpUsedRegs, hp1, hp1_dist);
  8021. if NewReg <> NR_NO then
  8022. begin
  8023. DebugMsg(SPeepholeOptimization + 'CMP/JE/CMP/@Lbl/SETE -> CMP/SETE/CMP/SETE/OR, removing conditional branch', p);
  8024. { Change the jump instruction into a SETcc instruction }
  8025. taicpu(hp1).opcode := A_SETcc;
  8026. taicpu(hp1).opsize := S_B;
  8027. taicpu(hp1).loadreg(0, NewReg);
  8028. { This is now a dead label }
  8029. tai_label(p_label).labsym.decrefs;
  8030. { Prefer adding before the next instruction so the FLAGS
  8031. register is deallicated first }
  8032. AsmL.InsertBefore(
  8033. taicpu.op_reg_reg(A_OR, S_B, NewReg, taicpu(p_dist).oper[0]^.reg),
  8034. hp1_dist
  8035. );
  8036. Result := True;
  8037. { Don't exit yet, as p wasn't changed and hp1, while
  8038. modified, is still intact and might be optimised by the
  8039. SETcc optimisation below }
  8040. end;
  8041. end;
  8042. end;
  8043. end;
  8044. if (taicpu(p).oper[0]^.typ = top_const) and
  8045. MatchInstruction(hp1,A_Jcc,A_SETcc,[]) then
  8046. begin
  8047. if (taicpu(p).oper[0]^.val = 0) and
  8048. (taicpu(p).oper[1]^.typ = top_reg) then
  8049. begin
  8050. hp2 := p;
  8051. FirstMatch := True;
  8052. { When dealing with "cmp $0,%reg", only ZF and SF contain
  8053. anything meaningful once it's converted to "test %reg,%reg";
  8054. additionally, some jumps will always (or never) branch, so
  8055. evaluate every jump immediately following the
  8056. comparison, optimising the conditions if possible.
  8057. Similarly with SETcc... those that are always set to 0 or 1
  8058. are changed to MOV instructions }
  8059. while FirstMatch or { Saves calling GetNextInstruction unnecessarily }
  8060. (
  8061. GetNextInstruction(hp2, hp1) and
  8062. MatchInstruction(hp1,A_Jcc,A_SETcc,[])
  8063. ) do
  8064. begin
  8065. Prefetch(hp1.Next);
  8066. FirstMatch := False;
  8067. case taicpu(hp1).condition of
  8068. C_B, C_C, C_NAE, C_O:
  8069. { For B/NAE:
  8070. Will never branch since an unsigned integer can never be below zero
  8071. For C/O:
  8072. Result cannot overflow because 0 is being subtracted
  8073. }
  8074. begin
  8075. if taicpu(hp1).opcode = A_Jcc then
  8076. begin
  8077. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition B/C/NAE/O --> Never (jump removed)', hp1);
  8078. TAsmLabel(taicpu(hp1).oper[0]^.ref^.symbol).decrefs;
  8079. RemoveInstruction(hp1);
  8080. { Since hp1 was deleted, hp2 must not be updated }
  8081. Continue;
  8082. end
  8083. else
  8084. begin
  8085. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition B/C/NAE/O --> Never (set -> mov 0)', hp1);
  8086. { Convert "set(c) %reg" instruction to "movb 0,%reg" }
  8087. taicpu(hp1).opcode := A_MOV;
  8088. taicpu(hp1).ops := 2;
  8089. taicpu(hp1).condition := C_None;
  8090. taicpu(hp1).opsize := S_B;
  8091. taicpu(hp1).loadreg(1,taicpu(hp1).oper[0]^.reg);
  8092. taicpu(hp1).loadconst(0, 0);
  8093. end;
  8094. end;
  8095. C_BE, C_NA:
  8096. begin
  8097. { Will only branch if equal to zero }
  8098. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition BE/NA --> E', hp1);
  8099. taicpu(hp1).condition := C_E;
  8100. end;
  8101. C_A, C_NBE:
  8102. begin
  8103. { Will only branch if not equal to zero }
  8104. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition A/NBE --> NE', hp1);
  8105. taicpu(hp1).condition := C_NE;
  8106. end;
  8107. C_AE, C_NB, C_NC, C_NO:
  8108. begin
  8109. { Will always branch }
  8110. DebugMsg(SPeepholeOptimization + 'Cmpcc2Testcc - condition AE/NB/NC/NO --> Always', hp1);
  8111. if taicpu(hp1).opcode = A_Jcc then
  8112. begin
  8113. MakeUnconditional(taicpu(hp1));
  8114. { Any jumps/set that follow will now be dead code }
  8115. RemoveDeadCodeAfterJump(taicpu(hp1));
  8116. Break;
  8117. end
  8118. else
  8119. begin
  8120. { Convert "set(c) %reg" instruction to "movb 1,%reg" }
  8121. taicpu(hp1).opcode := A_MOV;
  8122. taicpu(hp1).ops := 2;
  8123. taicpu(hp1).condition := C_None;
  8124. taicpu(hp1).opsize := S_B;
  8125. taicpu(hp1).loadreg(1,taicpu(hp1).oper[0]^.reg);
  8126. taicpu(hp1).loadconst(0, 1);
  8127. end;
  8128. end;
  8129. C_None:
  8130. InternalError(2020012201);
  8131. C_P, C_PE, C_NP, C_PO:
  8132. { We can't handle parity checks and they should never be generated
  8133. after a general-purpose CMP (it's used in some floating-point
  8134. comparisons that don't use CMP) }
  8135. InternalError(2020012202);
  8136. else
  8137. { Zero/Equality, Sign, their complements and all of the
  8138. signed comparisons do not need to be converted };
  8139. end;
  8140. hp2 := hp1;
  8141. end;
  8142. { Convert the instruction to a TEST }
  8143. taicpu(p).opcode := A_TEST;
  8144. taicpu(p).loadreg(0,taicpu(p).oper[1]^.reg);
  8145. Result := True;
  8146. Exit;
  8147. end
  8148. else
  8149. begin
  8150. TransferUsedRegs(TmpUsedRegs);
  8151. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  8152. if not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs) then
  8153. begin
  8154. if (taicpu(p).oper[0]^.val = 1) and
  8155. (taicpu(hp1).condition in [C_L, C_NL, C_NGE, C_GE]) then
  8156. begin
  8157. { Convert; To:
  8158. cmp $1,r/m cmp $0,r/m
  8159. jl @lbl jle @lbl
  8160. (Also do inverted conditions)
  8161. }
  8162. DebugMsg(SPeepholeOptimization + 'Cmp1Jl2Cmp0Jle', p);
  8163. taicpu(p).oper[0]^.val := 0;
  8164. if taicpu(hp1).condition in [C_L, C_NGE] then
  8165. taicpu(hp1).condition := C_LE
  8166. else
  8167. taicpu(hp1).condition := C_NLE;
  8168. { If the instruction is now "cmp $0,%reg", convert it to a
  8169. TEST (and effectively do the work of the "cmp $0,%reg" in
  8170. the block above)
  8171. }
  8172. if (taicpu(p).oper[1]^.typ = top_reg) then
  8173. begin
  8174. taicpu(p).opcode := A_TEST;
  8175. taicpu(p).loadreg(0,taicpu(p).oper[1]^.reg);
  8176. end;
  8177. Result := True;
  8178. Exit;
  8179. end
  8180. else if (taicpu(p).oper[1]^.typ = top_reg)
  8181. {$ifdef x86_64}
  8182. and (taicpu(p).opsize <> S_Q) { S_Q will never happen: cmp with 64 bit constants is not possible }
  8183. {$endif x86_64}
  8184. then
  8185. begin
  8186. { cmp register,$8000 neg register
  8187. je target --> jo target
  8188. .... only if register is deallocated before jump.}
  8189. case Taicpu(p).opsize of
  8190. S_B: v:=$80;
  8191. S_W: v:=$8000;
  8192. S_L: v:=qword($80000000);
  8193. else
  8194. internalerror(2013112905);
  8195. end;
  8196. if (taicpu(p).oper[0]^.val=v) and
  8197. (Taicpu(hp1).condition in [C_E,C_NE]) then
  8198. begin
  8199. TransferUsedRegs(TmpUsedRegs);
  8200. UpdateUsedRegs(TmpUsedRegs,tai(p.next));
  8201. if not(RegInUsedRegs(Taicpu(p).oper[1]^.reg, TmpUsedRegs)) then
  8202. begin
  8203. DebugMsg(SPeepholeOptimization + 'CmpJe2NegJo done',p);
  8204. Taicpu(p).opcode:=A_NEG;
  8205. Taicpu(p).loadoper(0,Taicpu(p).oper[1]^);
  8206. Taicpu(p).clearop(1);
  8207. Taicpu(p).ops:=1;
  8208. if Taicpu(hp1).condition=C_E then
  8209. Taicpu(hp1).condition:=C_O
  8210. else
  8211. Taicpu(hp1).condition:=C_NO;
  8212. Result:=true;
  8213. exit;
  8214. end;
  8215. end;
  8216. end;
  8217. end;
  8218. end;
  8219. end;
  8220. if TrySwapMovCmp(p, hp1) then
  8221. begin
  8222. Result := True;
  8223. Exit;
  8224. end;
  8225. end;
  8226. function TX86AsmOptimizer.OptPass1PXor(var p: tai): boolean;
  8227. var
  8228. hp1: tai;
  8229. begin
  8230. {
  8231. remove the second (v)pxor from
  8232. pxor reg,reg
  8233. ...
  8234. pxor reg,reg
  8235. }
  8236. Result:=false;
  8237. if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^) and
  8238. MatchOpType(taicpu(p),top_reg,top_reg) and
  8239. GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[0]^.reg) and
  8240. MatchInstruction(hp1,taicpu(p).opcode,[taicpu(p).opsize]) and
  8241. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[0]^) and
  8242. MatchOperand(taicpu(hp1).oper[0]^,taicpu(hp1).oper[1]^) then
  8243. begin
  8244. DebugMsg(SPeepholeOptimization + 'PXorPXor2PXor done',hp1);
  8245. RemoveInstruction(hp1);
  8246. Result:=true;
  8247. Exit;
  8248. end
  8249. {
  8250. replace
  8251. pxor reg1,reg1
  8252. movapd/s reg1,reg2
  8253. dealloc reg1
  8254. by
  8255. pxor reg2,reg2
  8256. }
  8257. else if GetNextInstruction(p,hp1) and
  8258. { we mix single and double opperations here because we assume that the compiler
  8259. generates vmovapd only after double operations and vmovaps only after single operations }
  8260. MatchInstruction(hp1,A_MOVAPD,A_MOVAPS,[S_NO]) and
  8261. MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^) and
  8262. MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^) and
  8263. (taicpu(p).oper[0]^.typ=top_reg) then
  8264. begin
  8265. TransferUsedRegs(TmpUsedRegs);
  8266. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  8267. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  8268. begin
  8269. taicpu(p).loadoper(0,taicpu(hp1).oper[1]^);
  8270. taicpu(p).loadoper(1,taicpu(hp1).oper[1]^);
  8271. DebugMsg(SPeepholeOptimization + 'PXorMovapd2PXor done',p);
  8272. RemoveInstruction(hp1);
  8273. result:=true;
  8274. end;
  8275. end;
  8276. end;
  8277. function TX86AsmOptimizer.OptPass1VPXor(var p: tai): boolean;
  8278. var
  8279. hp1: tai;
  8280. begin
  8281. {
  8282. remove the second (v)pxor from
  8283. (v)pxor reg,reg
  8284. ...
  8285. (v)pxor reg,reg
  8286. }
  8287. Result:=false;
  8288. if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^,taicpu(p).oper[2]^) and
  8289. MatchOpType(taicpu(p),top_reg,top_reg,top_reg) then
  8290. begin
  8291. if GetNextInstructionUsingReg(p,hp1,taicpu(p).oper[0]^.reg) and
  8292. MatchInstruction(hp1,taicpu(p).opcode,[taicpu(p).opsize]) and
  8293. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[0]^) and
  8294. MatchOperand(taicpu(hp1).oper[0]^,taicpu(hp1).oper[1]^,taicpu(hp1).oper[2]^) then
  8295. begin
  8296. DebugMsg(SPeepholeOptimization + 'VPXorVPXor2VPXor done',hp1);
  8297. RemoveInstruction(hp1);
  8298. Result:=true;
  8299. Exit;
  8300. end;
  8301. {$ifdef x86_64}
  8302. {
  8303. replace
  8304. vpxor reg1,reg1,reg1
  8305. vmov reg,mem
  8306. by
  8307. movq $0,mem
  8308. }
  8309. if GetNextInstruction(p,hp1) and
  8310. MatchInstruction(hp1,A_VMOVSD,[]) and
  8311. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  8312. MatchOpType(taicpu(hp1),top_reg,top_ref) then
  8313. begin
  8314. TransferUsedRegs(TmpUsedRegs);
  8315. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  8316. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  8317. begin
  8318. taicpu(hp1).loadconst(0,0);
  8319. taicpu(hp1).opcode:=A_MOV;
  8320. taicpu(hp1).opsize:=S_Q;
  8321. DebugMsg(SPeepholeOptimization + 'VPXorVMov2Mov done',p);
  8322. RemoveCurrentP(p);
  8323. result:=true;
  8324. Exit;
  8325. end;
  8326. end;
  8327. {$endif x86_64}
  8328. end
  8329. {
  8330. replace
  8331. vpxor reg1,reg1,reg2
  8332. by
  8333. vpxor reg2,reg2,reg2
  8334. to avoid unncessary data dependencies
  8335. }
  8336. else if MatchOperand(taicpu(p).oper[0]^,taicpu(p).oper[1]^) and
  8337. MatchOpType(taicpu(p),top_reg,top_reg,top_reg) then
  8338. begin
  8339. DebugMsg(SPeepholeOptimization + 'VPXor2VPXor done',p);
  8340. { avoid unncessary data dependency }
  8341. taicpu(p).loadreg(0,taicpu(p).oper[2]^.reg);
  8342. taicpu(p).loadreg(1,taicpu(p).oper[2]^.reg);
  8343. result:=true;
  8344. exit;
  8345. end;
  8346. Result:=OptPass1VOP(p);
  8347. end;
  8348. function TX86AsmOptimizer.OptPass1Imul(var p: tai): boolean;
  8349. var
  8350. hp1 : tai;
  8351. begin
  8352. result:=false;
  8353. { replace
  8354. IMul const,%mreg1,%mreg2
  8355. Mov %reg2,%mreg3
  8356. dealloc %mreg3
  8357. by
  8358. Imul const,%mreg1,%mreg23
  8359. }
  8360. if (taicpu(p).ops=3) and
  8361. GetNextInstruction(p,hp1) and
  8362. MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) and
  8363. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  8364. (taicpu(hp1).oper[1]^.typ=top_reg) then
  8365. begin
  8366. TransferUsedRegs(TmpUsedRegs);
  8367. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  8368. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  8369. begin
  8370. taicpu(p).loadoper(2,taicpu(hp1).oper[1]^);
  8371. DebugMsg(SPeepholeOptimization + 'ImulMov2Imul done',p);
  8372. RemoveInstruction(hp1);
  8373. result:=true;
  8374. end;
  8375. end;
  8376. end;
  8377. function TX86AsmOptimizer.OptPass1SHXX(var p: tai): boolean;
  8378. var
  8379. hp1 : tai;
  8380. begin
  8381. result:=false;
  8382. { replace
  8383. IMul %reg0,%reg1,%reg2
  8384. Mov %reg2,%reg3
  8385. dealloc %reg2
  8386. by
  8387. Imul %reg0,%reg1,%reg3
  8388. }
  8389. if GetNextInstruction(p,hp1) and
  8390. MatchInstruction(hp1,A_MOV,[taicpu(p).opsize]) and
  8391. MatchOperand(taicpu(p).oper[2]^,taicpu(hp1).oper[0]^) and
  8392. (taicpu(hp1).oper[1]^.typ=top_reg) then
  8393. begin
  8394. TransferUsedRegs(TmpUsedRegs);
  8395. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  8396. if not(RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg,hp1,TmpUsedRegs)) then
  8397. begin
  8398. taicpu(p).loadoper(2,taicpu(hp1).oper[1]^);
  8399. DebugMsg(SPeepholeOptimization + 'SHXXMov2SHXX done',p);
  8400. RemoveInstruction(hp1);
  8401. result:=true;
  8402. end;
  8403. end;
  8404. end;
  8405. function TX86AsmOptimizer.OptPass1_V_Cvtss2sd(var p: tai): boolean;
  8406. var
  8407. hp1: tai;
  8408. begin
  8409. Result:=false;
  8410. { get rid of
  8411. (v)cvtss2sd reg0,<reg1,>reg2
  8412. (v)cvtss2sd reg2,<reg2,>reg0
  8413. }
  8414. if GetNextInstruction(p,hp1) and
  8415. (((taicpu(p).opcode=A_CVTSS2SD) and MatchInstruction(hp1,A_CVTSD2SS,[taicpu(p).opsize]) and
  8416. MatchOperand(taicpu(p).oper[0]^,taicpu(hp1).oper[1]^) and MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^)) or
  8417. ((taicpu(p).opcode=A_VCVTSS2SD) and MatchInstruction(hp1,A_VCVTSD2SS,[taicpu(p).opsize]) and
  8418. MatchOpType(taicpu(p),top_reg,top_reg,top_reg) and
  8419. MatchOpType(taicpu(hp1),top_reg,top_reg,top_reg) and
  8420. (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) and
  8421. (getsupreg(taicpu(hp1).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg)) and
  8422. (getsupreg(taicpu(p).oper[2]^.reg)=getsupreg(taicpu(hp1).oper[0]^.reg))
  8423. )
  8424. ) then
  8425. begin
  8426. if ((taicpu(p).opcode=A_CVTSS2SD) and (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg))) or
  8427. ((taicpu(p).opcode=A_VCVTSS2SD) and (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[2]^.reg))) then
  8428. begin
  8429. DebugMsg(SPeepholeOptimization + '(V)Cvtss2CvtSd(V)Cvtsd2ss2Nop done',p);
  8430. RemoveCurrentP(p);
  8431. RemoveInstruction(hp1);
  8432. end
  8433. else
  8434. begin
  8435. DebugMsg(SPeepholeOptimization + '(V)Cvtss2CvtSd(V)Cvtsd2ss2Vmovaps done',p);
  8436. if taicpu(hp1).opcode=A_CVTSD2SS then
  8437. begin
  8438. taicpu(p).loadreg(1,taicpu(hp1).oper[1]^.reg);
  8439. taicpu(p).opcode:=A_MOVAPS;
  8440. end
  8441. else
  8442. begin
  8443. taicpu(p).loadreg(1,taicpu(hp1).oper[2]^.reg);
  8444. taicpu(p).opcode:=A_VMOVAPS;
  8445. end;
  8446. taicpu(p).ops:=2;
  8447. RemoveInstruction(hp1);
  8448. end;
  8449. Result:=true;
  8450. Exit;
  8451. end;
  8452. end;
  8453. function TX86AsmOptimizer.OptPass1Jcc(var p : tai) : boolean;
  8454. var
  8455. hp1, hp2, hp3, hp4, hp5: tai;
  8456. ThisReg: TRegister;
  8457. begin
  8458. Result := False;
  8459. if not GetNextInstruction(p,hp1) then
  8460. Exit;
  8461. {
  8462. convert
  8463. j<c> .L1
  8464. mov 1,reg
  8465. jmp .L2
  8466. .L1
  8467. mov 0,reg
  8468. .L2
  8469. into
  8470. mov 0,reg
  8471. set<not(c)> reg
  8472. take care of alignment and that the mov 0,reg is not converted into a xor as this
  8473. would destroy the flag contents
  8474. Use MOVZX if size is preferred, since while mov 0,reg is bigger, it can be
  8475. executed at the same time as a previous comparison.
  8476. set<not(c)> reg
  8477. movzx reg, reg
  8478. }
  8479. if MatchInstruction(hp1,A_MOV,[]) and
  8480. (taicpu(hp1).oper[0]^.typ = top_const) and
  8481. (
  8482. (
  8483. (taicpu(hp1).oper[1]^.typ = top_reg)
  8484. {$ifdef i386}
  8485. { Under i386, ESI, EDI, EBP and ESP
  8486. don't have an 8-bit representation }
  8487. and not (getsupreg(taicpu(hp1).oper[1]^.reg) in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  8488. {$endif i386}
  8489. ) or (
  8490. {$ifdef i386}
  8491. (taicpu(hp1).oper[1]^.typ <> top_reg) and
  8492. {$endif i386}
  8493. (taicpu(hp1).opsize = S_B)
  8494. )
  8495. ) and
  8496. GetNextInstruction(hp1,hp2) and
  8497. MatchInstruction(hp2,A_JMP,[]) and (taicpu(hp2).oper[0]^.ref^.refaddr=addr_full) and
  8498. GetNextInstruction(hp2,hp3) and
  8499. FindLabel(tasmlabel(taicpu(p).oper[0]^.ref^.symbol), hp3) and
  8500. GetNextInstruction(hp3,hp4) and
  8501. MatchInstruction(hp4,A_MOV,[taicpu(hp1).opsize]) and
  8502. (taicpu(hp4).oper[0]^.typ = top_const) and
  8503. (
  8504. ((taicpu(hp1).oper[0]^.val = 0) and (taicpu(hp4).oper[0]^.val = 1)) or
  8505. ((taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0))
  8506. ) and
  8507. MatchOperand(taicpu(hp1).oper[1]^,taicpu(hp4).oper[1]^) and
  8508. GetNextInstruction(hp4,hp5) and
  8509. FindLabel(tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol), hp5) then
  8510. begin
  8511. if (taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0) then
  8512. taicpu(p).condition := inverse_cond(taicpu(p).condition);
  8513. tai_label(hp3).labsym.DecRefs;
  8514. { If this isn't the only reference to the middle label, we can
  8515. still make a saving - only that the first jump and everything
  8516. that follows will remain. }
  8517. if (tai_label(hp3).labsym.getrefs = 0) then
  8518. begin
  8519. if (taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0) then
  8520. DebugMsg(SPeepholeOptimization + 'J(c)Mov1JmpMov0 -> Set(~c)',p)
  8521. else
  8522. DebugMsg(SPeepholeOptimization + 'J(c)Mov0JmpMov1 -> Set(c)',p);
  8523. { remove jump, first label and second MOV (also catching any aligns) }
  8524. repeat
  8525. if not GetNextInstruction(hp2, hp3) then
  8526. InternalError(2021040810);
  8527. RemoveInstruction(hp2);
  8528. hp2 := hp3;
  8529. until hp2 = hp5;
  8530. { Don't decrement reference count before the removal loop
  8531. above, otherwise GetNextInstruction won't stop on the
  8532. the label }
  8533. tai_label(hp5).labsym.DecRefs;
  8534. end
  8535. else
  8536. begin
  8537. if (taicpu(hp1).oper[0]^.val = 1) and (taicpu(hp4).oper[0]^.val = 0) then
  8538. DebugMsg(SPeepholeOptimization + 'J(c)Mov1JmpMov0 -> Set(~c) (partial)',p)
  8539. else
  8540. DebugMsg(SPeepholeOptimization + 'J(c)Mov0JmpMov1 -> Set(c) (partial)',p);
  8541. end;
  8542. taicpu(p).opcode:=A_SETcc;
  8543. taicpu(p).opsize:=S_B;
  8544. taicpu(p).is_jmp:=False;
  8545. if taicpu(hp1).opsize=S_B then
  8546. begin
  8547. taicpu(p).loadoper(0, taicpu(hp1).oper[1]^);
  8548. if taicpu(hp1).oper[1]^.typ = top_reg then
  8549. AllocRegBetween(taicpu(hp1).oper[1]^.reg, p, hp2, UsedRegs);
  8550. RemoveInstruction(hp1);
  8551. end
  8552. else
  8553. begin
  8554. { Will be a register because the size can't be S_B otherwise }
  8555. ThisReg := newreg(R_INTREGISTER,getsupreg(taicpu(hp1).oper[1]^.reg), R_SUBL);
  8556. taicpu(p).loadreg(0, ThisReg);
  8557. AllocRegBetween(ThisReg, p, hp2, UsedRegs);
  8558. if (cs_opt_size in current_settings.optimizerswitches) and IsMOVZXAcceptable then
  8559. begin
  8560. case taicpu(hp1).opsize of
  8561. S_W:
  8562. taicpu(hp1).opsize := S_BW;
  8563. S_L:
  8564. taicpu(hp1).opsize := S_BL;
  8565. {$ifdef x86_64}
  8566. S_Q:
  8567. begin
  8568. taicpu(hp1).opsize := S_BL;
  8569. { Change the destination register to 32-bit }
  8570. taicpu(hp1).loadreg(1, newreg(R_INTREGISTER,getsupreg(ThisReg), R_SUBD));
  8571. end;
  8572. {$endif x86_64}
  8573. else
  8574. InternalError(2021040820);
  8575. end;
  8576. taicpu(hp1).opcode := A_MOVZX;
  8577. taicpu(hp1).loadreg(0, ThisReg);
  8578. end
  8579. else
  8580. begin
  8581. AllocRegBetween(NR_FLAGS,p,hp1,UsedRegs);
  8582. { hp1 is already a MOV instruction with the correct register }
  8583. taicpu(hp1).loadconst(0, 0);
  8584. { Inserting it right before p will guarantee that the flags are also tracked }
  8585. asml.Remove(hp1);
  8586. asml.InsertBefore(hp1, p);
  8587. end;
  8588. end;
  8589. Result:=true;
  8590. exit;
  8591. end
  8592. else if MatchInstruction(hp1, A_CLC, A_STC, []) then
  8593. Result := TryJccStcClcOpt(p, hp1)
  8594. else if (hp1.typ = ait_label) then
  8595. Result := DoSETccLblRETOpt(p, tai_label(hp1));
  8596. end;
  8597. function TX86AsmOptimizer.OptPass1VMOVDQ(var p: tai): Boolean;
  8598. var
  8599. hp1, hp2, hp3: tai;
  8600. SourceRef, TargetRef: TReference;
  8601. CurrentReg: TRegister;
  8602. begin
  8603. { VMOVDQU/CMOVDQA shouldn't have even been generated }
  8604. if not UseAVX then
  8605. InternalError(2021100501);
  8606. Result := False;
  8607. { Look for the following to simplify:
  8608. vmovdqa/u x(mem1), %xmmreg
  8609. vmovdqa/u %xmmreg, y(mem2)
  8610. vmovdqa/u x+16(mem1), %xmmreg
  8611. vmovdqa/u %xmmreg, y+16(mem2)
  8612. Change to:
  8613. vmovdqa/u x(mem1), %ymmreg
  8614. vmovdqa/u %ymmreg, y(mem2)
  8615. vpxor %ymmreg, %ymmreg, %ymmreg
  8616. ( The VPXOR instruction is to zero the upper half, thus removing the
  8617. need to call the potentially expensive VZEROUPPER instruction. Other
  8618. peephole optimisations can remove VPXOR if it's unnecessary )
  8619. }
  8620. TransferUsedRegs(TmpUsedRegs);
  8621. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  8622. { NOTE: In the optimisations below, if the references dictate that an
  8623. aligned move is possible (i.e. VMOVDQA), the existing instructions
  8624. should already be VMOVDQA because if (x mod 32) = 0, then (x mod 16) = 0 }
  8625. if (taicpu(p).opsize = S_XMM) and
  8626. MatchOpType(taicpu(p), top_ref, top_reg) and
  8627. GetNextInstruction(p, hp1) and
  8628. MatchInstruction(hp1, A_VMOVDQA, A_VMOVDQU, [S_XMM]) and
  8629. MatchOpType(taicpu(hp1), top_reg, top_ref) and
  8630. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) then
  8631. begin
  8632. SourceRef := taicpu(p).oper[0]^.ref^;
  8633. TargetRef := taicpu(hp1).oper[1]^.ref^;
  8634. if GetNextInstruction(hp1, hp2) and
  8635. MatchInstruction(hp2, A_VMOVDQA, A_VMOVDQU, [S_XMM]) and
  8636. MatchOpType(taicpu(hp2), top_ref, top_reg) then
  8637. begin
  8638. { Delay calling GetNextInstruction(hp2, hp3) for as long as possible }
  8639. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  8640. Inc(SourceRef.offset, 16);
  8641. { Reuse the register in the first block move }
  8642. CurrentReg := newreg(R_MMREGISTER, getsupreg(taicpu(p).oper[1]^.reg), R_SUBMMY);
  8643. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) and
  8644. not RefsMightOverlap(taicpu(p).oper[0]^.ref^, TargetRef, 32) then
  8645. begin
  8646. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  8647. Inc(TargetRef.offset, 16);
  8648. if GetNextInstruction(hp2, hp3) and
  8649. MatchInstruction(hp3, A_VMOVDQA, A_VMOVDQU, [S_XMM]) and
  8650. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  8651. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  8652. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  8653. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  8654. begin
  8655. { Update the register tracking to the new size }
  8656. AllocRegBetween(CurrentReg, p, hp2, UsedRegs);
  8657. { Remember that the offsets are 16 ahead }
  8658. { Switch to unaligned if the memory isn't on a 32-byte boundary }
  8659. if not (
  8660. ((SourceRef.offset mod 32) = 16) and
  8661. (SourceRef.alignment >= 32) and ((SourceRef.alignment mod 32) = 0)
  8662. ) then
  8663. taicpu(p).opcode := A_VMOVDQU;
  8664. taicpu(p).opsize := S_YMM;
  8665. taicpu(p).oper[1]^.reg := CurrentReg;
  8666. if not (
  8667. ((TargetRef.offset mod 32) = 16) and
  8668. (TargetRef.alignment >= 32) and ((TargetRef.alignment mod 32) = 0)
  8669. ) then
  8670. taicpu(hp1).opcode := A_VMOVDQU;
  8671. taicpu(hp1).opsize := S_YMM;
  8672. taicpu(hp1).oper[0]^.reg := CurrentReg;
  8673. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(CurrentReg) + ' to merge a pair of memory moves (VmovdqxVmovdqxVmovdqxVmovdqx2VmovdqyVmovdqy 1)', p);
  8674. { If pi_uses_ymm is set, VZEROUPPER is present to do this for us }
  8675. if (pi_uses_ymm in current_procinfo.flags) then
  8676. RemoveInstruction(hp2)
  8677. else
  8678. begin
  8679. { Upper 128 bits will be set to zero; change to XMM
  8680. to avoid requirement of AVX2 }
  8681. setsubreg(CurrentReg, R_SUBMMX);
  8682. taicpu(hp2).opcode := A_VPXOR;
  8683. taicpu(hp2).opsize := S_XMM;
  8684. taicpu(hp2).loadreg(0, CurrentReg);
  8685. taicpu(hp2).loadreg(1, CurrentReg);
  8686. taicpu(hp2).loadreg(2, CurrentReg);
  8687. taicpu(hp2).ops := 3;
  8688. end;
  8689. RemoveInstruction(hp3);
  8690. Result := True;
  8691. Exit;
  8692. end;
  8693. end
  8694. else
  8695. begin
  8696. { See if the next references are 16 less rather than 16 greater }
  8697. Dec(SourceRef.offset, 32); { -16 the other way }
  8698. if RefsEqual(SourceRef, taicpu(hp2).oper[0]^.ref^) then
  8699. begin
  8700. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  8701. Dec(TargetRef.offset, 16); { Only 16, not 32, as it wasn't incremented unlike SourceRef }
  8702. if not RefsMightOverlap(SourceRef, TargetRef, 32) and
  8703. GetNextInstruction(hp2, hp3) and
  8704. MatchInstruction(hp3, A_MOV, [taicpu(p).opsize]) and
  8705. MatchOpType(taicpu(hp3), top_reg, top_ref) and
  8706. (taicpu(hp2).oper[1]^.reg = taicpu(hp3).oper[0]^.reg) and
  8707. RefsEqual(TargetRef, taicpu(hp3).oper[1]^.ref^) and
  8708. not RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp3, TmpUsedRegs) then
  8709. begin
  8710. { Update the register tracking to the new size }
  8711. AllocRegBetween(CurrentReg, hp2, hp3, UsedRegs);
  8712. { hp2 and hp3 are the starting offsets, so mod = 0 this time }
  8713. { Switch to unaligned if the memory isn't on a 32-byte boundary }
  8714. if not(
  8715. ((SourceRef.offset mod 32) = 0) and
  8716. (SourceRef.alignment >= 32) and ((SourceRef.alignment mod 32) = 0)
  8717. ) then
  8718. taicpu(hp2).opcode := A_VMOVDQU;
  8719. taicpu(hp2).opsize := S_YMM;
  8720. taicpu(hp2).oper[1]^.reg := CurrentReg;
  8721. if not (
  8722. ((TargetRef.offset mod 32) = 0) and
  8723. (TargetRef.alignment >= 32) and ((TargetRef.alignment mod 32) = 0)
  8724. ) then
  8725. taicpu(hp3).opcode := A_VMOVDQU;
  8726. taicpu(hp3).opsize := S_YMM;
  8727. taicpu(hp3).oper[0]^.reg := CurrentReg;
  8728. DebugMsg(SPeepholeOptimization + 'Used ' + debug_regname(CurrentReg) + ' to merge a pair of memory moves (VmovdqxVmovdqxVmovdqxVmovdqx2VmovdqyVmovdqy 2)', p);
  8729. { If pi_uses_ymm is set, VZEROUPPER is present to do this for us }
  8730. if (pi_uses_ymm in current_procinfo.flags) then
  8731. RemoveInstruction(hp1)
  8732. else
  8733. begin
  8734. { Upper 128 bits will be set to zero; change to
  8735. XMM to avoid requirement of AVX2 }
  8736. setsubreg(CurrentReg, R_SUBMMX);
  8737. taicpu(hp1).opcode := A_VPXOR;
  8738. taicpu(hp1).opsize := S_XMM;
  8739. taicpu(hp1).loadreg(0, CurrentReg);
  8740. taicpu(hp1).loadreg(1, CurrentReg);
  8741. taicpu(hp1).loadreg(2, CurrentReg);
  8742. taicpu(hp1).ops := 3;
  8743. Asml.Remove(hp1);
  8744. Asml.InsertAfter(hp1, hp3); { Register deallocations will be after hp3 }
  8745. end;
  8746. RemoveCurrentP(p, hp2);
  8747. Result := True;
  8748. Exit;
  8749. end;
  8750. end;
  8751. end;
  8752. end;
  8753. end;
  8754. end;
  8755. function TX86AsmOptimizer.CheckJumpMovTransferOpt(var p: tai; hp1: tai; LoopCount: Integer; out Count: Integer): Boolean;
  8756. var
  8757. hp2, hp3, first_assignment: tai;
  8758. IncCount, OperIdx: Integer;
  8759. OrigLabel: TAsmLabel;
  8760. begin
  8761. Count := 0;
  8762. Result := False;
  8763. first_assignment := nil;
  8764. if (LoopCount >= 20) then
  8765. begin
  8766. { Guard against infinite loops }
  8767. Exit;
  8768. end;
  8769. if (taicpu(p).oper[0]^.typ <> top_ref) or
  8770. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) or
  8771. (taicpu(p).oper[0]^.ref^.base <> NR_NO) or
  8772. (taicpu(p).oper[0]^.ref^.index <> NR_NO) or
  8773. not (taicpu(p).oper[0]^.ref^.symbol is TAsmLabel) then
  8774. Exit;
  8775. OrigLabel := TAsmLabel(taicpu(p).oper[0]^.ref^.symbol);
  8776. {
  8777. change
  8778. jmp .L1
  8779. ...
  8780. .L1:
  8781. mov ##, ## ( multiple movs possible )
  8782. jmp/ret
  8783. into
  8784. mov ##, ##
  8785. jmp/ret
  8786. }
  8787. if not Assigned(hp1) then
  8788. begin
  8789. hp1 := GetLabelWithSym(OrigLabel);
  8790. if not Assigned(hp1) or not SkipLabels(hp1, hp1) then
  8791. Exit;
  8792. end;
  8793. hp2 := hp1;
  8794. while Assigned(hp2) do
  8795. begin
  8796. if Assigned(hp2) and (hp2.typ = ait_label) then
  8797. SkipLabels(hp2,hp2);
  8798. if not Assigned(hp2) or (hp2.typ <> ait_instruction) then
  8799. Break;
  8800. case taicpu(hp2).opcode of
  8801. A_MOVSD:
  8802. begin
  8803. if taicpu(hp2).ops = 0 then
  8804. { Wrong MOVSD }
  8805. Break;
  8806. Inc(Count);
  8807. if Count >= 5 then
  8808. { Too many to be worthwhile }
  8809. Break;
  8810. GetNextInstruction(hp2, hp2);
  8811. Continue;
  8812. end;
  8813. A_MOV,
  8814. A_MOVD,
  8815. A_MOVQ,
  8816. A_MOVSX,
  8817. {$ifdef x86_64}
  8818. A_MOVSXD,
  8819. {$endif x86_64}
  8820. A_MOVZX,
  8821. A_MOVAPS,
  8822. A_MOVUPS,
  8823. A_MOVSS,
  8824. A_MOVAPD,
  8825. A_MOVUPD,
  8826. A_MOVDQA,
  8827. A_MOVDQU,
  8828. A_VMOVSS,
  8829. A_VMOVAPS,
  8830. A_VMOVUPS,
  8831. A_VMOVSD,
  8832. A_VMOVAPD,
  8833. A_VMOVUPD,
  8834. A_VMOVDQA,
  8835. A_VMOVDQU:
  8836. begin
  8837. Inc(Count);
  8838. if Count >= 5 then
  8839. { Too many to be worthwhile }
  8840. Break;
  8841. GetNextInstruction(hp2, hp2);
  8842. Continue;
  8843. end;
  8844. A_JMP:
  8845. begin
  8846. { Guard against infinite loops }
  8847. if taicpu(hp2).oper[0]^.ref^.symbol = OrigLabel then
  8848. Exit;
  8849. { Analyse this jump first in case it also duplicates assignments }
  8850. if CheckJumpMovTransferOpt(hp2, nil, LoopCount + 1, IncCount) then
  8851. begin
  8852. { Something did change! }
  8853. Result := True;
  8854. Inc(Count, IncCount);
  8855. if Count >= 5 then
  8856. begin
  8857. { Too many to be worthwhile }
  8858. Exit;
  8859. end;
  8860. if MatchInstruction(hp2, [A_JMP, A_RET], []) then
  8861. Break;
  8862. end;
  8863. Result := True;
  8864. Break;
  8865. end;
  8866. A_RET:
  8867. begin
  8868. Result := True;
  8869. Break;
  8870. end;
  8871. else
  8872. Break;
  8873. end;
  8874. end;
  8875. if Result then
  8876. begin
  8877. { A count of zero can happen when CheckJumpMovTransferOpt is called recursively }
  8878. if Count = 0 then
  8879. begin
  8880. Result := False;
  8881. Exit;
  8882. end;
  8883. TransferUsedRegs(TmpUsedRegs);
  8884. hp3 := p;
  8885. DebugMsg(SPeepholeOptimization + 'Duplicated ' + debug_tostr(Count) + ' assignment(s) and redirected jump', p);
  8886. while True do
  8887. begin
  8888. if Assigned(hp1) and (hp1.typ = ait_label) then
  8889. SkipLabels(hp1,hp1);
  8890. case hp1.typ of
  8891. ait_regalloc:
  8892. if tai_regalloc(hp1).ratype = ra_dealloc then
  8893. begin
  8894. { Duplicate the register deallocation... }
  8895. hp3:=tai(hp1.getcopy);
  8896. if first_assignment = nil then
  8897. first_assignment := hp3;
  8898. asml.InsertBefore(hp3, p);
  8899. { ... but also reallocate it after the jump }
  8900. hp3:=tai(hp1.getcopy);
  8901. tai_regalloc(hp3).ratype := ra_alloc;
  8902. asml.InsertAfter(hp3, p);
  8903. end;
  8904. ait_instruction:
  8905. case taicpu(hp1).opcode of
  8906. A_JMP:
  8907. begin
  8908. { Change the original jump to the new destination }
  8909. OrigLabel.decrefs;
  8910. taicpu(hp1).oper[0]^.ref^.symbol.increfs;
  8911. taicpu(p).loadref(0, taicpu(hp1).oper[0]^.ref^);
  8912. { Set p to the first duplicated assignment so it can get optimised if needs be }
  8913. if not Assigned(first_assignment) then
  8914. InternalError(2021040810)
  8915. else
  8916. p := first_assignment;
  8917. Exit;
  8918. end;
  8919. A_RET:
  8920. begin
  8921. { Now change the jump into a RET instruction }
  8922. ConvertJumpToRET(p, hp1);
  8923. { Set p to the first duplicated assignment so it can get optimised if needs be }
  8924. if not Assigned(first_assignment) then
  8925. InternalError(2021040811)
  8926. else
  8927. p := first_assignment;
  8928. Exit;
  8929. end;
  8930. else
  8931. begin
  8932. { Duplicate the MOV instruction }
  8933. hp3:=tai(hp1.getcopy);
  8934. if first_assignment = nil then
  8935. first_assignment := hp3;
  8936. asml.InsertBefore(hp3, p);
  8937. { Make sure the compiler knows about any final registers written here }
  8938. for OperIdx := 0 to taicpu(hp3).ops - 1 do
  8939. with taicpu(hp3).oper[OperIdx]^ do
  8940. begin
  8941. case typ of
  8942. top_ref:
  8943. begin
  8944. if (ref^.base <> NR_NO) and
  8945. (getsupreg(ref^.base) <> RS_STACK_POINTER_REG) and
  8946. (
  8947. (getsupreg(ref^.base) <> RS_FRAME_POINTER_REG) or
  8948. (
  8949. { Allow the frame pointer if it's not being used by the procedure as such }
  8950. Assigned(current_procinfo) and
  8951. (current_procinfo.framepointer <> NR_FRAME_POINTER_REG)
  8952. )
  8953. )
  8954. {$ifdef x86_64} and (ref^.base <> NR_RIP) {$endif x86_64}
  8955. then
  8956. begin
  8957. AllocRegBetween(ref^.base, hp3, p, TmpUsedRegs);
  8958. if not Assigned(first_assignment) then
  8959. IncludeRegInUsedRegs(ref^.base, UsedRegs);
  8960. end;
  8961. if (ref^.index <> NR_NO) and
  8962. (getsupreg(ref^.index) <> RS_STACK_POINTER_REG) and
  8963. (
  8964. (getsupreg(ref^.index) <> RS_FRAME_POINTER_REG) or
  8965. (
  8966. { Allow the frame pointer if it's not being used by the procedure as such }
  8967. Assigned(current_procinfo) and
  8968. (current_procinfo.framepointer <> NR_FRAME_POINTER_REG)
  8969. )
  8970. )
  8971. {$ifdef x86_64} and (ref^.index <> NR_RIP) {$endif x86_64} and
  8972. (ref^.index <> ref^.base) then
  8973. begin
  8974. AllocRegBetween(ref^.index, hp3, p, TmpUsedRegs);
  8975. if not Assigned(first_assignment) then
  8976. IncludeRegInUsedRegs(ref^.index, UsedRegs);
  8977. end;
  8978. end;
  8979. top_reg:
  8980. begin
  8981. AllocRegBetween(reg, hp3, p, TmpUsedRegs);
  8982. if not Assigned(first_assignment) then
  8983. IncludeRegInUsedRegs(reg, UsedRegs);
  8984. end;
  8985. else
  8986. ;
  8987. end;
  8988. end;
  8989. end;
  8990. end;
  8991. else
  8992. InternalError(2021040720);
  8993. end;
  8994. if not GetNextInstruction(hp1, hp1, [ait_regalloc]) then
  8995. { Should have dropped out earlier }
  8996. InternalError(2021040710);
  8997. end;
  8998. end;
  8999. end;
  9000. const
  9001. WriteOp: array[0..3] of set of TInsChange = (
  9002. [Ch_Wop1, Ch_RWop1, Ch_Mop1],
  9003. [Ch_Wop2, Ch_RWop2, Ch_Mop2],
  9004. [Ch_Wop3, Ch_RWop3, Ch_Mop3],
  9005. [Ch_Wop4, Ch_RWop4, Ch_Mop4]);
  9006. RegWriteFlags: array[0..7] of set of TInsChange = (
  9007. { The order is important: EAX, ECX, EDX, EBX, ESI, EDI, EBP, ESP }
  9008. [Ch_WEAX, Ch_RWEAX, Ch_MEAX{$ifdef x86_64}, Ch_WRAX, Ch_RWRAX, Ch_MRAX{$endif x86_64}],
  9009. [Ch_WECX, Ch_RWECX, Ch_MECX{$ifdef x86_64}, Ch_WRCX, Ch_RWRCX, Ch_MRCX{$endif x86_64}],
  9010. [Ch_WEDX, Ch_RWEDX, Ch_MEDX{$ifdef x86_64}, Ch_WRDX, Ch_RWRDX, Ch_MRDX{$endif x86_64}],
  9011. [Ch_WEBX, Ch_RWEBX, Ch_MEBX{$ifdef x86_64}, Ch_WRBX, Ch_RWRBX, Ch_MRBX{$endif x86_64}],
  9012. [Ch_WESI, Ch_RWESI, Ch_MESI{$ifdef x86_64}, Ch_WRSI, Ch_RWRSI, Ch_MRSI{$endif x86_64}],
  9013. [Ch_WEDI, Ch_RWEDI, Ch_MEDI{$ifdef x86_64}, Ch_WRDI, Ch_RWRDI, Ch_MRDI{$endif x86_64}],
  9014. [Ch_WEBP, Ch_RWEBP, Ch_MEBP{$ifdef x86_64}, Ch_WRBP, Ch_RWRBP, Ch_MRBP{$endif x86_64}],
  9015. [Ch_WESP, Ch_RWESP, Ch_MESP{$ifdef x86_64}, Ch_WRSP, Ch_RWRSP, Ch_MRSP{$endif x86_64}]);
  9016. function TX86AsmOptimizer.TrySwapMovOp(var p, hp1: tai): Boolean;
  9017. var
  9018. hp2: tai;
  9019. X: Integer;
  9020. begin
  9021. { If we have something like:
  9022. op ###,###
  9023. mov ###,###
  9024. Try to move the MOV instruction to before OP as long as OP and MOV don't
  9025. interfere in regards to what they write to.
  9026. NOTE: p must be a 2-operand instruction
  9027. }
  9028. Result := False;
  9029. if (hp1.typ <> ait_instruction) or
  9030. taicpu(hp1).is_jmp or
  9031. RegInInstruction(NR_DEFAULTFLAGS, hp1) then
  9032. Exit;
  9033. { NOP is a pipeline fence, likely marking the beginning of the function
  9034. epilogue, so drop out. Similarly, drop out if POP or RET are
  9035. encountered }
  9036. if MatchInstruction(hp1, A_NOP, A_POP, A_RET, []) then
  9037. Exit;
  9038. if (taicpu(hp1).opcode = A_MOVSD) and
  9039. (taicpu(hp1).ops = 0) then
  9040. { Wrong MOVSD }
  9041. Exit;
  9042. { Check for writes to specific registers first }
  9043. { EAX, ECX, EDX, EBX, ESI, EDI, EBP, ESP in that order }
  9044. for X := 0 to 7 do
  9045. if (RegWriteFlags[X] * InsProp[taicpu(hp1).opcode].Ch <> [])
  9046. and RegInInstruction(newreg(R_INTREGISTER, TSuperRegister(X), R_SUBWHOLE), p) then
  9047. Exit;
  9048. for X := 0 to taicpu(hp1).ops - 1 do
  9049. begin
  9050. { Check to see if this operand writes to something }
  9051. if ((WriteOp[X] * InsProp[taicpu(hp1).opcode].Ch) <> []) and
  9052. { And matches something in the CMP/TEST instruction }
  9053. (
  9054. MatchOperand(taicpu(hp1).oper[X]^, taicpu(p).oper[0]^) or
  9055. MatchOperand(taicpu(hp1).oper[X]^, taicpu(p).oper[1]^) or
  9056. (
  9057. { If it's a register, make sure the register written to doesn't
  9058. appear in the cmp instruction as part of a reference }
  9059. (taicpu(hp1).oper[X]^.typ = top_reg) and
  9060. RegInInstruction(taicpu(hp1).oper[X]^.reg, p)
  9061. )
  9062. ) then
  9063. Exit;
  9064. end;
  9065. { Check p to make sure it doesn't write to something that affects hp1 }
  9066. { Check for writes to specific registers first }
  9067. { EAX, ECX, EDX, EBX, ESI, EDI, EBP, ESP in that order }
  9068. for X := 0 to 7 do
  9069. if (RegWriteFlags[X] * InsProp[taicpu(p).opcode].Ch <> [])
  9070. and RegInInstruction(newreg(R_INTREGISTER, TSuperRegister(X), R_SUBWHOLE), hp1) then
  9071. Exit;
  9072. for X := 0 to taicpu(p).ops - 1 do
  9073. begin
  9074. { Check to see if this operand writes to something }
  9075. if ((WriteOp[X] * InsProp[taicpu(p).opcode].Ch) <> []) and
  9076. { And matches something in hp1 }
  9077. (taicpu(p).oper[X]^.typ = top_reg) and
  9078. RegInInstruction(taicpu(p).oper[X]^.reg, hp1) then
  9079. Exit;
  9080. end;
  9081. { The instruction can be safely moved }
  9082. asml.Remove(hp1);
  9083. { Try to insert after the last instructions where the FLAGS register is not
  9084. yet in use, so "mov $0,%reg" can be optimised into "xor %reg,%reg" later }
  9085. if SetAndTest(FindRegAllocBackward(NR_DEFAULTFLAGS, tai(p.Previous)), hp2) then
  9086. asml.InsertBefore(hp1, hp2)
  9087. { Failing that, try to insert after the last instructions where the
  9088. FLAGS register is not yet in use }
  9089. else if GetLastInstruction(p, hp2) and
  9090. (
  9091. (hp2.typ <> ait_instruction) or
  9092. { Don't insert after an instruction that uses the flags when p doesn't use them }
  9093. RegInInstruction(NR_DEFAULTFLAGS, p) or
  9094. not RegInInstruction(NR_DEFAULTFLAGS, hp2)
  9095. ) then
  9096. asml.InsertAfter(hp1, hp2)
  9097. else
  9098. { Note, if p.Previous is nil (even if it should logically never be the
  9099. case), FindRegAllocBackward immediately exits with False and so we
  9100. safely land here (we can't just pass p because FindRegAllocBackward
  9101. immediately exits on an instruction). [Kit] }
  9102. asml.InsertBefore(hp1, p);
  9103. DebugMsg(SPeepholeOptimization + 'Swapped ' + debug_op2str(taicpu(p).opcode) + ' and ' + debug_op2str(taicpu(hp1).opcode) + ' instructions to improve optimisation potential', hp1);
  9104. { We can't trust UsedRegs because we're looking backwards, although we
  9105. know the registers are allocated after p at the very least, so manually
  9106. create tai_regalloc objects if needed }
  9107. for X := 0 to taicpu(hp1).ops - 1 do
  9108. case taicpu(hp1).oper[X]^.typ of
  9109. top_reg:
  9110. begin
  9111. asml.InsertBefore(tai_regalloc.alloc(taicpu(hp1).oper[X]^.reg, nil), hp1);
  9112. IncludeRegInUsedRegs(taicpu(hp1).oper[X]^.reg, UsedRegs);
  9113. AllocRegBetween(taicpu(hp1).oper[X]^.reg, hp1, p, UsedRegs);
  9114. end;
  9115. top_ref:
  9116. begin
  9117. if taicpu(hp1).oper[X]^.ref^.base <> NR_NO then
  9118. begin
  9119. asml.InsertBefore(tai_regalloc.alloc(taicpu(hp1).oper[X]^.ref^.base, nil), hp1);
  9120. IncludeRegInUsedRegs(taicpu(hp1).oper[X]^.ref^.base, UsedRegs);
  9121. AllocRegBetween(taicpu(hp1).oper[X]^.ref^.base, hp1, p, UsedRegs);
  9122. end;
  9123. if taicpu(hp1).oper[X]^.ref^.index <> NR_NO then
  9124. begin
  9125. asml.InsertBefore(tai_regalloc.alloc(taicpu(hp1).oper[X]^.ref^.index, nil), hp1);
  9126. IncludeRegInUsedRegs(taicpu(hp1).oper[X]^.ref^.index, UsedRegs);
  9127. AllocRegBetween(taicpu(hp1).oper[X]^.ref^.index, hp1, p, UsedRegs);
  9128. end;
  9129. end;
  9130. else
  9131. ;
  9132. end;
  9133. Result := True;
  9134. end;
  9135. function TX86AsmOptimizer.TrySwapMovCmp(var p, hp1: tai): Boolean;
  9136. var
  9137. hp2: tai;
  9138. X: Integer;
  9139. begin
  9140. { If we have something like:
  9141. cmp ###,%reg1
  9142. mov 0,%reg2
  9143. And no modified registers are shared, move the instruction to before
  9144. the comparison as this means it can be optimised without worrying
  9145. about the FLAGS register. (CMP/MOV is generated by
  9146. "J(c)Mov1JmpMov0 -> Set(~c)", among other things).
  9147. As long as the second instruction doesn't use the flags or one of the
  9148. registers used by CMP or TEST (also check any references that use the
  9149. registers), then it can be moved prior to the comparison.
  9150. }
  9151. Result := False;
  9152. if not TrySwapMovOp(p, hp1) then
  9153. Exit;
  9154. if taicpu(hp1).opcode = A_LEA then
  9155. { The flags will be overwritten by the CMP/TEST instruction }
  9156. ConvertLEA(taicpu(hp1));
  9157. Result := True;
  9158. { Can we move it one further back? }
  9159. if GetLastInstruction(hp1, hp2) and (hp2.typ = ait_instruction) and
  9160. { Check to see if CMP/TEST is a comparison against zero }
  9161. (
  9162. (
  9163. (taicpu(p).opcode = A_CMP) and
  9164. MatchOperand(taicpu(p).oper[0]^, 0)
  9165. ) or
  9166. (
  9167. (taicpu(p).opcode = A_TEST) and
  9168. (
  9169. OpsEqual(taicpu(p).oper[0]^, taicpu(p).oper[1]^) or
  9170. MatchOperand(taicpu(p).oper[0]^, -1)
  9171. )
  9172. )
  9173. ) and
  9174. { These instructions set the zero flag if the result is zero }
  9175. MatchInstruction(hp2, [A_ADD, A_SUB, A_OR, A_XOR, A_AND, A_POPCNT, A_LZCNT], []) and
  9176. OpsEqual(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^) then
  9177. { Looks like we can - if successful, this benefits PostPeepholeOptTestOr }
  9178. TrySwapMovOp(hp2, hp1);
  9179. end;
  9180. function TX86AsmOptimizer.OptPass1STCCLC(var p: tai): Boolean;
  9181. var
  9182. hp1, hp2, p_last, p_dist, hp1_dist: tai;
  9183. JumpLabel: TAsmLabel;
  9184. TmpBool: Boolean;
  9185. begin
  9186. Result := False;
  9187. { Look for:
  9188. stc/clc
  9189. j(c) .L1
  9190. ...
  9191. .L1:
  9192. set(n)cb %reg
  9193. (flags deallocated)
  9194. j(c) .L2
  9195. Change to:
  9196. mov $0/$1,%reg (depending on if the carry bit is cleared or not)
  9197. j(c) .L2
  9198. }
  9199. p_last := p;
  9200. while GetNextInstruction(p_last, hp1) and
  9201. (hp1.typ = ait_instruction) and
  9202. IsJumpToLabel(taicpu(hp1)) do
  9203. begin
  9204. if DoJumpOptimizations(hp1, TmpBool) then
  9205. { Re-evaluate from p_last. Probably could be faster, but it's guaranteed to be correct }
  9206. Continue;
  9207. JumpLabel := TAsmLabel(taicpu(hp1).oper[0]^.ref^.symbol);
  9208. if not Assigned(JumpLabel) then
  9209. InternalError(2024012801);
  9210. { Optimise the J(c); stc/clc optimisation first since this will
  9211. get missed if the main optimisation takes place }
  9212. if (taicpu(hp1).opcode = A_JCC) then
  9213. begin
  9214. if GetNextInstruction(hp1, hp2) and
  9215. MatchInstruction(hp2, A_CLC, A_STC, []) and
  9216. TryJccStcClcOpt(hp1, hp2) then
  9217. begin
  9218. Result := True;
  9219. Exit;
  9220. end;
  9221. hp2 := nil; { Suppress compiler warning }
  9222. if (taicpu(hp1).condition in [C_C, C_NC]) and
  9223. { Make sure the flags aren't used again }
  9224. SetAndTest(FindRegDealloc(NR_DEFAULTFLAGS, tai(hp1.Next)), hp2) then
  9225. begin
  9226. { clc + jc = False; clc + jnc = True; stc + jc = True; stc + jnc = False }
  9227. if ((taicpu(p).opcode = A_STC) xor (taicpu(hp1).condition = C_NC)) then
  9228. begin
  9229. if (taicpu(p).opcode = A_STC) then
  9230. DebugMsg(SPeepholeOptimization + 'STC; JC -> JMP (Deterministic jump) (StcJc2Jmp)', p)
  9231. else
  9232. DebugMsg(SPeepholeOptimization + 'CLC; JNC -> JMP (Deterministic jump) (ClcJnc2Jmp)', p);
  9233. MakeUnconditional(taicpu(hp1));
  9234. { Move the jump to after the flag deallocations }
  9235. Asml.Remove(hp1);
  9236. Asml.InsertAfter(hp1, hp2);
  9237. RemoveCurrentP(p); { hp1 may not be the immediate next instruction }
  9238. Result := True;
  9239. Exit;
  9240. end
  9241. else
  9242. begin
  9243. if (taicpu(p).opcode = A_STC) then
  9244. DebugMsg(SPeepholeOptimization + 'STC; JNC -> NOP (Deterministic jump) (StcJnc2Nop)', p)
  9245. else
  9246. DebugMsg(SPeepholeOptimization + 'CLC; JC -> NOP (Deterministic jump) (ClcJc2Nop)', p);
  9247. { In this case, the jump is deterministic in that it will never be taken }
  9248. JumpLabel.DecRefs;
  9249. RemoveInstruction(hp1);
  9250. RemoveCurrentP(p); { hp1 may not have been the immediate next instruction }
  9251. Result := True;
  9252. Exit;
  9253. end;
  9254. end;
  9255. end;
  9256. hp2 := nil; { Suppress compiler warning }
  9257. if
  9258. { Make sure the carry flag doesn't appear in the jump conditions }
  9259. not (taicpu(hp1).condition in [C_AE, C_NB, C_NC, C_B, C_C, C_NAE, C_BE, C_NA]) and
  9260. SetAndTest(getlabelwithsym(JumpLabel), hp2) and
  9261. GetNextInstruction(hp2, p_dist) and
  9262. MatchInstruction(p_dist, A_Jcc, A_SETcc, []) and
  9263. (taicpu(p_dist).condition in [C_C, C_NC]) then
  9264. begin
  9265. case taicpu(p_dist).opcode of
  9266. A_Jcc:
  9267. begin
  9268. if DoJumpOptimizations(p_dist, TmpBool) then
  9269. { Re-evaluate from p_last. Probably could be faster, but it's guaranteed to be correct }
  9270. Continue;
  9271. { clc + jc = False; clc + jnc = True; stc + jc = True; stc + jnc = False }
  9272. if ((taicpu(p).opcode = A_STC) xor (taicpu(p_dist).condition = C_NC)) then
  9273. begin
  9274. DebugMsg(SPeepholeOptimization + 'STC/CLC; JMP/Jcc; ... J(N)C -> JMP/Jcc (StcClcJ(c)2Jmp)', p);
  9275. JumpLabel.decrefs;
  9276. taicpu(hp1).loadsymbol(0, taicpu(p_dist).oper[0]^.ref^.symbol, 0);
  9277. RemoveCurrentP(p); { hp1 may not be the immediate next instruction }
  9278. Result := True;
  9279. Exit;
  9280. end
  9281. else if GetNextInstruction(p_dist, hp1_dist) and
  9282. (hp1_dist.typ = ait_label) then
  9283. begin
  9284. DebugMsg(SPeepholeOptimization + 'STC/CLC; JMP/Jcc; ... J(N)C; .Lbl -> JMP/Jcc .Lbl (StcClcJ(~c)Lbl2Jmp)', p);
  9285. JumpLabel.decrefs;
  9286. taicpu(hp1).loadsymbol(0, tai_label(hp1_dist).labsym, 0);
  9287. RemoveCurrentP(p); { hp1 may not be the immediate next instruction }
  9288. Result := True;
  9289. Exit;
  9290. end;
  9291. end;
  9292. A_SETcc:
  9293. if { Make sure the flags aren't used again }
  9294. SetAndTest(FindRegDealloc(NR_DEFAULTFLAGS, tai(p_dist.Next)), hp2) and
  9295. GetNextInstruction(hp2, hp1_dist) and
  9296. (hp1_dist.typ = ait_instruction) and
  9297. IsJumpToLabel(taicpu(hp1_dist)) and
  9298. not (taicpu(hp1_dist).condition in [C_AE, C_NB, C_NC, C_B, C_C, C_NAE, C_BE, C_NA]) and
  9299. { This works if hp1_dist or both are regular JMP instructions }
  9300. condition_in(taicpu(hp1).condition, taicpu(hp1_dist).condition) and
  9301. (
  9302. (taicpu(p_dist).oper[0]^.typ <> top_reg) or
  9303. { Make sure the register isn't still in use, otherwise it
  9304. may get corrupted (fixes #40659) }
  9305. not RegUsedBetween(taicpu(p_dist).oper[0]^.reg, p, p_dist)
  9306. ) then
  9307. begin
  9308. taicpu(p).allocate_oper(2);
  9309. taicpu(p).ops := 2;
  9310. { clc + setc = 0; clc + setnc = 1; stc + setc = 1; stc + setnc = 0 }
  9311. taicpu(p).loadconst(0, TCGInt((taicpu(p).opcode = A_STC) xor (taicpu(p_dist).condition = C_NC)));
  9312. taicpu(p).loadoper(1, taicpu(p_dist).oper[0]^);
  9313. taicpu(p).opcode := A_MOV;
  9314. taicpu(p).opsize := S_B;
  9315. if (taicpu(p_dist).oper[0]^.typ = top_reg) then
  9316. AllocRegBetween(taicpu(p_dist).oper[0]^.reg, p, hp1, UsedRegs);
  9317. DebugMsg(SPeepholeOptimization + 'STC/CLC; JMP; ... SET(N)C; JMP -> MOV; JMP (StcClcSet(c)2Mov)', p);
  9318. JumpLabel.decrefs;
  9319. taicpu(hp1).loadsymbol(0, taicpu(hp1_dist).oper[0]^.ref^.symbol, 0);
  9320. { If a flag allocation is found, try to move it to after the MOV so "mov $0,%reg" gets optimised to "xor %reg,%reg" }
  9321. if SetAndTest(FindRegAllocBackward(NR_DEFAULTFLAGS, tai(p.Previous)), hp2) and
  9322. (tai_regalloc(hp2).ratype = ra_alloc) then
  9323. begin
  9324. Asml.Remove(hp2);
  9325. Asml.InsertAfter(hp2, p);
  9326. end;
  9327. Result := True;
  9328. Exit;
  9329. end;
  9330. else
  9331. ;
  9332. end;
  9333. end;
  9334. p_last := hp1;
  9335. end;
  9336. end;
  9337. function TX86AsmOptimizer.TryJccStcClcOpt(var p, hp1: tai): Boolean;
  9338. var
  9339. hp2, hp3: tai;
  9340. TempBool: Boolean;
  9341. begin
  9342. Result := False;
  9343. {
  9344. j(c) .L1
  9345. stc/clc
  9346. .L1:
  9347. jc/jnc .L2
  9348. (Flags deallocated)
  9349. Change to:
  9350. j)c) .L1
  9351. jmp .L2
  9352. .L1:
  9353. jc/jnc .L2
  9354. Then call DoJumpOptimizations to convert to:
  9355. j(nc) .L2
  9356. .L1: (may become a dead label)
  9357. jc/jnc .L2
  9358. }
  9359. if GetNextInstruction(hp1, hp2) and
  9360. (hp2.typ = ait_label) and
  9361. (tai_label(hp2).labsym = TAsmLabel(taicpu(p).oper[0]^.ref^.symbol)) and
  9362. GetNextInstruction(hp2, hp3) and
  9363. MatchInstruction(hp3, A_Jcc, []) and
  9364. (
  9365. (
  9366. (taicpu(hp3).condition = C_C) and
  9367. (taicpu(hp1).opcode = A_STC)
  9368. ) or (
  9369. (taicpu(hp3).condition = C_NC) and
  9370. (taicpu(hp1).opcode = A_CLC)
  9371. )
  9372. ) and
  9373. { Make sure the flags aren't used again }
  9374. Assigned(FindRegDealloc(NR_DEFAULTFLAGS, tai(hp3.Next))) then
  9375. begin
  9376. taicpu(hp1).allocate_oper(1);
  9377. taicpu(hp1).ops := 1;
  9378. taicpu(hp1).loadsymbol(0, TAsmLabel(taicpu(hp3).oper[0]^.ref^.symbol), 0);
  9379. taicpu(hp1).opcode := A_JMP;
  9380. taicpu(hp1).is_jmp := True;
  9381. TempBool := True; { Prevent compiler warnings }
  9382. if DoJumpOptimizations(p, TempBool) then
  9383. Result := True
  9384. else
  9385. Include(OptsToCheck, aoc_ForceNewIteration);
  9386. end;
  9387. end;
  9388. function TX86AsmOptimizer.OptPass2STCCLC(var p: tai): Boolean;
  9389. begin
  9390. { This generally only executes under -O3 and above }
  9391. Result := (aoc_DoPass2JccOpts in OptsToCheck) and OptPass1STCCLC(p);
  9392. end;
  9393. function TX86AsmOptimizer.OptPass2CMOVcc(var p: tai): Boolean;
  9394. var
  9395. hp1, hp2: tai;
  9396. FoundComparison: Boolean;
  9397. begin
  9398. { Run the pass 1 optimisations as well, since they may have some effect
  9399. after the CMOV blocks are created in OptPass2Jcc }
  9400. Result := False;
  9401. { Result := OptPass1CMOVcc(p);
  9402. if Result then
  9403. Exit;}
  9404. { Sometimes, the CMOV optimisations in OptPass2Jcc are a bit overzealous
  9405. and make a slightly inefficent result on branching-type blocks, notably
  9406. when setting a function result then jumping to the function epilogue.
  9407. In this case, change:
  9408. cmov(c) %reg1,%reg2
  9409. j(c) @lbl
  9410. (%reg2 deallocated)
  9411. To:
  9412. mov %reg11,%reg2
  9413. j(c) @lbl
  9414. Note, we can't use GetNextInstructionUsingReg to find the conditional
  9415. jump because if it's not present, we may end up with a jump that's
  9416. completely unrelated.
  9417. }
  9418. hp1 := p;
  9419. while GetNextInstruction(hp1, hp1) and
  9420. MatchInstruction(hp1, A_MOV, A_CMOVcc, []) do { loop };
  9421. if (hp1.typ = ait_instruction) and
  9422. (taicpu(hp1).opcode = A_Jcc) and
  9423. condition_in(taicpu(hp1).condition, taicpu(p).condition) then
  9424. begin
  9425. TransferUsedRegs(TmpUsedRegs);
  9426. UpdateUsedRegsBetween(TmpUsedRegs, p, hp1);
  9427. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) or
  9428. (
  9429. { See if we can find a more distant instruction that overwrites
  9430. the destination register }
  9431. (cs_opt_level3 in current_settings.optimizerswitches) and
  9432. GetNextInstructionUsingReg(hp1, hp2, taicpu(p).oper[1]^.reg) and
  9433. RegLoadedWithNewValue(taicpu(p).oper[1]^.reg, hp2)
  9434. ) then
  9435. begin
  9436. if (taicpu(p).oper[0]^.typ = top_reg) then
  9437. begin
  9438. { Search backwards to see if the source register is set to a
  9439. constant }
  9440. FoundComparison := False;
  9441. hp1 := p;
  9442. while GetLastInstruction(hp1, hp1) and (hp1.typ = ait_instruction) do
  9443. begin
  9444. if RegModifiedByInstruction(NR_DEFAULTFLAGS, hp1) then
  9445. begin
  9446. FoundComparison := True;
  9447. Continue;
  9448. end;
  9449. { Once we find the CMP, TEST or similar instruction, we
  9450. have to stop if we find anything other than a MOV }
  9451. if FoundComparison and (taicpu(hp1).opcode <> A_MOV) then
  9452. Break;
  9453. if RegModifiedByInstruction(taicpu(p).oper[1]^.reg, hp1) then
  9454. { Destination register was modified }
  9455. Break;
  9456. if (taicpu(hp1).opcode = A_MOV) and MatchOpType(taicpu(hp1), top_const, toP_reg)
  9457. and (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[0]^.reg) then
  9458. begin
  9459. { Found a constant! }
  9460. taicpu(p).loadconst(0, taicpu(hp1).oper[0]^.val);
  9461. if not RegUsedAfterInstruction(taicpu(hp1).oper[1]^.reg, p, UsedRegs) then
  9462. { The source register is no longer in use }
  9463. RemoveInstruction(hp1);
  9464. Break;
  9465. end;
  9466. if RegModifiedByInstruction(taicpu(p).oper[0]^.reg, hp1) then
  9467. { Some other instruction has modified the source register }
  9468. Break;
  9469. end;
  9470. end;
  9471. DebugMsg(SPeepholeOptimization + 'CMOVcc/Jcc -> MOV/Jcc since register is not used if not branching', p);
  9472. taicpu(p).opcode := A_MOV;
  9473. taicpu(p).condition := C_None;
  9474. { Rely on the post peephole stage to put the MOV before the
  9475. CMP/TEST instruction that appears prior }
  9476. Result := True;
  9477. Exit;
  9478. end;
  9479. end;
  9480. end;
  9481. function TX86AsmOptimizer.OptPass2MOV(var p : tai) : boolean;
  9482. function IsXCHGAcceptable: Boolean; inline;
  9483. begin
  9484. { Always accept if optimising for size }
  9485. Result := (cs_opt_size in current_settings.optimizerswitches) or
  9486. { From the Pentium M onwards, XCHG only has a latency of 2 rather
  9487. than 3, so it becomes a saving compared to three MOVs with two of
  9488. them able to execute simultaneously. [Kit] }
  9489. (CPUX86_HINT_FAST_XCHG in cpu_optimization_hints[current_settings.optimizecputype]);
  9490. end;
  9491. var
  9492. NewRef: TReference;
  9493. hp1, hp2, hp3, hp4: Tai;
  9494. {$ifndef x86_64}
  9495. OperIdx: Integer;
  9496. {$endif x86_64}
  9497. NewInstr : Taicpu;
  9498. NewAligh : Tai_align;
  9499. DestLabel: TAsmLabel;
  9500. TempTracking: TAllUsedRegs;
  9501. function TryMovArith2Lea(InputInstr: tai): Boolean;
  9502. var
  9503. NextInstr: tai;
  9504. begin
  9505. Result := False;
  9506. UpdateUsedRegs(TmpUsedRegs, tai(InputInstr.Next));
  9507. if not GetNextInstruction(InputInstr, NextInstr) or
  9508. (
  9509. { The FLAGS register isn't always tracked properly, so do not
  9510. perform this optimisation if a conditional statement follows }
  9511. not RegReadByInstruction(NR_DEFAULTFLAGS, NextInstr) and
  9512. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, NextInstr, TmpUsedRegs)
  9513. ) then
  9514. begin
  9515. reference_reset(NewRef, 1, []);
  9516. NewRef.base := taicpu(p).oper[0]^.reg;
  9517. NewRef.scalefactor := 1;
  9518. if taicpu(InputInstr).opcode = A_ADD then
  9519. begin
  9520. DebugMsg(SPeepholeOptimization + 'MovAdd2Lea', p);
  9521. NewRef.offset := taicpu(InputInstr).oper[0]^.val;
  9522. end
  9523. else
  9524. begin
  9525. DebugMsg(SPeepholeOptimization + 'MovSub2Lea', p);
  9526. NewRef.offset := -taicpu(InputInstr).oper[0]^.val;
  9527. end;
  9528. taicpu(p).opcode := A_LEA;
  9529. taicpu(p).loadref(0, NewRef);
  9530. { For the sake of debugging, have the line info match the
  9531. arithmetic instruction rather than the MOV instruction }
  9532. taicpu(p).fileinfo := taicpu(InputInstr).fileinfo;
  9533. RemoveInstruction(InputInstr);
  9534. Result := True;
  9535. end;
  9536. end;
  9537. begin
  9538. Result:=false;
  9539. { This optimisation adds an instruction, so only do it for speed }
  9540. if not (cs_opt_size in current_settings.optimizerswitches) and
  9541. MatchOpType(taicpu(p), top_const, top_reg) and
  9542. (taicpu(p).oper[0]^.val = 0) then
  9543. begin
  9544. { To avoid compiler warning }
  9545. DestLabel := nil;
  9546. if (p.typ <> ait_instruction) or (taicpu(p).oper[1]^.typ <> top_reg) then
  9547. InternalError(2021040750);
  9548. if not GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[1]^.reg) then
  9549. Exit;
  9550. case hp1.typ of
  9551. ait_label:
  9552. begin
  9553. { Change:
  9554. mov $0,%reg mov $0,%reg
  9555. @Lbl1: @Lbl1:
  9556. test %reg,%reg / cmp $0,%reg test %reg,%reg / mov $0,%reg
  9557. je @Lbl2 jne @Lbl2
  9558. To: To:
  9559. mov $0,%reg mov $0,%reg
  9560. jmp @Lbl2 jmp @Lbl3
  9561. (align) (align)
  9562. @Lbl1: @Lbl1:
  9563. test %reg,%reg / cmp $0,%reg test %reg,%reg / cmp $0,%reg
  9564. je @Lbl2 je @Lbl2
  9565. @Lbl3: <-- Only if label exists
  9566. (Not if it's optimised for size)
  9567. }
  9568. if not GetNextInstruction(hp1, hp2) then
  9569. Exit;
  9570. if (hp2.typ = ait_instruction) and
  9571. (
  9572. { Register sizes must exactly match }
  9573. (
  9574. (taicpu(hp2).opcode = A_CMP) and
  9575. MatchOperand(taicpu(hp2).oper[0]^, 0) and
  9576. MatchOperand(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^.reg)
  9577. ) or (
  9578. (taicpu(hp2).opcode = A_TEST) and
  9579. MatchOperand(taicpu(hp2).oper[0]^, taicpu(p).oper[1]^.reg) and
  9580. MatchOperand(taicpu(hp2).oper[1]^, taicpu(p).oper[1]^.reg)
  9581. )
  9582. ) and GetNextInstruction(hp2, hp3) and
  9583. (hp3.typ = ait_instruction) and
  9584. (taicpu(hp3).opcode = A_JCC) and
  9585. (taicpu(hp3).oper[0]^.typ=top_ref) and (taicpu(hp3).oper[0]^.ref^.refaddr=addr_full) and (taicpu(hp3).oper[0]^.ref^.base=NR_NO) and
  9586. (taicpu(hp3).oper[0]^.ref^.index=NR_NO) and (taicpu(hp3).oper[0]^.ref^.symbol is tasmlabel) then
  9587. begin
  9588. { Check condition of jump }
  9589. { Always true? }
  9590. if condition_in(C_E, taicpu(hp3).condition) then
  9591. begin
  9592. { Copy label symbol and obtain matching label entry for the
  9593. conditional jump, as this will be our destination}
  9594. DestLabel := tasmlabel(taicpu(hp3).oper[0]^.ref^.symbol);
  9595. DebugMsg(SPeepholeOptimization + 'Mov0LblCmp0Je -> Mov0JmpLblCmp0Je', p);
  9596. Result := True;
  9597. end
  9598. { Always false? }
  9599. else if condition_in(C_NE, taicpu(hp3).condition) and GetNextInstruction(hp3, hp2) then
  9600. begin
  9601. { This is only worth it if there's a jump to take }
  9602. case hp2.typ of
  9603. ait_instruction:
  9604. begin
  9605. if taicpu(hp2).opcode = A_JMP then
  9606. begin
  9607. DestLabel := tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol);
  9608. { An unconditional jump follows the conditional jump which will always be false,
  9609. so use this jump's destination for the new jump }
  9610. DebugMsg(SPeepholeOptimization + 'Mov0LblCmp0Jne -> Mov0JmpLblCmp0Jne (with JMP)', p);
  9611. Result := True;
  9612. end
  9613. else if taicpu(hp2).opcode = A_JCC then
  9614. begin
  9615. DestLabel := tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol);
  9616. if condition_in(C_E, taicpu(hp2).condition) then
  9617. begin
  9618. { A second conditional jump follows the conditional jump which will always be false,
  9619. while the second jump is always True, so use this jump's destination for the new jump }
  9620. DebugMsg(SPeepholeOptimization + 'Mov0LblCmp0Jne -> Mov0JmpLblCmp0Jne (with second Jcc)', p);
  9621. Result := True;
  9622. end;
  9623. { Don't risk it if the jump isn't always true (Result remains False) }
  9624. end;
  9625. end;
  9626. else
  9627. { If anything else don't optimise };
  9628. end;
  9629. end;
  9630. if Result then
  9631. begin
  9632. { Just so we have something to insert as a paremeter}
  9633. reference_reset(NewRef, 1, []);
  9634. NewInstr := taicpu.op_ref(A_JMP, S_NO, NewRef);
  9635. { Now actually load the correct parameter (this also
  9636. increases the reference count) }
  9637. NewInstr.loadsymbol(0, DestLabel, 0);
  9638. if (cs_opt_level3 in current_settings.optimizerswitches) then
  9639. begin
  9640. { Get instruction before original label (may not be p under -O3) }
  9641. if not GetLastInstruction(hp1, hp2) then
  9642. { Shouldn't fail here }
  9643. InternalError(2021040701);
  9644. end
  9645. else
  9646. hp2 := p;
  9647. taicpu(NewInstr).fileinfo := taicpu(hp2).fileinfo;
  9648. AsmL.InsertAfter(NewInstr, hp2);
  9649. { Add new alignment field }
  9650. (* AsmL.InsertAfter(
  9651. cai_align.create_max(
  9652. current_settings.alignment.jumpalign,
  9653. current_settings.alignment.jumpalignskipmax
  9654. ),
  9655. NewInstr
  9656. ); *)
  9657. end;
  9658. Exit;
  9659. end;
  9660. end;
  9661. else
  9662. ;
  9663. end;
  9664. end;
  9665. if not GetNextInstruction(p, hp1) then
  9666. Exit;
  9667. if MatchInstruction(hp1, A_CMP, A_TEST, []) then
  9668. begin
  9669. if (taicpu(hp1).opsize = taicpu(p).opsize) and DoMovCmpMemOpt(p, hp1) then
  9670. begin
  9671. Result := True;
  9672. Exit;
  9673. end;
  9674. { This optimisation is only effective on a second run of Pass 2,
  9675. hence -O3 or above.
  9676. Change:
  9677. mov %reg1,%reg2
  9678. cmp/test (contains %reg1)
  9679. mov x, %reg1
  9680. (another mov or a j(c))
  9681. To:
  9682. mov %reg1,%reg2
  9683. mov x, %reg1
  9684. cmp (%reg1 replaced with %reg2)
  9685. (another mov or a j(c))
  9686. The requirement of an additional MOV or a jump ensures there
  9687. isn't performance loss, since a j(c) will permit macro-fusion
  9688. with the cmp instruction, while another MOV likely means it's
  9689. not all being executed in a single cycle due to parallelisation.
  9690. }
  9691. if (cs_opt_level3 in current_settings.optimizerswitches) and
  9692. MatchOpType(taicpu(p), top_reg, top_reg) and
  9693. RegInInstruction(taicpu(p).oper[0]^.reg, taicpu(hp1)) and
  9694. GetNextInstruction(hp1, hp2) and
  9695. MatchInstruction(hp2, A_MOV, []) and
  9696. (taicpu(hp2).oper[1]^.typ = top_reg) and
  9697. { Registers don't have to be the same size in this case }
  9698. SuperRegistersEqual(taicpu(hp2).oper[1]^.reg, taicpu(p).oper[0]^.reg) and
  9699. GetNextInstruction(hp2, hp3) and
  9700. MatchInstruction(hp3, A_MOV, A_Jcc, []) and
  9701. { Make sure the operands in the camparison can be safely replaced }
  9702. (
  9703. not RegInOp(taicpu(p).oper[0]^.reg, taicpu(hp1).oper[0]^) or
  9704. ReplaceRegisterInOper(taicpu(hp1), 0, taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg)
  9705. ) and
  9706. (
  9707. not RegInOp(taicpu(p).oper[0]^.reg, taicpu(hp1).oper[1]^) or
  9708. ReplaceRegisterInOper(taicpu(hp1), 1, taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg)
  9709. ) then
  9710. begin
  9711. DebugMsg(SPeepholeOptimization + 'MOV/CMP/MOV -> MOV/MOV/CMP', p);
  9712. AsmL.Remove(hp2);
  9713. AsmL.InsertAfter(hp2, p);
  9714. Result := True;
  9715. Exit;
  9716. end;
  9717. end;
  9718. if MatchInstruction(hp1, A_JMP, [S_NO]) then
  9719. begin
  9720. { Sometimes the MOVs that OptPass2JMP produces can be improved
  9721. further, but we can't just put this jump optimisation in pass 1
  9722. because it tends to perform worse when conditional jumps are
  9723. nearby (e.g. when converting CMOV instructions). [Kit] }
  9724. CopyUsedRegs(TempTracking);
  9725. UpdateUsedRegs(tai(p.Next));
  9726. if OptPass2JMP(hp1) then
  9727. begin
  9728. { Restore register state }
  9729. RestoreUsedRegs(TempTracking);
  9730. ReleaseUsedRegs(TempTracking);
  9731. { call OptPass1MOV once to potentially merge any MOVs that were created }
  9732. OptPass1MOV(p);
  9733. Result := True;
  9734. Exit;
  9735. end;
  9736. { If OptPass2JMP returned False, no optimisations were done to
  9737. the jump and there are no further optimisations that can be done
  9738. to the MOV instruction on this pass other than FuncMov2Func }
  9739. { Restore register state }
  9740. RestoreUsedRegs(TempTracking);
  9741. ReleaseUsedRegs(TempTracking);
  9742. Result := FuncMov2Func(p, hp1);
  9743. Exit;
  9744. end;
  9745. if MatchOpType(taicpu(p),top_reg,top_reg) and
  9746. (taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) and
  9747. MatchInstruction(hp1,A_ADD,A_SUB,[taicpu(p).opsize]) and
  9748. (taicpu(hp1).oper[1]^.typ = top_reg) and
  9749. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  9750. begin
  9751. { Change:
  9752. movl/q %reg1,%reg2 movl/q %reg1,%reg2
  9753. addl/q $x,%reg2 subl/q $x,%reg2
  9754. To:
  9755. leal/q x(%reg1),%reg2 leal/q -x(%reg1),%reg2
  9756. }
  9757. if (taicpu(hp1).oper[0]^.typ = top_const) and
  9758. { be lazy, checking separately for sub would be slightly better }
  9759. (abs(taicpu(hp1).oper[0]^.val)<=$7fffffff) then
  9760. begin
  9761. TransferUsedRegs(TmpUsedRegs);
  9762. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  9763. if TryMovArith2Lea(hp1) then
  9764. begin
  9765. Result := True;
  9766. Exit;
  9767. end
  9768. end
  9769. else if not RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^) and
  9770. GetNextInstructionUsingReg(hp1, hp2, taicpu(p).oper[1]^.reg) and
  9771. { Same as above, but also adds or subtracts to %reg2 in between.
  9772. It's still valid as long as the flags aren't in use }
  9773. MatchInstruction(hp2,A_ADD,A_SUB,[taicpu(p).opsize]) and
  9774. MatchOpType(taicpu(hp2), top_const, top_reg) and
  9775. (taicpu(hp2).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  9776. { be lazy, checking separately for sub would be slightly better }
  9777. (abs(taicpu(hp2).oper[0]^.val)<=$7fffffff) then
  9778. begin
  9779. TransferUsedRegs(TmpUsedRegs);
  9780. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  9781. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  9782. if TryMovArith2Lea(hp2) then
  9783. begin
  9784. Result := True;
  9785. Exit;
  9786. end;
  9787. end;
  9788. end;
  9789. if MatchOpType(taicpu(p),top_reg,top_reg) and
  9790. {$ifdef x86_64}
  9791. MatchInstruction(hp1,A_MOVZX,A_MOVSX,A_MOVSXD,[]) and
  9792. {$else x86_64}
  9793. MatchInstruction(hp1,A_MOVZX,A_MOVSX,[]) and
  9794. {$endif x86_64}
  9795. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  9796. (taicpu(hp1).oper[0]^.reg = taicpu(p).oper[1]^.reg) then
  9797. { mov reg1, reg2 mov reg1, reg2
  9798. movzx/sx reg2, reg3 to movzx/sx reg1, reg3}
  9799. begin
  9800. taicpu(hp1).oper[0]^.reg := taicpu(p).oper[0]^.reg;
  9801. DebugMsg(SPeepholeOptimization + 'mov %reg1,%reg2; movzx/sx %reg2,%reg3 -> mov %reg1,%reg2;movzx/sx %reg1,%reg3',p);
  9802. { Don't remove the MOV command without first checking that reg2 isn't used afterwards,
  9803. or unless supreg(reg3) = supreg(reg2)). [Kit] }
  9804. TransferUsedRegs(TmpUsedRegs);
  9805. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  9806. if (getsupreg(taicpu(p).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) or
  9807. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs)
  9808. then
  9809. begin
  9810. RemoveCurrentP(p, hp1);
  9811. Result:=true;
  9812. end;
  9813. Exit;
  9814. end;
  9815. if MatchOpType(taicpu(p),top_reg,top_reg) and
  9816. IsXCHGAcceptable and
  9817. { XCHG doesn't support 8-bit registers }
  9818. (taicpu(p).opsize <> S_B) and
  9819. MatchInstruction(hp1, A_MOV, []) and
  9820. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  9821. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[0]^.reg) and
  9822. GetNextInstruction(hp1, hp2) and
  9823. MatchInstruction(hp2, A_MOV, []) and
  9824. { Don't need to call MatchOpType for hp2 because the operand matches below cover for it }
  9825. MatchOperand(taicpu(hp2).oper[0]^, taicpu(p).oper[1]^.reg) and
  9826. MatchOperand(taicpu(hp2).oper[1]^, taicpu(hp1).oper[0]^.reg) then
  9827. begin
  9828. { mov %reg1,%reg2
  9829. mov %reg3,%reg1 -> xchg %reg3,%reg1
  9830. mov %reg2,%reg3
  9831. (%reg2 not used afterwards)
  9832. Note that xchg takes 3 cycles to execute, and generally mov's take
  9833. only one cycle apiece, but the first two mov's can be executed in
  9834. parallel, only taking 2 cycles overall. Older processors should
  9835. therefore only optimise for size. [Kit]
  9836. }
  9837. TransferUsedRegs(TmpUsedRegs);
  9838. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  9839. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  9840. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp2, TmpUsedRegs) then
  9841. begin
  9842. DebugMsg(SPeepholeOptimization + 'MovMovMov2XChg', p);
  9843. AllocRegBetween(taicpu(hp2).oper[1]^.reg, p, hp1, UsedRegs);
  9844. taicpu(hp1).opcode := A_XCHG;
  9845. RemoveCurrentP(p, hp1);
  9846. RemoveInstruction(hp2);
  9847. Result := True;
  9848. Exit;
  9849. end;
  9850. end;
  9851. if MatchOpType(taicpu(p),top_reg,top_reg) and
  9852. MatchInstruction(hp1, A_SAR, []) then
  9853. begin
  9854. if MatchOperand(taicpu(hp1).oper[0]^, 31) then
  9855. begin
  9856. { the use of %edx also covers the opsize being S_L }
  9857. if MatchOperand(taicpu(hp1).oper[1]^, NR_EDX) then
  9858. begin
  9859. { Note it has to be specifically "movl %eax,%edx", and those specific sub-registers }
  9860. if (taicpu(p).oper[0]^.reg = NR_EAX) and
  9861. (taicpu(p).oper[1]^.reg = NR_EDX) then
  9862. begin
  9863. { Change:
  9864. movl %eax,%edx
  9865. sarl $31,%edx
  9866. To:
  9867. cltd
  9868. }
  9869. DebugMsg(SPeepholeOptimization + 'MovSar2Cltd', p);
  9870. RemoveInstruction(hp1);
  9871. taicpu(p).opcode := A_CDQ;
  9872. taicpu(p).opsize := S_NO;
  9873. taicpu(p).clearop(1);
  9874. taicpu(p).clearop(0);
  9875. taicpu(p).ops:=0;
  9876. Result := True;
  9877. Exit;
  9878. end
  9879. else if (cs_opt_size in current_settings.optimizerswitches) and
  9880. (taicpu(p).oper[0]^.reg = NR_EDX) and
  9881. (taicpu(p).oper[1]^.reg = NR_EAX) then
  9882. begin
  9883. { Change:
  9884. movl %edx,%eax
  9885. sarl $31,%edx
  9886. To:
  9887. movl %edx,%eax
  9888. cltd
  9889. Note that this creates a dependency between the two instructions,
  9890. so only perform if optimising for size.
  9891. }
  9892. DebugMsg(SPeepholeOptimization + 'MovSar2MovCltd', p);
  9893. taicpu(hp1).opcode := A_CDQ;
  9894. taicpu(hp1).opsize := S_NO;
  9895. taicpu(hp1).clearop(1);
  9896. taicpu(hp1).clearop(0);
  9897. taicpu(hp1).ops:=0;
  9898. Include(OptsToCheck, aoc_ForceNewIteration);
  9899. Exit;
  9900. end;
  9901. {$ifndef x86_64}
  9902. end
  9903. { Don't bother if CMOV is supported, because a more optimal
  9904. sequence would have been generated for the Abs() intrinsic }
  9905. else if not(CPUX86_HAS_CMOV in cpu_capabilities[current_settings.cputype]) and
  9906. { the use of %eax also covers the opsize being S_L }
  9907. MatchOperand(taicpu(hp1).oper[1]^, NR_EAX) and
  9908. (taicpu(p).oper[0]^.reg = NR_EAX) and
  9909. (taicpu(p).oper[1]^.reg = NR_EDX) and
  9910. GetNextInstruction(hp1, hp2) and
  9911. MatchInstruction(hp2, A_XOR, [S_L]) and
  9912. MatchOperand(taicpu(hp2).oper[0]^, NR_EAX) and
  9913. MatchOperand(taicpu(hp2).oper[1]^, NR_EDX) and
  9914. GetNextInstruction(hp2, hp3) and
  9915. MatchInstruction(hp3, A_SUB, [S_L]) and
  9916. MatchOperand(taicpu(hp3).oper[0]^, NR_EAX) and
  9917. MatchOperand(taicpu(hp3).oper[1]^, NR_EDX) then
  9918. begin
  9919. { Change:
  9920. movl %eax,%edx
  9921. sarl $31,%eax
  9922. xorl %eax,%edx
  9923. subl %eax,%edx
  9924. (Instruction that uses %edx)
  9925. (%eax deallocated)
  9926. (%edx deallocated)
  9927. To:
  9928. cltd
  9929. xorl %edx,%eax <-- Note the registers have swapped
  9930. subl %edx,%eax
  9931. (Instruction that uses %eax) <-- %eax rather than %edx
  9932. }
  9933. TransferUsedRegs(TmpUsedRegs);
  9934. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  9935. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  9936. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  9937. if not RegUsedAfterInstruction(NR_EAX, hp3, TmpUsedRegs) then
  9938. begin
  9939. if GetNextInstruction(hp3, hp4) and
  9940. not RegModifiedByInstruction(NR_EDX, hp4) and
  9941. not RegUsedAfterInstruction(NR_EDX, hp4, TmpUsedRegs) then
  9942. begin
  9943. DebugMsg(SPeepholeOptimization + 'abs() intrinsic optimisation', p);
  9944. taicpu(p).opcode := A_CDQ;
  9945. taicpu(p).clearop(1);
  9946. taicpu(p).clearop(0);
  9947. taicpu(p).ops:=0;
  9948. RemoveInstruction(hp1);
  9949. taicpu(hp2).loadreg(0, NR_EDX);
  9950. taicpu(hp2).loadreg(1, NR_EAX);
  9951. taicpu(hp3).loadreg(0, NR_EDX);
  9952. taicpu(hp3).loadreg(1, NR_EAX);
  9953. AllocRegBetween(NR_EAX, hp3, hp4, TmpUsedRegs);
  9954. { Convert references in the following instruction (hp4) from %edx to %eax }
  9955. for OperIdx := 0 to taicpu(hp4).ops - 1 do
  9956. with taicpu(hp4).oper[OperIdx]^ do
  9957. case typ of
  9958. top_reg:
  9959. if getsupreg(reg) = RS_EDX then
  9960. reg := newreg(R_INTREGISTER,RS_EAX,getsubreg(reg));
  9961. top_ref:
  9962. begin
  9963. if getsupreg(reg) = RS_EDX then
  9964. ref^.base := newreg(R_INTREGISTER,RS_EAX,getsubreg(reg));
  9965. if getsupreg(reg) = RS_EDX then
  9966. ref^.index := newreg(R_INTREGISTER,RS_EAX,getsubreg(reg));
  9967. end;
  9968. else
  9969. ;
  9970. end;
  9971. Result := True;
  9972. Exit;
  9973. end;
  9974. end;
  9975. {$else x86_64}
  9976. end;
  9977. end
  9978. else if MatchOperand(taicpu(hp1).oper[0]^, 63) and
  9979. { the use of %rdx also covers the opsize being S_Q }
  9980. MatchOperand(taicpu(hp1).oper[1]^, NR_RDX) then
  9981. begin
  9982. { Note it has to be specifically "movq %rax,%rdx", and those specific sub-registers }
  9983. if (taicpu(p).oper[0]^.reg = NR_RAX) and
  9984. (taicpu(p).oper[1]^.reg = NR_RDX) then
  9985. begin
  9986. { Change:
  9987. movq %rax,%rdx
  9988. sarq $63,%rdx
  9989. To:
  9990. cqto
  9991. }
  9992. DebugMsg(SPeepholeOptimization + 'MovSar2Cqto', p);
  9993. RemoveInstruction(hp1);
  9994. taicpu(p).opcode := A_CQO;
  9995. taicpu(p).opsize := S_NO;
  9996. taicpu(p).clearop(1);
  9997. taicpu(p).clearop(0);
  9998. taicpu(p).ops:=0;
  9999. Result := True;
  10000. Exit;
  10001. end
  10002. else if (cs_opt_size in current_settings.optimizerswitches) and
  10003. (taicpu(p).oper[0]^.reg = NR_RDX) and
  10004. (taicpu(p).oper[1]^.reg = NR_RAX) then
  10005. begin
  10006. { Change:
  10007. movq %rdx,%rax
  10008. sarq $63,%rdx
  10009. To:
  10010. movq %rdx,%rax
  10011. cqto
  10012. Note that this creates a dependency between the two instructions,
  10013. so only perform if optimising for size.
  10014. }
  10015. DebugMsg(SPeepholeOptimization + 'MovSar2MovCqto', p);
  10016. taicpu(hp1).opcode := A_CQO;
  10017. taicpu(hp1).opsize := S_NO;
  10018. taicpu(hp1).clearop(1);
  10019. taicpu(hp1).clearop(0);
  10020. taicpu(hp1).ops:=0;
  10021. Include(OptsToCheck, aoc_ForceNewIteration);
  10022. Exit;
  10023. {$endif x86_64}
  10024. end;
  10025. end;
  10026. end;
  10027. if MatchInstruction(hp1, A_MOV, []) and
  10028. (taicpu(hp1).oper[1]^.typ = top_reg) then
  10029. { Though "GetNextInstruction" could be factored out, along with
  10030. the instructions that depend on hp2, it is an expensive call that
  10031. should be delayed for as long as possible, hence we do cheaper
  10032. checks first that are likely to be False. [Kit] }
  10033. begin
  10034. if (
  10035. (
  10036. MatchOperand(taicpu(p).oper[1]^, NR_EDX) and
  10037. (taicpu(hp1).oper[1]^.reg = NR_EAX) and
  10038. (
  10039. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  10040. MatchOperand(taicpu(hp1).oper[0]^, NR_EDX)
  10041. )
  10042. ) or
  10043. (
  10044. MatchOperand(taicpu(p).oper[1]^, NR_EAX) and
  10045. (taicpu(hp1).oper[1]^.reg = NR_EDX) and
  10046. (
  10047. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  10048. MatchOperand(taicpu(hp1).oper[0]^, NR_EAX)
  10049. )
  10050. )
  10051. ) and
  10052. GetNextInstruction(hp1, hp2) and
  10053. MatchInstruction(hp2, A_SAR, []) and
  10054. MatchOperand(taicpu(hp2).oper[0]^, 31) then
  10055. begin
  10056. if MatchOperand(taicpu(hp2).oper[1]^, NR_EDX) then
  10057. begin
  10058. { Change:
  10059. movl r/m,%edx movl r/m,%eax movl r/m,%edx movl r/m,%eax
  10060. movl %edx,%eax or movl %eax,%edx or movl r/m,%eax or movl r/m,%edx
  10061. sarl $31,%edx sarl $31,%edx sarl $31,%edx sarl $31,%edx
  10062. To:
  10063. movl r/m,%eax <- Note the change in register
  10064. cltd
  10065. }
  10066. DebugMsg(SPeepholeOptimization + 'MovMovSar2MovCltd', p);
  10067. AllocRegBetween(NR_EAX, p, hp1, UsedRegs);
  10068. taicpu(p).loadreg(1, NR_EAX);
  10069. taicpu(hp1).opcode := A_CDQ;
  10070. taicpu(hp1).clearop(1);
  10071. taicpu(hp1).clearop(0);
  10072. taicpu(hp1).ops:=0;
  10073. RemoveInstruction(hp2);
  10074. Include(OptsToCheck, aoc_ForceNewIteration);
  10075. (*
  10076. {$ifdef x86_64}
  10077. end
  10078. else if MatchOperand(taicpu(hp2).oper[1]^, NR_RDX) and
  10079. { This code sequence does not get generated - however it might become useful
  10080. if and when 128-bit signed integer types make an appearance, so the code
  10081. is kept here for when it is eventually needed. [Kit] }
  10082. (
  10083. (
  10084. (taicpu(hp1).oper[1]^.reg = NR_RAX) and
  10085. (
  10086. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  10087. MatchOperand(taicpu(hp1).oper[0]^, NR_RDX)
  10088. )
  10089. ) or
  10090. (
  10091. (taicpu(hp1).oper[1]^.reg = NR_RDX) and
  10092. (
  10093. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^) or
  10094. MatchOperand(taicpu(hp1).oper[0]^, NR_RAX)
  10095. )
  10096. )
  10097. ) and
  10098. GetNextInstruction(hp1, hp2) and
  10099. MatchInstruction(hp2, A_SAR, [S_Q]) and
  10100. MatchOperand(taicpu(hp2).oper[0]^, 63) and
  10101. MatchOperand(taicpu(hp2).oper[1]^, NR_RDX) then
  10102. begin
  10103. { Change:
  10104. movq r/m,%rdx movq r/m,%rax movq r/m,%rdx movq r/m,%rax
  10105. movq %rdx,%rax or movq %rax,%rdx or movq r/m,%rax or movq r/m,%rdx
  10106. sarq $63,%rdx sarq $63,%rdx sarq $63,%rdx sarq $63,%rdx
  10107. To:
  10108. movq r/m,%rax <- Note the change in register
  10109. cqto
  10110. }
  10111. DebugMsg(SPeepholeOptimization + 'MovMovSar2MovCqto', p);
  10112. AllocRegBetween(NR_RAX, p, hp1, UsedRegs);
  10113. taicpu(p).loadreg(1, NR_RAX);
  10114. taicpu(hp1).opcode := A_CQO;
  10115. taicpu(hp1).clearop(1);
  10116. taicpu(hp1).clearop(0);
  10117. taicpu(hp1).ops:=0;
  10118. RemoveInstruction(hp2);
  10119. Include(OptsToCheck, aoc_ForceNewIteration);
  10120. {$endif x86_64}
  10121. *)
  10122. end;
  10123. end;
  10124. {$ifdef x86_64}
  10125. end;
  10126. if (taicpu(p).opsize = S_L) and
  10127. (taicpu(p).oper[1]^.typ = top_reg) and
  10128. (
  10129. MatchInstruction(hp1, A_MOV,[]) and
  10130. (taicpu(hp1).opsize = S_L) and
  10131. (taicpu(hp1).oper[1]^.typ = top_reg)
  10132. ) and (
  10133. GetNextInstruction(hp1, hp2) and
  10134. (tai(hp2).typ=ait_instruction) and
  10135. (taicpu(hp2).opsize = S_Q) and
  10136. (
  10137. (
  10138. MatchInstruction(hp2, A_ADD,[]) and
  10139. (taicpu(hp2).opsize = S_Q) and
  10140. (taicpu(hp2).oper[0]^.typ = top_reg) and (taicpu(hp2).oper[1]^.typ = top_reg) and
  10141. (
  10142. (
  10143. (getsupreg(taicpu(hp2).oper[0]^.reg) = getsupreg(taicpu(p).oper[1]^.reg)) and
  10144. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg))
  10145. ) or (
  10146. (getsupreg(taicpu(hp2).oper[0]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) and
  10147. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(p).oper[1]^.reg))
  10148. )
  10149. )
  10150. ) or (
  10151. MatchInstruction(hp2, A_LEA,[]) and
  10152. (taicpu(hp2).oper[0]^.ref^.offset = 0) and
  10153. (taicpu(hp2).oper[0]^.ref^.scalefactor <= 1) and
  10154. (
  10155. (
  10156. (getsupreg(taicpu(hp2).oper[0]^.ref^.base) = getsupreg(taicpu(p).oper[1]^.reg)) and
  10157. (getsupreg(taicpu(hp2).oper[0]^.ref^.index) = getsupreg(taicpu(hp1).oper[1]^.reg))
  10158. ) or (
  10159. (getsupreg(taicpu(hp2).oper[0]^.ref^.base) = getsupreg(taicpu(hp1).oper[1]^.reg)) and
  10160. (getsupreg(taicpu(hp2).oper[0]^.ref^.index) = getsupreg(taicpu(p).oper[1]^.reg))
  10161. )
  10162. ) and (
  10163. (
  10164. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg))
  10165. ) or (
  10166. (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(p).oper[1]^.reg))
  10167. )
  10168. )
  10169. )
  10170. )
  10171. ) and (
  10172. GetNextInstruction(hp2, hp3) and
  10173. MatchInstruction(hp3, A_SHR,[]) and
  10174. (taicpu(hp3).opsize = S_Q) and
  10175. (taicpu(hp3).oper[0]^.typ = top_const) and (taicpu(hp2).oper[1]^.typ = top_reg) and
  10176. (taicpu(hp3).oper[0]^.val = 1) and
  10177. (taicpu(hp3).oper[1]^.reg = taicpu(hp2).oper[1]^.reg)
  10178. ) then
  10179. begin
  10180. { Change movl x, reg1d movl x, reg1d
  10181. movl y, reg2d movl y, reg2d
  10182. addq reg2q,reg1q or leaq (reg1q,reg2q),reg1q
  10183. shrq $1, reg1q shrq $1, reg1q
  10184. ( reg1d and reg2d can be switched around in the first two instructions )
  10185. To movl x, reg1d
  10186. addl y, reg1d
  10187. rcrl $1, reg1d
  10188. This corresponds to the common expression (x + y) shr 1, where
  10189. x and y are Cardinals (replacing "shr 1" with "div 2" produces
  10190. smaller code, but won't account for x + y causing an overflow). [Kit]
  10191. }
  10192. DebugMsg(SPeepholeOptimization + 'MovMov*Shr2MovMov*Rcr', p);
  10193. if (getsupreg(taicpu(hp2).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) then
  10194. begin
  10195. { Change first MOV command to have the same register as the final output }
  10196. taicpu(p).oper[1]^.reg := taicpu(hp1).oper[1]^.reg;
  10197. AllocRegBetween(taicpu(hp1).oper[1]^.reg, p, hp1, UsedRegs);
  10198. Result := True;
  10199. end
  10200. else
  10201. begin
  10202. taicpu(hp1).oper[1]^.reg := taicpu(p).oper[1]^.reg;
  10203. Include(OptsToCheck, aoc_ForceNewIteration);
  10204. end;
  10205. { Change second MOV command to an ADD command. This is easier than
  10206. converting the existing command because it means we don't have to
  10207. touch 'y', which might be a complicated reference, and also the
  10208. fact that the third command might either be ADD or LEA. [Kit] }
  10209. taicpu(hp1).opcode := A_ADD;
  10210. { Delete old ADD/LEA instruction }
  10211. RemoveInstruction(hp2);
  10212. { Convert "shrq $1, reg1q" to "rcr $1, reg1d" }
  10213. taicpu(hp3).opcode := A_RCR;
  10214. taicpu(hp3).changeopsize(S_L);
  10215. setsubreg(taicpu(hp3).oper[1]^.reg, R_SUBD);
  10216. { Don't need to Exit yet as p is still a MOV and hp1 hasn't been
  10217. called, so FuncMov2Func below is safe to call }
  10218. {$endif x86_64}
  10219. end;
  10220. {$ifdef x86_64}
  10221. { Note, this optimisation was moved from Pass 1 because the CMOV
  10222. optimisations in OptPass2Jcc fall foul of the loss of information
  10223. about the upper 32 bits of the target register. Fixes #41317. }
  10224. { Change:
  10225. movl %reg1l,%reg2l
  10226. movq %reg2q,%reg3q (%reg1 <> %reg3)
  10227. To:
  10228. movl %reg1l,%reg2l
  10229. movl %reg1l,%reg3l (Upper 32 bits of %reg3q will be zero)
  10230. }
  10231. if MatchOpType(taicpu(p), top_reg, top_reg) and
  10232. (taicpu(p).opsize = S_L) and
  10233. (
  10234. not(cs_opt_level3 in current_settings.optimizerswitches) or
  10235. { Look further ahead for this one }
  10236. GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[1]^.reg)
  10237. ) and
  10238. MatchInstruction(hp1,A_MOV,[S_Q]) and
  10239. not RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp1) and
  10240. MatchOpType(taicpu(hp1), top_reg, top_reg) and
  10241. SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^.reg) then
  10242. begin
  10243. TransferUsedRegs(TmpUsedRegs);
  10244. UpdateUsedRegsBetween(TmpUsedRegs, tai(p.Next), hp1);
  10245. taicpu(hp1).opsize := S_L;
  10246. taicpu(hp1).loadreg(0, taicpu(p).oper[0]^.reg);
  10247. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  10248. AllocRegBetween(taicpu(p).oper[0]^.reg, p, hp1, UsedRegs);
  10249. DebugMsg(SPeepholeOptimization + 'Made 32-to-64-bit zero extension more efficient (MovlMovq2MovlMovl 1)', hp1);
  10250. if not RegUsedAfterInstruction(taicpu(p).oper[0]^.reg, hp1, TmpUsedRegs) then
  10251. begin
  10252. DebugMsg(SPeepholeOptimization + 'Mov2Nop 8 done', p);
  10253. RemoveCurrentP(p);
  10254. Result := True;
  10255. Exit;
  10256. end
  10257. else
  10258. begin
  10259. { Initial instruction wasn't actually changed }
  10260. Include(OptsToCheck, aoc_ForceNewIteration);
  10261. end;
  10262. end;
  10263. {$endif x86_64}
  10264. if FuncMov2Func(p, hp1) then
  10265. begin
  10266. Result := True;
  10267. Exit;
  10268. end;
  10269. end;
  10270. {$push}
  10271. {$q-}{$r-}
  10272. function TX86AsmOptimizer.OptPass2Movx(var p : tai) : boolean;
  10273. var
  10274. ThisReg: TRegister;
  10275. MinSize, MaxSize, TryShiftDown, TargetSize: TOpSize;
  10276. TargetSubReg: TSubRegister;
  10277. hp1, hp2: tai;
  10278. RegInUse, RegChanged, p_removed, hp1_removed: Boolean;
  10279. { Store list of found instructions so we don't have to call
  10280. GetNextInstructionUsingReg multiple times }
  10281. InstrList: array of taicpu;
  10282. InstrMax, Index: Integer;
  10283. UpperLimit, SignedUpperLimit, SignedUpperLimitBottom,
  10284. LowerLimit, SignedLowerLimit, SignedLowerLimitBottom,
  10285. TryShiftDownLimit, TryShiftDownSignedLimit, TryShiftDownSignedLimitLower,
  10286. WorkingValue: TCgInt;
  10287. PreMessage: string;
  10288. { Data flow analysis }
  10289. TestValMin, TestValMax, TestValSignedMax: TCgInt;
  10290. BitwiseOnly, OrXorUsed,
  10291. ShiftDownOverflow, UpperSignedOverflow, UpperUnsignedOverflow, LowerSignedOverflow, LowerUnsignedOverflow: Boolean;
  10292. function CheckOverflowConditions: Boolean;
  10293. begin
  10294. Result := True;
  10295. if (TestValSignedMax > SignedUpperLimit) then
  10296. UpperSignedOverflow := True;
  10297. if (TestValSignedMax > SignedLowerLimit) or (TestValSignedMax < SignedLowerLimitBottom) then
  10298. LowerSignedOverflow := True;
  10299. if (TestValMin > LowerLimit) or (TestValMax > LowerLimit) then
  10300. LowerUnsignedOverflow := True;
  10301. if (TestValMin > UpperLimit) or (TestValMax > UpperLimit) or (TestValSignedMax > UpperLimit) or
  10302. (TestValMin < SignedUpperLimitBottom) or (TestValMax < SignedUpperLimitBottom) or (TestValSignedMax < SignedUpperLimitBottom) then
  10303. begin
  10304. { Absolute overflow }
  10305. Result := False;
  10306. Exit;
  10307. end;
  10308. if not ShiftDownOverflow and (TryShiftDown <> S_NO) and
  10309. ((TestValMin > TryShiftDownLimit) or (TestValMax > TryShiftDownLimit)) then
  10310. ShiftDownOverflow := True;
  10311. if (TestValMin < 0) or (TestValMax < 0) then
  10312. begin
  10313. LowerUnsignedOverflow := True;
  10314. UpperUnsignedOverflow := True;
  10315. end;
  10316. end;
  10317. function AdjustInitialLoadAndSize: Boolean;
  10318. begin
  10319. Result := False;
  10320. if not p_removed then
  10321. begin
  10322. if TargetSize = MinSize then
  10323. begin
  10324. { Convert the input MOVZX to a MOV }
  10325. if (taicpu(p).oper[0]^.typ = top_reg) and
  10326. SuperRegistersEqual(taicpu(p).oper[0]^.reg, ThisReg) then
  10327. begin
  10328. { Or remove it completely! }
  10329. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 1', p);
  10330. RemoveCurrentP(p);
  10331. p_removed := True;
  10332. end
  10333. else
  10334. begin
  10335. DebugMsg(SPeepholeOptimization + 'Movzx2Mov 1', p);
  10336. taicpu(p).opcode := A_MOV;
  10337. taicpu(p).oper[1]^.reg := ThisReg;
  10338. taicpu(p).opsize := TargetSize;
  10339. end;
  10340. Result := True;
  10341. end
  10342. else if TargetSize <> MaxSize then
  10343. begin
  10344. case MaxSize of
  10345. S_L:
  10346. if TargetSize = S_W then
  10347. begin
  10348. DebugMsg(SPeepholeOptimization + 'movzbl2movzbw', p);
  10349. taicpu(p).opsize := S_BW;
  10350. taicpu(p).oper[1]^.reg := ThisReg;
  10351. Result := True;
  10352. end
  10353. else
  10354. InternalError(2020112341);
  10355. S_W:
  10356. if TargetSize = S_L then
  10357. begin
  10358. DebugMsg(SPeepholeOptimization + 'movzbw2movzbl', p);
  10359. taicpu(p).opsize := S_BL;
  10360. taicpu(p).oper[1]^.reg := ThisReg;
  10361. Result := True;
  10362. end
  10363. else
  10364. InternalError(2020112342);
  10365. else
  10366. ;
  10367. end;
  10368. end
  10369. else if not hp1_removed and not RegInUse then
  10370. begin
  10371. { If we have something like:
  10372. movzbl (oper),%regd
  10373. add x, %regd
  10374. movzbl %regb, %regd
  10375. We can reduce the register size to the input of the final
  10376. movzbl instruction. Overflows won't have any effect.
  10377. }
  10378. if (taicpu(p).opsize in [S_BW, S_BL]) and
  10379. (taicpu(hp1).opsize in [S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}]) then
  10380. begin
  10381. TargetSize := S_B;
  10382. setsubreg(ThisReg, R_SUBL);
  10383. Result := True;
  10384. end
  10385. else if (taicpu(p).opsize = S_WL) and
  10386. (taicpu(hp1).opsize in [S_WL{$ifdef x86_64}, S_BQ{$endif x86_64}]) then
  10387. begin
  10388. TargetSize := S_W;
  10389. setsubreg(ThisReg, R_SUBW);
  10390. Result := True;
  10391. end;
  10392. if Result then
  10393. begin
  10394. { Convert the input MOVZX to a MOV }
  10395. if (taicpu(p).oper[0]^.typ = top_reg) and
  10396. SuperRegistersEqual(taicpu(p).oper[0]^.reg, ThisReg) then
  10397. begin
  10398. { Or remove it completely! }
  10399. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 1a', p);
  10400. RemoveCurrentP(p);
  10401. p_removed := True;
  10402. end
  10403. else
  10404. begin
  10405. DebugMsg(SPeepholeOptimization + 'Movzx2Mov 1a', p);
  10406. taicpu(p).opcode := A_MOV;
  10407. taicpu(p).oper[1]^.reg := ThisReg;
  10408. taicpu(p).opsize := TargetSize;
  10409. end;
  10410. end;
  10411. end;
  10412. end;
  10413. end;
  10414. procedure AdjustFinalLoad;
  10415. begin
  10416. if not LowerUnsignedOverflow then
  10417. begin
  10418. if ((TargetSize = S_L) and (taicpu(hp1).opsize in [S_L, S_BL, S_WL])) or
  10419. ((TargetSize = S_W) and (taicpu(hp1).opsize in [S_W, S_BW])) then
  10420. begin
  10421. { Convert the output MOVZX to a MOV }
  10422. if SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  10423. begin
  10424. { Make sure the zero-expansion covers at least the minimum size (fixes i40003) }
  10425. if (MinSize = S_B) or
  10426. (not ShiftDownOverflow and (TryShiftDown = S_B)) or
  10427. ((MinSize = S_W) and (taicpu(hp1).opsize = S_WL)) then
  10428. begin
  10429. { Remove it completely! }
  10430. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 2', hp1);
  10431. { Be careful; if p = hp1 and p was also removed, p
  10432. will become a dangling pointer }
  10433. if p = hp1 then
  10434. begin
  10435. RemoveCurrentp(p); { p = hp1 and will then become the next instruction }
  10436. p_removed := True;
  10437. end
  10438. else
  10439. RemoveInstruction(hp1);
  10440. hp1_removed := True;
  10441. end;
  10442. end
  10443. else
  10444. begin
  10445. DebugMsg(SPeepholeOptimization + 'Movzx2Mov 2', hp1);
  10446. taicpu(hp1).opcode := A_MOV;
  10447. taicpu(hp1).oper[0]^.reg := ThisReg;
  10448. taicpu(hp1).opsize := TargetSize;
  10449. end;
  10450. end
  10451. else if (TargetSize = S_B) and (MaxSize = S_W) and (taicpu(hp1).opsize = S_WL) then
  10452. begin
  10453. { Need to change the size of the output }
  10454. DebugMsg(SPeepholeOptimization + 'movzwl2movzbl 2', hp1);
  10455. taicpu(hp1).oper[0]^.reg := ThisReg;
  10456. taicpu(hp1).opsize := S_BL;
  10457. end;
  10458. end;
  10459. end;
  10460. function CompressInstructions: Boolean;
  10461. var
  10462. LocalIndex: Integer;
  10463. begin
  10464. Result := False;
  10465. { The objective here is to try to find a combination that
  10466. removes one of the MOV/Z instructions. }
  10467. if (
  10468. (taicpu(p).oper[0]^.typ <> top_reg) or
  10469. not SuperRegistersEqual(taicpu(p).oper[0]^.reg, ThisReg)
  10470. ) and
  10471. (taicpu(hp1).oper[1]^.typ = top_reg) and
  10472. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  10473. begin
  10474. { Make a preference to remove the second MOVZX instruction }
  10475. case taicpu(hp1).opsize of
  10476. S_BL, S_WL:
  10477. begin
  10478. TargetSize := S_L;
  10479. TargetSubReg := R_SUBD;
  10480. end;
  10481. S_BW:
  10482. begin
  10483. TargetSize := S_W;
  10484. TargetSubReg := R_SUBW;
  10485. end;
  10486. else
  10487. InternalError(2020112302);
  10488. end;
  10489. end
  10490. else
  10491. begin
  10492. if LowerUnsignedOverflow and not UpperUnsignedOverflow then
  10493. begin
  10494. { Exceeded lower bound but not upper bound }
  10495. TargetSize := MaxSize;
  10496. end
  10497. else if not LowerUnsignedOverflow then
  10498. begin
  10499. { Size didn't exceed lower bound }
  10500. TargetSize := MinSize;
  10501. end
  10502. else
  10503. Exit;
  10504. end;
  10505. case TargetSize of
  10506. S_B:
  10507. TargetSubReg := R_SUBL;
  10508. S_W:
  10509. TargetSubReg := R_SUBW;
  10510. S_L:
  10511. TargetSubReg := R_SUBD;
  10512. else
  10513. InternalError(2020112350);
  10514. end;
  10515. { Update the register to its new size }
  10516. setsubreg(ThisReg, TargetSubReg);
  10517. RegInUse := False;
  10518. if not SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  10519. begin
  10520. { Check to see if the active register is used afterwards;
  10521. if not, we can change it and make a saving. }
  10522. TransferUsedRegs(TmpUsedRegs);
  10523. { The target register may be marked as in use to cross
  10524. a jump to a distant label, so exclude it }
  10525. ExcludeRegFromUsedRegs(taicpu(hp1).oper[1]^.reg, TmpUsedRegs);
  10526. hp2 := p;
  10527. repeat
  10528. { Explicitly check for the excluded register (don't include the first
  10529. instruction as it may be reading from here }
  10530. if ((p <> hp2) and (RegInInstruction(taicpu(hp1).oper[1]^.reg, hp2))) or
  10531. RegInUsedRegs(taicpu(hp1).oper[1]^.reg, TmpUsedRegs) then
  10532. begin
  10533. RegInUse := True;
  10534. Break;
  10535. end;
  10536. UpdateUsedRegs(TmpUsedRegs, tai(hp2.next));
  10537. if not GetNextInstruction(hp2, hp2) then
  10538. InternalError(2020112340);
  10539. until (hp2 = hp1);
  10540. if not RegInUse and RegUsedAfterInstruction(ThisReg, hp1, TmpUsedRegs) then
  10541. { We might still be able to get away with this }
  10542. RegInUse := not
  10543. (
  10544. GetNextInstructionUsingReg(hp1, hp2, ThisReg) and
  10545. (hp2.typ = ait_instruction) and
  10546. (
  10547. { Under -O1 and -O2, GetNextInstructionUsingReg may return an
  10548. instruction that doesn't actually contain ThisReg }
  10549. (cs_opt_level3 in current_settings.optimizerswitches) or
  10550. RegInInstruction(ThisReg, hp2)
  10551. ) and
  10552. RegLoadedWithNewValue(ThisReg, hp2)
  10553. );
  10554. if not RegInUse then
  10555. begin
  10556. { Force the register size to the same as this instruction so it can be removed}
  10557. if (taicpu(hp1).opsize in [S_L, S_BL, S_WL]) then
  10558. begin
  10559. TargetSize := S_L;
  10560. TargetSubReg := R_SUBD;
  10561. end
  10562. else if (taicpu(hp1).opsize in [S_W, S_BW]) then
  10563. begin
  10564. TargetSize := S_W;
  10565. TargetSubReg := R_SUBW;
  10566. end;
  10567. ThisReg := taicpu(hp1).oper[1]^.reg;
  10568. setsubreg(ThisReg, TargetSubReg);
  10569. RegChanged := True;
  10570. DebugMsg(SPeepholeOptimization + 'Simplified register usage so ' + debug_regname(ThisReg) + ' = ' + debug_regname(taicpu(p).oper[1]^.reg), p);
  10571. TransferUsedRegs(TmpUsedRegs);
  10572. AllocRegBetween(ThisReg, p, hp1, TmpUsedRegs);
  10573. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 3', hp1);
  10574. if p = hp1 then
  10575. begin
  10576. RemoveCurrentp(p); { p = hp1 and will then become the next instruction }
  10577. p_removed := True;
  10578. end
  10579. else
  10580. RemoveInstruction(hp1);
  10581. hp1_removed := True;
  10582. { Instruction will become "mov %reg,%reg" }
  10583. if not p_removed and (taicpu(p).opcode = A_MOV) and
  10584. MatchOperand(taicpu(p).oper[0]^, ThisReg) then
  10585. begin
  10586. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 6', p);
  10587. RemoveCurrentP(p);
  10588. p_removed := True;
  10589. end
  10590. else
  10591. taicpu(p).oper[1]^.reg := ThisReg;
  10592. Result := True;
  10593. end
  10594. else
  10595. begin
  10596. if TargetSize <> MaxSize then
  10597. begin
  10598. { Since the register is in use, we have to force it to
  10599. MaxSize otherwise part of it may become undefined later on }
  10600. TargetSize := MaxSize;
  10601. case TargetSize of
  10602. S_B:
  10603. TargetSubReg := R_SUBL;
  10604. S_W:
  10605. TargetSubReg := R_SUBW;
  10606. S_L:
  10607. TargetSubReg := R_SUBD;
  10608. else
  10609. InternalError(2020112351);
  10610. end;
  10611. setsubreg(ThisReg, TargetSubReg);
  10612. end;
  10613. AdjustFinalLoad;
  10614. end;
  10615. end
  10616. else
  10617. AdjustFinalLoad;
  10618. Result := AdjustInitialLoadAndSize or Result;
  10619. { Now go through every instruction we found and change the
  10620. size. If TargetSize = MaxSize, then almost no changes are
  10621. needed and Result can remain False if it hasn't been set
  10622. yet.
  10623. If RegChanged is True, then the register requires changing
  10624. and so the point about TargetSize = MaxSize doesn't apply. }
  10625. if ((TargetSize <> MaxSize) or RegChanged) and (InstrMax >= 0) then
  10626. begin
  10627. for LocalIndex := 0 to InstrMax do
  10628. begin
  10629. { If p_removed is true, then the original MOV/Z was removed
  10630. and removing the AND instruction may not be safe if it
  10631. appears first }
  10632. if (InstrList[LocalIndex].oper[InstrList[LocalIndex].ops - 1]^.typ <> top_reg) then
  10633. InternalError(2020112310);
  10634. if InstrList[LocalIndex].oper[0]^.typ = top_reg then
  10635. InstrList[LocalIndex].oper[0]^.reg := ThisReg;
  10636. InstrList[LocalIndex].oper[InstrList[LocalIndex].ops - 1]^.reg := ThisReg;
  10637. InstrList[LocalIndex].opsize := TargetSize;
  10638. end;
  10639. Result := True;
  10640. end;
  10641. end;
  10642. begin
  10643. Result := False;
  10644. p_removed := False;
  10645. hp1_removed := False;
  10646. ThisReg := taicpu(p).oper[1]^.reg;
  10647. { Check for:
  10648. movs/z ###,%ecx (or %cx or %rcx)
  10649. ...
  10650. shl/shr/sar/rcl/rcr/ror/rol %cl,###
  10651. (dealloc %ecx)
  10652. Change to:
  10653. mov ###,%cl (if ### = %cl, then remove completely)
  10654. ...
  10655. shl/shr/sar/rcl/rcr/ror/rol %cl,###
  10656. }
  10657. if (getsupreg(ThisReg) = RS_ECX) and
  10658. GetNextInstructionUsingReg(p, hp1, NR_ECX) and
  10659. (hp1.typ = ait_instruction) and
  10660. (
  10661. { Under -O1 and -O2, GetNextInstructionUsingReg may return an
  10662. instruction that doesn't actually contain ECX }
  10663. (cs_opt_level3 in current_settings.optimizerswitches) or
  10664. RegInInstruction(NR_ECX, hp1) or
  10665. (
  10666. { It's common for the shift/rotate's read/write register to be
  10667. initialised in between, so under -O2 and under, search ahead
  10668. one more instruction
  10669. }
  10670. GetNextInstruction(hp1, hp1) and
  10671. (hp1.typ = ait_instruction) and
  10672. RegInInstruction(NR_ECX, hp1)
  10673. )
  10674. ) and
  10675. MatchInstruction(hp1, [A_SHL, A_SHR, A_SAR, A_ROR, A_ROL, A_RCR, A_RCL], []) and
  10676. (taicpu(hp1).oper[0]^.typ = top_reg) { This is enough to determine that it's %cl } then
  10677. begin
  10678. TransferUsedRegs(TmpUsedRegs);
  10679. hp2 := p;
  10680. repeat
  10681. UpdateUsedRegs(TmpUsedRegs, tai(hp2.Next));
  10682. until not GetNextInstruction(hp2, hp2) or (hp2 = hp1);
  10683. if not RegUsedAfterInstruction(NR_CL, hp1, TmpUsedRegs) then
  10684. begin
  10685. case taicpu(p).opsize of
  10686. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  10687. if MatchOperand(taicpu(p).oper[0]^, NR_CL) then
  10688. begin
  10689. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 3a', p);
  10690. RemoveCurrentP(p);
  10691. end
  10692. else
  10693. begin
  10694. taicpu(p).opcode := A_MOV;
  10695. taicpu(p).opsize := S_B;
  10696. taicpu(p).oper[1]^.reg := NR_CL;
  10697. DebugMsg(SPeepholeOptimization + 'MovxOp2MovOp 1', p);
  10698. end;
  10699. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  10700. if MatchOperand(taicpu(p).oper[0]^, NR_CX) then
  10701. begin
  10702. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 3b', p);
  10703. RemoveCurrentP(p);
  10704. end
  10705. else
  10706. begin
  10707. taicpu(p).opcode := A_MOV;
  10708. taicpu(p).opsize := S_W;
  10709. taicpu(p).oper[1]^.reg := NR_CX;
  10710. DebugMsg(SPeepholeOptimization + 'MovxOp2MovOp 2', p);
  10711. end;
  10712. {$ifdef x86_64}
  10713. S_LQ:
  10714. if MatchOperand(taicpu(p).oper[0]^, NR_ECX) then
  10715. begin
  10716. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 3c', p);
  10717. RemoveCurrentP(p);
  10718. end
  10719. else
  10720. begin
  10721. taicpu(p).opcode := A_MOV;
  10722. taicpu(p).opsize := S_L;
  10723. taicpu(p).oper[1]^.reg := NR_ECX;
  10724. DebugMsg(SPeepholeOptimization + 'MovxOp2MovOp 3', p);
  10725. end;
  10726. {$endif x86_64}
  10727. else
  10728. InternalError(2021120401);
  10729. end;
  10730. Result := True;
  10731. Exit;
  10732. end;
  10733. end;
  10734. { This is anything but quick! }
  10735. if not(cs_opt_level2 in current_settings.optimizerswitches) then
  10736. Exit;
  10737. SetLength(InstrList, 0);
  10738. InstrMax := -1;
  10739. case taicpu(p).opsize of
  10740. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  10741. begin
  10742. {$if defined(i386) or defined(i8086)}
  10743. { If the target size is 8-bit, make sure we can actually encode it }
  10744. if not (GetSupReg(ThisReg) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX]) then
  10745. Exit;
  10746. {$endif i386 or i8086}
  10747. LowerLimit := $FF;
  10748. SignedLowerLimit := $7F;
  10749. SignedLowerLimitBottom := -128;
  10750. MinSize := S_B;
  10751. if taicpu(p).opsize = S_BW then
  10752. begin
  10753. MaxSize := S_W;
  10754. UpperLimit := $FFFF;
  10755. SignedUpperLimit := $7FFF;
  10756. SignedUpperLimitBottom := -32768;
  10757. end
  10758. else
  10759. begin
  10760. { Keep at a 32-bit limit for BQ as well since one can't really optimise otherwise }
  10761. MaxSize := S_L;
  10762. UpperLimit := $FFFFFFFF;
  10763. SignedUpperLimit := $7FFFFFFF;
  10764. SignedUpperLimitBottom := -2147483648;
  10765. end;
  10766. end;
  10767. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  10768. begin
  10769. { Keep at a 32-bit limit for WQ as well since one can't really optimise otherwise }
  10770. LowerLimit := $FFFF;
  10771. SignedLowerLimit := $7FFF;
  10772. SignedLowerLimitBottom := -32768;
  10773. UpperLimit := $FFFFFFFF;
  10774. SignedUpperLimit := $7FFFFFFF;
  10775. SignedUpperLimitBottom := -2147483648;
  10776. MinSize := S_W;
  10777. MaxSize := S_L;
  10778. end;
  10779. {$ifdef x86_64}
  10780. S_LQ:
  10781. begin
  10782. { Both the lower and upper limits are set to 32-bit. If a limit
  10783. is breached, then optimisation is impossible }
  10784. LowerLimit := $FFFFFFFF;
  10785. SignedLowerLimit := $7FFFFFFF;
  10786. SignedLowerLimitBottom := -2147483648;
  10787. UpperLimit := $FFFFFFFF;
  10788. SignedUpperLimit := $7FFFFFFF;
  10789. SignedUpperLimitBottom := -2147483648;
  10790. MinSize := S_L;
  10791. MaxSize := S_L;
  10792. end;
  10793. {$endif x86_64}
  10794. else
  10795. InternalError(2020112301);
  10796. end;
  10797. TestValMin := 0;
  10798. TestValMax := LowerLimit;
  10799. TestValSignedMax := SignedLowerLimit;
  10800. TryShiftDownLimit := LowerLimit;
  10801. TryShiftDown := S_NO;
  10802. ShiftDownOverflow := False;
  10803. RegChanged := False;
  10804. BitwiseOnly := True;
  10805. OrXorUsed := False;
  10806. UpperSignedOverflow := False;
  10807. LowerSignedOverflow := False;
  10808. UpperUnsignedOverflow := False;
  10809. LowerUnsignedOverflow := False;
  10810. hp1 := p;
  10811. while GetNextInstructionUsingReg(hp1, hp1, ThisReg) and
  10812. (hp1.typ = ait_instruction) and
  10813. (
  10814. { Under -O1 and -O2, GetNextInstructionUsingReg may return an
  10815. instruction that doesn't actually contain ThisReg }
  10816. (cs_opt_level3 in current_settings.optimizerswitches) or
  10817. { This allows this Movx optimisation to work through the SETcc instructions
  10818. inserted by the 'CMP/JE/CMP/@Lbl/SETE -> CMP/SETE/CMP/SETE/OR'
  10819. optimisation on -O1 and -O2 (on -O3, GetNextInstructionUsingReg will
  10820. skip over these SETcc instructions). }
  10821. (taicpu(hp1).opcode = A_SETcc) or
  10822. RegInInstruction(ThisReg, hp1)
  10823. ) do
  10824. begin
  10825. case taicpu(hp1).opcode of
  10826. A_INC,A_DEC:
  10827. begin
  10828. { Has to be an exact match on the register }
  10829. if not MatchOperand(taicpu(hp1).oper[0]^, ThisReg) then
  10830. Break;
  10831. if taicpu(hp1).opcode = A_INC then
  10832. begin
  10833. Inc(TestValMin);
  10834. Inc(TestValMax);
  10835. Inc(TestValSignedMax);
  10836. end
  10837. else
  10838. begin
  10839. Dec(TestValMin);
  10840. Dec(TestValMax);
  10841. Dec(TestValSignedMax);
  10842. end;
  10843. end;
  10844. A_TEST, A_CMP:
  10845. begin
  10846. if (
  10847. { Too high a risk of non-linear behaviour that breaks DFA
  10848. here, unless it's cmp $0,%reg, which is equivalent to
  10849. test %reg,%reg }
  10850. OrXorUsed and
  10851. (taicpu(hp1).opcode = A_CMP) and
  10852. not Matchoperand(taicpu(hp1).oper[0]^, 0)
  10853. ) or
  10854. (taicpu(hp1).oper[1]^.typ <> top_reg) or
  10855. { Has to be an exact match on the register }
  10856. (taicpu(hp1).oper[1]^.reg <> ThisReg) or
  10857. (
  10858. { Permit "test %reg,%reg" }
  10859. (taicpu(hp1).opcode = A_TEST) and
  10860. (taicpu(hp1).oper[0]^.typ = top_reg) and
  10861. (taicpu(hp1).oper[0]^.reg <> ThisReg)
  10862. ) or
  10863. (taicpu(hp1).oper[0]^.typ <> top_const) or
  10864. { Make sure the comparison value is not smaller than the
  10865. smallest allowed signed value for the minimum size (e.g.
  10866. -128 for 8-bit) }
  10867. not (
  10868. ((taicpu(hp1).oper[0]^.val and LowerLimit) = taicpu(hp1).oper[0]^.val) or
  10869. { Is it in the negative range? }
  10870. (
  10871. (taicpu(hp1).oper[0]^.val < 0) and
  10872. (taicpu(hp1).oper[0]^.val >= SignedLowerLimitBottom)
  10873. )
  10874. ) then
  10875. Break;
  10876. { Check to see if the active register is used afterwards }
  10877. TransferUsedRegs(TmpUsedRegs);
  10878. IncludeRegInUsedRegs(ThisReg, TmpUsedRegs);
  10879. if not RegUsedAfterInstruction(ThisReg, hp1, TmpUsedRegs) then
  10880. begin
  10881. { Make sure the comparison or any previous instructions
  10882. hasn't pushed the test values outside of the range of
  10883. MinSize }
  10884. if LowerUnsignedOverflow and not UpperUnsignedOverflow then
  10885. begin
  10886. { Exceeded lower bound but not upper bound }
  10887. Exit;
  10888. end
  10889. else if not LowerSignedOverflow or not LowerUnsignedOverflow then
  10890. begin
  10891. { Size didn't exceed lower bound }
  10892. TargetSize := MinSize;
  10893. end
  10894. else
  10895. Break;
  10896. case TargetSize of
  10897. S_B:
  10898. TargetSubReg := R_SUBL;
  10899. S_W:
  10900. TargetSubReg := R_SUBW;
  10901. S_L:
  10902. TargetSubReg := R_SUBD;
  10903. else
  10904. InternalError(2021051002);
  10905. end;
  10906. if TargetSize <> MaxSize then
  10907. begin
  10908. { Update the register to its new size }
  10909. setsubreg(ThisReg, TargetSubReg);
  10910. DebugMsg(SPeepholeOptimization + 'CMP instruction resized thanks to register size optimisation (see MOV/Z assignment above)', hp1);
  10911. taicpu(hp1).oper[1]^.reg := ThisReg;
  10912. taicpu(hp1).opsize := TargetSize;
  10913. { Convert the input MOVZX to a MOV if necessary }
  10914. AdjustInitialLoadAndSize;
  10915. if (InstrMax >= 0) then
  10916. begin
  10917. for Index := 0 to InstrMax do
  10918. begin
  10919. { If p_removed is true, then the original MOV/Z was removed
  10920. and removing the AND instruction may not be safe if it
  10921. appears first }
  10922. if (InstrList[Index].oper[InstrList[Index].ops - 1]^.typ <> top_reg) then
  10923. InternalError(2020112311);
  10924. if InstrList[Index].oper[0]^.typ = top_reg then
  10925. InstrList[Index].oper[0]^.reg := ThisReg;
  10926. InstrList[Index].oper[InstrList[Index].ops - 1]^.reg := ThisReg;
  10927. InstrList[Index].opsize := MinSize;
  10928. end;
  10929. end;
  10930. Result := True;
  10931. end;
  10932. Exit;
  10933. end;
  10934. end;
  10935. A_SETcc:
  10936. begin
  10937. { This allows this Movx optimisation to work through the SETcc instructions
  10938. inserted by the 'CMP/JE/CMP/@Lbl/SETE -> CMP/SETE/CMP/SETE/OR'
  10939. optimisation on -O1 and -O2 (on -O3, GetNextInstructionUsingReg will
  10940. skip over these SETcc instructions). }
  10941. if (cs_opt_level3 in current_settings.optimizerswitches) or
  10942. { Of course, break out if the current register is used }
  10943. RegInOp(ThisReg, taicpu(hp1).oper[0]^) then
  10944. Break
  10945. else
  10946. { We must use Continue so the instruction doesn't get added
  10947. to InstrList }
  10948. Continue;
  10949. end;
  10950. A_ADD,A_SUB,A_AND,A_OR,A_XOR,A_SHL,A_SHR,A_SAR:
  10951. begin
  10952. if
  10953. (taicpu(hp1).oper[1]^.typ <> top_reg) or
  10954. { Has to be an exact match on the register }
  10955. (taicpu(hp1).oper[1]^.reg <> ThisReg) or not
  10956. (
  10957. (
  10958. (taicpu(hp1).oper[0]^.typ = top_const) and
  10959. (
  10960. (
  10961. (taicpu(hp1).opcode = A_SHL) and
  10962. (
  10963. ((MinSize = S_B) and (taicpu(hp1).oper[0]^.val < 8)) or
  10964. ((MinSize = S_W) and (taicpu(hp1).oper[0]^.val < 16)) or
  10965. ((MinSize = S_L) and (taicpu(hp1).oper[0]^.val < 32))
  10966. )
  10967. ) or (
  10968. (taicpu(hp1).opcode <> A_SHL) and
  10969. (
  10970. ((taicpu(hp1).oper[0]^.val and UpperLimit) = taicpu(hp1).oper[0]^.val) or
  10971. { Is it in the negative range? }
  10972. (((not taicpu(hp1).oper[0]^.val) and (UpperLimit shr 1)) = (not taicpu(hp1).oper[0]^.val))
  10973. )
  10974. )
  10975. )
  10976. ) or (
  10977. MatchOperand(taicpu(hp1).oper[0]^, taicpu(hp1).oper[1]^.reg) and
  10978. ((taicpu(hp1).opcode = A_ADD) or (taicpu(hp1).opcode = A_AND) or (taicpu(hp1).opcode = A_SUB))
  10979. )
  10980. ) then
  10981. Break;
  10982. { Only process OR and XOR if there are only bitwise operations,
  10983. since otherwise they can too easily fool the data flow
  10984. analysis (they can cause non-linear behaviour) }
  10985. case taicpu(hp1).opcode of
  10986. A_ADD:
  10987. begin
  10988. if OrXorUsed then
  10989. { Too high a risk of non-linear behaviour that breaks DFA here }
  10990. Break
  10991. else
  10992. BitwiseOnly := False;
  10993. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  10994. begin
  10995. TestValMin := TestValMin * 2;
  10996. TestValMax := TestValMax * 2;
  10997. TestValSignedMax := TestValSignedMax * 2;
  10998. end
  10999. else
  11000. begin
  11001. WorkingValue := taicpu(hp1).oper[0]^.val;
  11002. TestValMin := TestValMin + WorkingValue;
  11003. TestValMax := TestValMax + WorkingValue;
  11004. TestValSignedMax := TestValSignedMax + WorkingValue;
  11005. end;
  11006. end;
  11007. A_SUB:
  11008. begin
  11009. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  11010. begin
  11011. TestValMin := 0;
  11012. TestValMax := 0;
  11013. TestValSignedMax := 0;
  11014. end
  11015. else
  11016. begin
  11017. if OrXorUsed then
  11018. { Too high a risk of non-linear behaviour that breaks DFA here }
  11019. Break
  11020. else
  11021. BitwiseOnly := False;
  11022. WorkingValue := taicpu(hp1).oper[0]^.val;
  11023. TestValMin := TestValMin - WorkingValue;
  11024. TestValMax := TestValMax - WorkingValue;
  11025. TestValSignedMax := TestValSignedMax - WorkingValue;
  11026. end;
  11027. end;
  11028. A_AND:
  11029. if (taicpu(hp1).oper[0]^.typ = top_const) then
  11030. begin
  11031. { we might be able to go smaller if AND appears first }
  11032. if InstrMax = -1 then
  11033. case MinSize of
  11034. S_B:
  11035. ;
  11036. S_W:
  11037. if ((taicpu(hp1).oper[0]^.val and $FF) = taicpu(hp1).oper[0]^.val) or
  11038. ((not(taicpu(hp1).oper[0]^.val) and $7F) = (not taicpu(hp1).oper[0]^.val)) then
  11039. begin
  11040. TryShiftDown := S_B;
  11041. TryShiftDownLimit := $FF;
  11042. end;
  11043. S_L:
  11044. if ((taicpu(hp1).oper[0]^.val and $FF) = taicpu(hp1).oper[0]^.val) or
  11045. ((not(taicpu(hp1).oper[0]^.val) and $7F) = (not taicpu(hp1).oper[0]^.val)) then
  11046. begin
  11047. TryShiftDown := S_B;
  11048. TryShiftDownLimit := $FF;
  11049. end
  11050. else if ((taicpu(hp1).oper[0]^.val and $FFFF) = taicpu(hp1).oper[0]^.val) or
  11051. ((not(taicpu(hp1).oper[0]^.val) and $7FFF) = (not taicpu(hp1).oper[0]^.val)) then
  11052. begin
  11053. TryShiftDown := S_W;
  11054. TryShiftDownLimit := $FFFF;
  11055. end;
  11056. else
  11057. InternalError(2020112320);
  11058. end;
  11059. WorkingValue := taicpu(hp1).oper[0]^.val;
  11060. TestValMin := TestValMin and WorkingValue;
  11061. TestValMax := TestValMax and WorkingValue;
  11062. TestValSignedMax := TestValSignedMax and WorkingValue;
  11063. end;
  11064. A_OR:
  11065. begin
  11066. if not BitwiseOnly then
  11067. Break;
  11068. OrXorUsed := True;
  11069. WorkingValue := taicpu(hp1).oper[0]^.val;
  11070. TestValMin := TestValMin or WorkingValue;
  11071. TestValMax := TestValMax or WorkingValue;
  11072. TestValSignedMax := TestValSignedMax or WorkingValue;
  11073. end;
  11074. A_XOR:
  11075. begin
  11076. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  11077. begin
  11078. TestValMin := 0;
  11079. TestValMax := 0;
  11080. TestValSignedMax := 0;
  11081. end
  11082. else
  11083. begin
  11084. if not BitwiseOnly then
  11085. Break;
  11086. OrXorUsed := True;
  11087. WorkingValue := taicpu(hp1).oper[0]^.val;
  11088. TestValMin := TestValMin xor WorkingValue;
  11089. TestValMax := TestValMax xor WorkingValue;
  11090. TestValSignedMax := TestValSignedMax xor WorkingValue;
  11091. end;
  11092. end;
  11093. A_SHL:
  11094. begin
  11095. BitwiseOnly := False;
  11096. WorkingValue := taicpu(hp1).oper[0]^.val;
  11097. TestValMin := TestValMin shl WorkingValue;
  11098. TestValMax := TestValMax shl WorkingValue;
  11099. TestValSignedMax := TestValSignedMax shl WorkingValue;
  11100. end;
  11101. A_SHR,
  11102. { The first instruction was MOVZX, so the value won't be negative }
  11103. A_SAR:
  11104. begin
  11105. if InstrMax <> -1 then
  11106. BitwiseOnly := False
  11107. else
  11108. { we might be able to go smaller if SHR appears first }
  11109. case MinSize of
  11110. S_B:
  11111. ;
  11112. S_W:
  11113. if (taicpu(hp1).oper[0]^.val >= 8) then
  11114. begin
  11115. TryShiftDown := S_B;
  11116. TryShiftDownLimit := $FF;
  11117. TryShiftDownSignedLimit := $7F;
  11118. TryShiftDownSignedLimitLower := -128;
  11119. end;
  11120. S_L:
  11121. if (taicpu(hp1).oper[0]^.val >= 24) then
  11122. begin
  11123. TryShiftDown := S_B;
  11124. TryShiftDownLimit := $FF;
  11125. TryShiftDownSignedLimit := $7F;
  11126. TryShiftDownSignedLimitLower := -128;
  11127. end
  11128. else if (taicpu(hp1).oper[0]^.val >= 16) then
  11129. begin
  11130. TryShiftDown := S_W;
  11131. TryShiftDownLimit := $FFFF;
  11132. TryShiftDownSignedLimit := $7FFF;
  11133. TryShiftDownSignedLimitLower := -32768;
  11134. end;
  11135. else
  11136. InternalError(2020112321);
  11137. end;
  11138. WorkingValue := taicpu(hp1).oper[0]^.val;
  11139. if taicpu(hp1).opcode = A_SAR then
  11140. begin
  11141. TestValMin := SarInt64(TestValMin, WorkingValue);
  11142. TestValMax := SarInt64(TestValMax, WorkingValue);
  11143. TestValSignedMax := SarInt64(TestValSignedMax, WorkingValue);
  11144. end
  11145. else
  11146. begin
  11147. TestValMin := TestValMin shr WorkingValue;
  11148. TestValMax := TestValMax shr WorkingValue;
  11149. TestValSignedMax := TestValSignedMax shr WorkingValue;
  11150. end;
  11151. end;
  11152. else
  11153. InternalError(2020112303);
  11154. end;
  11155. end;
  11156. (*
  11157. A_IMUL:
  11158. case taicpu(hp1).ops of
  11159. 2:
  11160. begin
  11161. if not MatchOpType(hp1, top_reg, top_reg) or
  11162. { Has to be an exact match on the register }
  11163. (taicpu(hp1).oper[0]^.reg <> ThisReg) or
  11164. (taicpu(hp1).oper[1]^.reg <> ThisReg) then
  11165. Break;
  11166. TestValMin := TestValMin * TestValMin;
  11167. TestValMax := TestValMax * TestValMax;
  11168. TestValSignedMax := TestValSignedMax * TestValMax;
  11169. end;
  11170. 3:
  11171. begin
  11172. if not MatchOpType(hp1, top_const, top_reg, top_reg) or
  11173. { Has to be an exact match on the register }
  11174. (taicpu(hp1).oper[1]^.reg <> ThisReg) or
  11175. (taicpu(hp1).oper[2]^.reg <> ThisReg) or
  11176. ((taicpu(hp1).oper[0]^.val and UpperLimit) = taicpu(hp1).oper[0]^.val) or
  11177. { Is it in the negative range? }
  11178. (((not taicpu(hp1).oper[0]^.val) and (UpperLimit shr 1)) = (not taicpu(hp1).oper[0]^.val)) then
  11179. Break;
  11180. TestValMin := TestValMin * taicpu(hp1).oper[0]^.val;
  11181. TestValMax := TestValMax * taicpu(hp1).oper[0]^.val;
  11182. TestValSignedMax := TestValSignedMax * taicpu(hp1).oper[0]^.val;
  11183. end;
  11184. else
  11185. Break;
  11186. end;
  11187. A_IDIV:
  11188. case taicpu(hp1).ops of
  11189. 3:
  11190. begin
  11191. if not MatchOpType(hp1, top_const, top_reg, top_reg) or
  11192. { Has to be an exact match on the register }
  11193. (taicpu(hp1).oper[1]^.reg <> ThisReg) or
  11194. (taicpu(hp1).oper[2]^.reg <> ThisReg) or
  11195. ((taicpu(hp1).oper[0]^.val and UpperLimit) = taicpu(hp1).oper[0]^.val) or
  11196. { Is it in the negative range? }
  11197. (((not taicpu(hp1).oper[0]^.val) and (UpperLimit shr 1)) = (not taicpu(hp1).oper[0]^.val)) then
  11198. Break;
  11199. TestValMin := TestValMin div taicpu(hp1).oper[0]^.val;
  11200. TestValMax := TestValMax div taicpu(hp1).oper[0]^.val;
  11201. TestValSignedMax := TestValSignedMax div taicpu(hp1).oper[0]^.val;
  11202. end;
  11203. else
  11204. Break;
  11205. end;
  11206. *)
  11207. A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  11208. begin
  11209. { If there are no instructions in between, then we might be able to make a saving }
  11210. if UpperSignedOverflow or (taicpu(hp1).oper[0]^.typ <> top_reg) or (taicpu(hp1).oper[0]^.reg <> ThisReg) then
  11211. Break;
  11212. { We have something like:
  11213. movzbw %dl,%dx
  11214. ...
  11215. movswl %dx,%edx
  11216. Change the latter to a zero-extension then enter the
  11217. A_MOVZX case branch.
  11218. }
  11219. {$ifdef x86_64}
  11220. if (taicpu(hp1).opsize = S_LQ) and SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  11221. begin
  11222. { this becomes a zero extension from 32-bit to 64-bit, but
  11223. the upper 32 bits are already zero, so just delete the
  11224. instruction }
  11225. DebugMsg(SPeepholeOptimization + 'MovzMovsxd2MovzNop', hp1);
  11226. RemoveInstruction(hp1);
  11227. Result := True;
  11228. Exit;
  11229. end
  11230. else
  11231. {$endif x86_64}
  11232. begin
  11233. DebugMsg(SPeepholeOptimization + 'MovzMovs2MovzMovz', hp1);
  11234. taicpu(hp1).opcode := A_MOVZX;
  11235. {$ifdef x86_64}
  11236. case taicpu(hp1).opsize of
  11237. S_BQ:
  11238. begin
  11239. taicpu(hp1).opsize := S_BL;
  11240. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  11241. end;
  11242. S_WQ:
  11243. begin
  11244. taicpu(hp1).opsize := S_WL;
  11245. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  11246. end;
  11247. S_LQ:
  11248. begin
  11249. taicpu(hp1).opcode := A_MOV;
  11250. taicpu(hp1).opsize := S_L;
  11251. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  11252. { In this instance, we need to break out because the
  11253. instruction is no longer MOVZX or MOVSXD }
  11254. Result := True;
  11255. Exit;
  11256. end;
  11257. else
  11258. ;
  11259. end;
  11260. {$endif x86_64}
  11261. Result := CompressInstructions;
  11262. Exit;
  11263. end;
  11264. end;
  11265. A_MOVZX:
  11266. begin
  11267. if UpperUnsignedOverflow or (taicpu(hp1).oper[0]^.typ <> top_reg) then
  11268. Break;
  11269. if (InstrMax = -1) then
  11270. begin
  11271. if SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, ThisReg) then
  11272. begin
  11273. { Optimise around i40003 }
  11274. { Check to see if the active register is used afterwards }
  11275. TransferUsedRegs(TmpUsedRegs);
  11276. IncludeRegInUsedRegs(ThisReg, TmpUsedRegs);
  11277. if (
  11278. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) or
  11279. not RegUsedAfterInstruction(ThisReg, hp1, TmpUsedRegs)
  11280. ) and
  11281. (taicpu(p).opsize = S_WL) and (taicpu(hp1).opsize = S_BL)
  11282. {$ifndef x86_64}
  11283. and (
  11284. (taicpu(p).oper[0]^.typ <> top_reg) or
  11285. { Cannot encode byte-sized ESI, EDI, EBP or ESP under i386 }
  11286. (GetSupReg(taicpu(p).oper[0]^.reg) in [RS_EAX, RS_EBX, RS_ECX, RS_EDX])
  11287. )
  11288. {$endif not x86_64}
  11289. then
  11290. begin
  11291. if (taicpu(p).oper[0]^.typ = top_reg) then
  11292. setsubreg(taicpu(p).oper[0]^.reg, R_SUBL);
  11293. DebugMsg(SPeepholeOptimization + 'movzwl2movzbl 1', p);
  11294. taicpu(p).opsize := S_BL;
  11295. { Only remove if the active register is overwritten }
  11296. if SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, ThisReg) then
  11297. begin
  11298. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 2a', hp1);
  11299. RemoveInstruction(hp1);
  11300. end;
  11301. Result := True;
  11302. Exit;
  11303. end;
  11304. end
  11305. else
  11306. begin
  11307. { Will return false if the second parameter isn't ThisReg
  11308. (can happen on -O2 and under) }
  11309. if Reg1WriteOverwritesReg2Entirely(taicpu(hp1).oper[1]^.reg, ThisReg) then
  11310. begin
  11311. { The two MOVZX instructions are adjacent, so remove the first one }
  11312. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 5', p);
  11313. RemoveCurrentP(p);
  11314. Result := True;
  11315. Exit;
  11316. end;
  11317. Break;
  11318. end;
  11319. end;
  11320. Result := CompressInstructions;
  11321. Exit;
  11322. end;
  11323. else
  11324. { This includes ADC, SBB and IDIV }
  11325. Break;
  11326. end;
  11327. if not CheckOverflowConditions then
  11328. Break;
  11329. { Contains highest index (so instruction count - 1) }
  11330. Inc(InstrMax);
  11331. if InstrMax > High(InstrList) then
  11332. SetLength(InstrList, InstrMax + LIST_STEP_SIZE);
  11333. InstrList[InstrMax] := taicpu(hp1);
  11334. end;
  11335. end;
  11336. {$pop}
  11337. function TX86AsmOptimizer.OptPass2Imul(var p : tai) : boolean;
  11338. var
  11339. hp1 : tai;
  11340. begin
  11341. Result:=false;
  11342. if (taicpu(p).ops >= 2) and
  11343. ((taicpu(p).oper[0]^.typ = top_const) or
  11344. ((taicpu(p).oper[0]^.typ = top_ref) and (taicpu(p).oper[0]^.ref^.refaddr=addr_full))) and
  11345. (taicpu(p).oper[1]^.typ = top_reg) and
  11346. ((taicpu(p).ops = 2) or
  11347. ((taicpu(p).oper[2]^.typ = top_reg) and
  11348. (taicpu(p).oper[2]^.reg = taicpu(p).oper[1]^.reg))) and
  11349. GetLastInstruction(p,hp1) and
  11350. MatchInstruction(hp1,A_MOV,[]) and
  11351. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  11352. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  11353. begin
  11354. TransferUsedRegs(TmpUsedRegs);
  11355. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,p,TmpUsedRegs)) or
  11356. ((taicpu(p).ops = 3) and (taicpu(p).oper[1]^.reg=taicpu(p).oper[2]^.reg)) then
  11357. { change
  11358. mov reg1,reg2
  11359. imul y,reg2 to imul y,reg1,reg2 }
  11360. begin
  11361. taicpu(p).ops := 3;
  11362. taicpu(p).loadreg(2,taicpu(p).oper[1]^.reg);
  11363. taicpu(p).loadreg(1,taicpu(hp1).oper[0]^.reg);
  11364. DebugMsg(SPeepholeOptimization + 'MovImul2Imul done',p);
  11365. RemoveInstruction(hp1);
  11366. result:=true;
  11367. end;
  11368. end;
  11369. end;
  11370. procedure TX86AsmOptimizer.ConvertJumpToRET(const p: tai; const ret_p: tai);
  11371. var
  11372. ThisLabel: TAsmLabel;
  11373. begin
  11374. ThisLabel := tasmlabel(taicpu(p).oper[0]^.ref^.symbol);
  11375. ThisLabel.decrefs;
  11376. taicpu(p).condition := C_None;
  11377. taicpu(p).opcode := A_RET;
  11378. taicpu(p).is_jmp := false;
  11379. taicpu(p).ops := taicpu(ret_p).ops;
  11380. case taicpu(ret_p).ops of
  11381. 0:
  11382. taicpu(p).clearop(0);
  11383. 1:
  11384. taicpu(p).loadconst(0,taicpu(ret_p).oper[0]^.val);
  11385. else
  11386. internalerror(2016041301);
  11387. end;
  11388. { If the original label is now dead, it might turn out that the label
  11389. immediately follows p. As a result, everything beyond it, which will
  11390. be just some final register configuration and a RET instruction, is
  11391. now dead code. [Kit] }
  11392. { NOTE: This is much faster than introducing a OptPass2RET routine and
  11393. running RemoveDeadCodeAfterJump for each RET instruction, because
  11394. this optimisation rarely happens and most RETs appear at the end of
  11395. routines where there is nothing that can be stripped. [Kit] }
  11396. if not ThisLabel.is_used then
  11397. RemoveDeadCodeAfterJump(p);
  11398. end;
  11399. function TX86AsmOptimizer.OptPass2SETcc(var p: tai): boolean;
  11400. var
  11401. hp1,hp2,next: tai; SetC, JumpC: TAsmCond;
  11402. Unconditional, PotentialModified: Boolean;
  11403. OperPtr: POper;
  11404. NewRef: TReference;
  11405. InstrList: array of taicpu;
  11406. InstrMax, Index: Integer;
  11407. const
  11408. {$ifdef DEBUG_AOPTCPU}
  11409. SNoFlags: shortstring = ' so the flags aren''t modified';
  11410. {$else DEBUG_AOPTCPU}
  11411. SNoFlags = '';
  11412. {$endif DEBUG_AOPTCPU}
  11413. begin
  11414. Result:=false;
  11415. if MatchOpType(taicpu(p),top_reg) and GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[0]^.reg) then
  11416. begin
  11417. if MatchInstruction(hp1, A_TEST, [S_B]) and
  11418. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  11419. (taicpu(hp1).oper[0]^.reg = taicpu(hp1).oper[1]^.reg) and
  11420. (taicpu(p).oper[0]^.reg = taicpu(hp1).oper[1]^.reg) and
  11421. GetNextInstruction(hp1, hp2) and
  11422. MatchInstruction(hp2, A_Jcc, A_SETcc, []) then
  11423. { Change from: To:
  11424. set(C) %reg j(~C) label
  11425. test %reg,%reg/cmp $0,%reg
  11426. je label
  11427. set(C) %reg j(C) label
  11428. test %reg,%reg/cmp $0,%reg
  11429. jne label
  11430. (Also do something similar with sete/setne instead of je/jne)
  11431. }
  11432. begin
  11433. { Before we do anything else, we need to check the instructions
  11434. in between SETcc and TEST to make sure they don't modify the
  11435. FLAGS register - if -O2 or under, there won't be any
  11436. instructions between SET and TEST }
  11437. TransferUsedRegs(TmpUsedRegs);
  11438. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  11439. if (cs_opt_level3 in current_settings.optimizerswitches) then
  11440. begin
  11441. next := p;
  11442. SetLength(InstrList, 0);
  11443. InstrMax := -1;
  11444. PotentialModified := False;
  11445. { Make a note of every instruction that modifies the FLAGS
  11446. register }
  11447. while GetNextInstruction(next, next) and (next <> hp1) do
  11448. begin
  11449. if next.typ <> ait_instruction then
  11450. { GetNextInstructionUsingReg should have returned False }
  11451. InternalError(2021051701);
  11452. if RegModifiedByInstruction(NR_DEFAULTFLAGS, next) then
  11453. begin
  11454. case taicpu(next).opcode of
  11455. A_SETcc,
  11456. A_CMOVcc,
  11457. A_Jcc:
  11458. begin
  11459. if PotentialModified then
  11460. { Not safe because the flags were modified earlier }
  11461. Exit
  11462. else
  11463. { Condition is the same as the initial SETcc, so this is safe
  11464. (don't add to instruction list though) }
  11465. Continue;
  11466. end;
  11467. A_ADD:
  11468. begin
  11469. if { LEA doesn't support 8-bit in general and 16-bit on x86-64 operands }
  11470. (taicpu(next).opsize in [S_B{$ifdef x86_64},S_W{$endif x86_64}]) or
  11471. (taicpu(next).oper[1]^.typ <> top_reg) or
  11472. { Must write to a register }
  11473. (taicpu(next).oper[0]^.typ = top_ref) then
  11474. { Require a constant or a register }
  11475. Exit;
  11476. PotentialModified := True;
  11477. end;
  11478. A_SUB:
  11479. begin
  11480. if { LEA doesn't support 8-bit in general and 16-bit on x86-64 operands }
  11481. (taicpu(next).opsize in [S_B{$ifdef x86_64},S_W{$endif x86_64}]) or
  11482. (taicpu(next).oper[1]^.typ <> top_reg) or
  11483. { Must write to a register }
  11484. (taicpu(next).oper[0]^.typ <> top_const) or
  11485. (taicpu(next).oper[0]^.val = $80000000) then
  11486. { Can't subtract a register with LEA - also
  11487. check that the value isn't -2^31, as this
  11488. can't be negated }
  11489. Exit;
  11490. PotentialModified := True;
  11491. end;
  11492. A_SAL,
  11493. A_SHL:
  11494. begin
  11495. if { LEA doesn't support 8-bit in general and 16-bit on x86-64 operands }
  11496. (taicpu(next).opsize in [S_B{$ifdef x86_64},S_W{$endif x86_64}]) or
  11497. (taicpu(next).oper[1]^.typ <> top_reg) or
  11498. { Must write to a register }
  11499. (taicpu(next).oper[0]^.typ <> top_const) or
  11500. (taicpu(next).oper[0]^.val < 0) or
  11501. (taicpu(next).oper[0]^.val > 3) then
  11502. Exit;
  11503. PotentialModified := True;
  11504. end;
  11505. A_IMUL:
  11506. begin
  11507. if (taicpu(next).ops <> 3) or
  11508. (taicpu(next).oper[1]^.typ <> top_reg) or
  11509. { Must write to a register }
  11510. (taicpu(next).oper[2]^.val in [2,3,4,5,8,9]) then
  11511. { We can convert "imul x,%reg1,%reg2" (where x = 2, 4 or 8)
  11512. to "lea (%reg1,x),%reg2". If x = 3, 5 or 9, we can
  11513. change this to "lea (%reg1,%reg1,(x-1)),%reg2" }
  11514. Exit
  11515. else
  11516. PotentialModified := True;
  11517. end;
  11518. else
  11519. { Don't know how to change this, so abort }
  11520. Exit;
  11521. end;
  11522. { Contains highest index (so instruction count - 1) }
  11523. Inc(InstrMax);
  11524. if InstrMax > High(InstrList) then
  11525. SetLength(InstrList, InstrMax + LIST_STEP_SIZE);
  11526. InstrList[InstrMax] := taicpu(next);
  11527. end;
  11528. UpdateUsedRegs(TmpUsedRegs, tai(next.next));
  11529. end;
  11530. if not Assigned(next) or (next <> hp1) then
  11531. { It should be equal to hp1 }
  11532. InternalError(2021051702);
  11533. { Cycle through each instruction and check to see if we can
  11534. change them to versions that don't modify the flags }
  11535. if (InstrMax >= 0) then
  11536. begin
  11537. for Index := 0 to InstrMax do
  11538. case InstrList[Index].opcode of
  11539. A_ADD:
  11540. begin
  11541. DebugMsg(SPeepholeOptimization + 'ADD -> LEA' + SNoFlags, InstrList[Index]);
  11542. InstrList[Index].opcode := A_LEA;
  11543. reference_reset(NewRef, 1, []);
  11544. NewRef.base := InstrList[Index].oper[1]^.reg;
  11545. if InstrList[Index].oper[0]^.typ = top_reg then
  11546. begin
  11547. NewRef.index := InstrList[Index].oper[0]^.reg;
  11548. NewRef.scalefactor := 1;
  11549. end
  11550. else
  11551. NewRef.offset := InstrList[Index].oper[0]^.val;
  11552. InstrList[Index].loadref(0, NewRef);
  11553. end;
  11554. A_SUB:
  11555. begin
  11556. DebugMsg(SPeepholeOptimization + 'SUB -> LEA' + SNoFlags, InstrList[Index]);
  11557. InstrList[Index].opcode := A_LEA;
  11558. reference_reset(NewRef, 1, []);
  11559. NewRef.base := InstrList[Index].oper[1]^.reg;
  11560. NewRef.offset := -InstrList[Index].oper[0]^.val;
  11561. InstrList[Index].loadref(0, NewRef);
  11562. end;
  11563. A_SHL,
  11564. A_SAL:
  11565. begin
  11566. DebugMsg(SPeepholeOptimization + 'SHL -> LEA' + SNoFlags, InstrList[Index]);
  11567. InstrList[Index].opcode := A_LEA;
  11568. reference_reset(NewRef, 1, []);
  11569. NewRef.index := InstrList[Index].oper[1]^.reg;
  11570. NewRef.scalefactor := 1 shl (InstrList[Index].oper[0]^.val);
  11571. InstrList[Index].loadref(0, NewRef);
  11572. end;
  11573. A_IMUL:
  11574. begin
  11575. DebugMsg(SPeepholeOptimization + 'IMUL -> LEA' + SNoFlags, InstrList[Index]);
  11576. InstrList[Index].opcode := A_LEA;
  11577. reference_reset(NewRef, 1, []);
  11578. NewRef.index := InstrList[Index].oper[1]^.reg;
  11579. case InstrList[Index].oper[0]^.val of
  11580. 2, 4, 8:
  11581. NewRef.scalefactor := InstrList[Index].oper[0]^.val;
  11582. else {3, 5 and 9}
  11583. begin
  11584. NewRef.scalefactor := InstrList[Index].oper[0]^.val - 1;
  11585. NewRef.base := InstrList[Index].oper[1]^.reg;
  11586. end;
  11587. end;
  11588. InstrList[Index].loadref(0, NewRef);
  11589. end;
  11590. else
  11591. InternalError(2021051710);
  11592. end;
  11593. end;
  11594. { Mark the FLAGS register as used across this whole block }
  11595. AllocRegBetween(NR_DEFAULTFLAGS, p, hp1, UsedRegs);
  11596. end;
  11597. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  11598. JumpC := taicpu(hp2).condition;
  11599. Unconditional := False;
  11600. if conditions_equal(JumpC, C_E) then
  11601. SetC := inverse_cond(taicpu(p).condition)
  11602. else if conditions_equal(JumpC, C_NE) then
  11603. SetC := taicpu(p).condition
  11604. else
  11605. { We've got something weird here (and inefficent) }
  11606. begin
  11607. DebugMsg('DEBUG: Inefficient jump - check code generation', p);
  11608. SetC := C_NONE;
  11609. { JAE/JNB will always branch (use 'condition_in', since C_AE <> C_NB normally) }
  11610. if condition_in(C_AE, JumpC) then
  11611. Unconditional := True
  11612. else
  11613. { Not sure what to do with this jump - drop out }
  11614. Exit;
  11615. end;
  11616. RemoveInstruction(hp1);
  11617. if Unconditional then
  11618. MakeUnconditional(taicpu(hp2))
  11619. else
  11620. begin
  11621. if SetC = C_NONE then
  11622. InternalError(2018061402);
  11623. taicpu(hp2).SetCondition(SetC);
  11624. end;
  11625. { as hp2 is a jump, we cannot use RegUsedAfterInstruction but we have to check if it is included in
  11626. TmpUsedRegs }
  11627. if not TmpUsedRegs[getregtype(taicpu(p).oper[0]^.reg)].IsUsed(taicpu(p).oper[0]^.reg) then
  11628. begin
  11629. RemoveCurrentp(p, hp2);
  11630. if taicpu(hp2).opcode = A_SETcc then
  11631. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/SETcc -> SETcc',p)
  11632. else
  11633. begin
  11634. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/Jcc -> Jcc',p);
  11635. if (cs_opt_level3 in current_settings.optimizerswitches) then
  11636. Include(OptsToCheck, aoc_DoPass2JccOpts);
  11637. end;
  11638. end
  11639. else
  11640. if taicpu(hp2).opcode = A_SETcc then
  11641. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/SETcc -> SETcc/SETcc',p)
  11642. else
  11643. DebugMsg(SPeepholeOptimization + 'SETcc/TEST/Jcc -> SETcc/Jcc',p);
  11644. Result := True;
  11645. end
  11646. else if
  11647. { Make sure the instructions are adjacent }
  11648. (
  11649. not (cs_opt_level3 in current_settings.optimizerswitches) or
  11650. GetNextInstruction(p, hp1)
  11651. ) and
  11652. MatchInstruction(hp1, A_MOV, [S_B]) and
  11653. { Writing to memory is allowed }
  11654. MatchOperand(taicpu(p).oper[0]^, taicpu(hp1).oper[0]^.reg) then
  11655. begin
  11656. {
  11657. Watch out for sequences such as:
  11658. set(c)b %regb
  11659. movb %regb,(ref)
  11660. movb $0,1(ref)
  11661. movb $0,2(ref)
  11662. movb $0,3(ref)
  11663. Much more efficient to turn it into:
  11664. movl $0,%regl
  11665. set(c)b %regb
  11666. movl %regl,(ref)
  11667. Or:
  11668. set(c)b %regb
  11669. movzbl %regb,%regl
  11670. movl %regl,(ref)
  11671. }
  11672. if (taicpu(hp1).oper[1]^.typ = top_ref) and
  11673. GetNextInstruction(hp1, hp2) and
  11674. MatchInstruction(hp2, A_MOV, [S_B]) and
  11675. (taicpu(hp2).oper[1]^.typ = top_ref) and
  11676. CheckMemoryWrite(taicpu(hp1), taicpu(hp2)) then
  11677. begin
  11678. { Don't do anything else except set Result to True }
  11679. end
  11680. else
  11681. begin
  11682. if taicpu(p).oper[0]^.typ = top_reg then
  11683. begin
  11684. TransferUsedRegs(TmpUsedRegs);
  11685. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  11686. end;
  11687. { If it's not a register, it's a memory address }
  11688. if (taicpu(p).oper[0]^.typ <> top_reg) or RegUsedAfterInstruction(taicpu(p).oper[0]^.reg, hp1, TmpUsedRegs) then
  11689. begin
  11690. { Even if the register is still in use, we can minimise the
  11691. pipeline stall by changing the MOV into another SETcc. }
  11692. taicpu(hp1).opcode := A_SETcc;
  11693. taicpu(hp1).condition := taicpu(p).condition;
  11694. if taicpu(hp1).oper[1]^.typ = top_ref then
  11695. begin
  11696. { Swapping the operand pointers like this is probably a
  11697. bit naughty, but it is far faster than using loadoper
  11698. to transfer the reference from oper[1] to oper[0] if
  11699. you take into account the extra procedure calls and
  11700. the memory allocation and deallocation required }
  11701. OperPtr := taicpu(hp1).oper[1];
  11702. taicpu(hp1).oper[1] := taicpu(hp1).oper[0];
  11703. taicpu(hp1).oper[0] := OperPtr;
  11704. end
  11705. else
  11706. taicpu(hp1).oper[0]^.reg := taicpu(hp1).oper[1]^.reg;
  11707. taicpu(hp1).clearop(1);
  11708. taicpu(hp1).ops := 1;
  11709. DebugMsg(SPeepholeOptimization + 'SETcc/Mov -> SETcc/SETcc',p);
  11710. end
  11711. else
  11712. begin
  11713. if taicpu(hp1).oper[1]^.typ = top_reg then
  11714. AllocRegBetween(taicpu(hp1).oper[1]^.reg,p,hp1,UsedRegs);
  11715. taicpu(p).loadoper(0, taicpu(hp1).oper[1]^);
  11716. RemoveInstruction(hp1);
  11717. DebugMsg(SPeepholeOptimization + 'SETcc/Mov -> SETcc',p);
  11718. end
  11719. end;
  11720. Result := True;
  11721. end;
  11722. end;
  11723. end;
  11724. function TX86AsmOptimizer.TryCmpCMovOpts(var p, hp1: tai): Boolean;
  11725. var
  11726. hp2, pCond, pFirstMOV, pLastMOV, pCMOV: tai;
  11727. TargetReg: TRegister;
  11728. condition, inverted_condition: TAsmCond;
  11729. FoundMOV: Boolean;
  11730. begin
  11731. Result := False;
  11732. { In some situations, the CMOV optimisations in OptPass2Jcc can't
  11733. create the most optimial instructions possible due to limited
  11734. register availability, and there are situations where two
  11735. complementary "simple" CMOV blocks are created which, after the fact
  11736. can be merged into a "double" block. For example:
  11737. movw $257,%ax
  11738. movw $2,%r8w
  11739. xorl r9d,%r9d
  11740. testw $16,18(%rcx)
  11741. cmovew %ax,%dx
  11742. cmovew %r8w,%bx
  11743. cmovel %r9d,%r14d
  11744. movw $1283,%ax
  11745. movw $4,%r8w
  11746. movl $9,%r9d
  11747. cmovnew %ax,%dx
  11748. cmovnew %r8w,%bx
  11749. cmovnel %r9d,%r14d
  11750. The CMOVNE instructions at the end can be removed, and the
  11751. destination registers copied into the MOV instructions directly
  11752. above them, before finally being moved to before the first CMOVE
  11753. instructions, to produce:
  11754. movw $257,%ax
  11755. movw $2,%r8w
  11756. xorl r9d,%r9d
  11757. testw $16,18(%rcx)
  11758. movw $1283,%dx
  11759. movw $4,%bx
  11760. movl $9,%r14d
  11761. cmovew %ax,%dx
  11762. cmovew %r8w,%bx
  11763. cmovel %r9d,%r14d
  11764. Which can then be later optimised to:
  11765. movw $257,%ax
  11766. movw $2,%r8w
  11767. xorl r9d,%r9d
  11768. movw $1283,%dx
  11769. movw $4,%bx
  11770. movl $9,%r14d
  11771. testw $16,18(%rcx)
  11772. cmovew %ax,%dx
  11773. cmovew %r8w,%bx
  11774. cmovel %r9d,%r14d
  11775. }
  11776. TargetReg := taicpu(hp1).oper[1]^.reg;
  11777. condition := taicpu(hp1).condition;
  11778. inverted_condition := inverse_cond(condition);
  11779. pFirstMov := nil;
  11780. pLastMov := nil;
  11781. pCMOV := nil;
  11782. if (p.typ = ait_instruction) then
  11783. pCond := p
  11784. else if not GetNextInstruction(p, pCond) then
  11785. InternalError(2024012501);
  11786. if not MatchInstruction(pCond, A_CMP, A_TEST, []) then
  11787. { We should get the CMP or TEST instructeion }
  11788. InternalError(2024012502);
  11789. if (
  11790. (taicpu(hp1).oper[0]^.typ = top_reg) or
  11791. IsRefSafe(taicpu(hp1).oper[0]^.ref)
  11792. ) then
  11793. begin
  11794. { We have to tread carefully here, hence why we're not using
  11795. GetNextInstructionUsingReg... we can only accept MOV and other
  11796. CMOV instructions. Anything else and we must drop out}
  11797. hp2 := hp1;
  11798. while GetNextInstruction(hp2, hp2) and (hp2 <> BlockEnd) do
  11799. begin
  11800. if (hp2.typ <> ait_instruction) then
  11801. Exit;
  11802. case taicpu(hp2).opcode of
  11803. A_MOV:
  11804. begin
  11805. if not Assigned(pFirstMov) then
  11806. pFirstMov := hp2;
  11807. pLastMOV := hp2;
  11808. if not MatchOpType(taicpu(hp2), top_const, top_reg) then
  11809. { Something different - drop out }
  11810. Exit;
  11811. { Otherwise, leave it for now }
  11812. end;
  11813. A_CMOVcc:
  11814. begin
  11815. if taicpu(hp2).condition = inverted_condition then
  11816. begin
  11817. { We found what we're looking for }
  11818. if taicpu(hp2).oper[1]^.reg = TargetReg then
  11819. begin
  11820. if (taicpu(hp2).oper[0]^.typ = top_reg) or
  11821. IsRefSafe(taicpu(hp2).oper[0]^.ref) then
  11822. begin
  11823. pCMOV := hp2;
  11824. Break;
  11825. end
  11826. else
  11827. { Unsafe reference - drop out }
  11828. Exit;
  11829. end;
  11830. end
  11831. else if taicpu(hp2).condition <> condition then
  11832. { Something weird - drop out }
  11833. Exit;
  11834. end;
  11835. else
  11836. { Invalid }
  11837. Exit;
  11838. end;
  11839. end;
  11840. if not Assigned(pCMOV) then
  11841. { No complementary CMOV found }
  11842. Exit;
  11843. if not Assigned(pFirstMov) or (taicpu(pCMOV).oper[0]^.typ = top_ref) then
  11844. begin
  11845. { Don't need to do anything special or search for a matching MOV }
  11846. Asml.Remove(pCMOV);
  11847. if RegInInstruction(TargetReg, pCond) then
  11848. { Make sure we don't overwrite the register if it's being used in the condition }
  11849. Asml.InsertAfter(pCMOV, pCond)
  11850. else
  11851. Asml.InsertBefore(pCMOV, pCond);
  11852. taicpu(pCMOV).opcode := A_MOV;
  11853. taicpu(pCMOV).condition := C_None;
  11854. { Don't need to worry about allocating new registers in these cases }
  11855. DebugMsg(SPeepholeOptimization + 'CMovCMov2MovCMov 2', pCMOV);
  11856. Result := True;
  11857. Exit;
  11858. end
  11859. else
  11860. begin
  11861. DebugMsg(SPeepholeOptimization + 'CMovCMov2MovCMov 1', hp1);
  11862. FoundMOV := False;
  11863. { Search for the MOV that sets the target register }
  11864. hp2 := pFirstMov;
  11865. repeat
  11866. if (taicpu(hp2).opcode = A_MOV) and
  11867. (taicpu(hp2).oper[1]^.typ = top_reg) and
  11868. SuperRegistersEqual(taicpu(hp2).oper[1]^.reg, taicpu(pCMOV).oper[0]^.reg) then
  11869. begin
  11870. { Change the destination }
  11871. taicpu(hp2).loadreg(1, newreg(R_INTREGISTER, getsupreg(TargetReg), getsubreg(taicpu(hp2).oper[1]^.reg)));
  11872. if not FoundMOV then
  11873. begin
  11874. FoundMOV := True;
  11875. { Make sure the register is allocated }
  11876. AllocRegBetween(TargetReg, p, hp2, UsedRegs);
  11877. end;
  11878. hp1 := tai(hp2.Previous);
  11879. Asml.Remove(hp2);
  11880. if RegInInstruction(TargetReg, pCond) then
  11881. { Make sure we don't overwrite the register if it's being used in the condition }
  11882. Asml.InsertAfter(hp2, pCond)
  11883. else
  11884. Asml.InsertBefore(hp2, pCond);
  11885. if (hp2 = pLastMov) then
  11886. { If the MOV instruction is the last one, "hp2 = pLastMOV" won't trigger }
  11887. Break;
  11888. hp2 := hp1;
  11889. end;
  11890. until (hp2 = pLastMOV) or not GetNextInstruction(hp2, hp2) or (hp2 = BlockEnd) or (hp2.typ <> ait_instruction);
  11891. if FoundMOV then
  11892. { Delete the CMOV }
  11893. RemoveInstruction(pCMOV)
  11894. else
  11895. begin
  11896. { If no MOV was found, we have to actually move and transmute the CMOV }
  11897. Asml.Remove(pCMOV);
  11898. if RegInInstruction(TargetReg, pCond) then
  11899. { Make sure we don't overwrite the register if it's being used in the condition }
  11900. Asml.InsertAfter(pCMOV, pCond)
  11901. else
  11902. Asml.InsertBefore(pCMOV, pCond);
  11903. taicpu(pCMOV).opcode := A_MOV;
  11904. taicpu(pCMOV).condition := C_None;
  11905. end;
  11906. Result := True;
  11907. Exit;
  11908. end;
  11909. end;
  11910. end;
  11911. function TX86AsmOptimizer.OptPass2Cmp(var p: tai): Boolean;
  11912. var
  11913. hp1, hp2, pCond: tai;
  11914. begin
  11915. Result := False;
  11916. { Search ahead for CMOV instructions }
  11917. if (cs_opt_level2 in current_settings.optimizerswitches) then
  11918. begin
  11919. hp1 := p;
  11920. hp2 := p;
  11921. pCond := nil; { To prevent compiler warnings }
  11922. { For TryCmpCMOVOpts, try to insert MOVs before the allocation of
  11923. DEFAULTFLAGS }
  11924. if not SetAndTest(FindRegAllocBackward(NR_DEFAULTFLAGS, p), pCond) or
  11925. (tai_regalloc(pCond).ratype = ra_dealloc) then
  11926. pCond := p;
  11927. while GetNextInstruction(hp1, hp1) and (hp1 <> BlockEnd) do
  11928. begin
  11929. if (hp1.typ <> ait_instruction) then
  11930. { Break out on markers and labels etc. }
  11931. Break;
  11932. case taicpu(hp1).opcode of
  11933. A_MOV:
  11934. { Ignore regular MOVs unless they are obviously not related
  11935. to a CMOV block }
  11936. if taicpu(hp1).oper[1]^.typ <> top_reg then
  11937. Break;
  11938. A_CMOVcc:
  11939. if TryCmpCMovOpts(pCond, hp1) then
  11940. begin
  11941. hp1 := hp2;
  11942. { p itself isn't changed, and we're still inside a
  11943. while loop to catch subsequent CMOVs, so just flag
  11944. a new iteration }
  11945. Include(OptsToCheck, aoc_ForceNewIteration);
  11946. Continue;
  11947. end;
  11948. else
  11949. { Drop out if we find anything else }
  11950. Break;
  11951. end;
  11952. hp2 := hp1;
  11953. end;
  11954. end;
  11955. end;
  11956. function TX86AsmOptimizer.OptPass2Test(var p: tai): Boolean;
  11957. var
  11958. hp1, hp2, pCond: tai;
  11959. SourceReg, TargetReg: TRegister;
  11960. begin
  11961. Result := False;
  11962. { In some situations, we end up with an inefficient arrangement of
  11963. instructions in the form of:
  11964. or %reg1,%reg2
  11965. (%reg1 deallocated)
  11966. test %reg2,%reg2
  11967. mov x,%reg2
  11968. we may be able to swap and rearrange the registers to produce:
  11969. or %reg2,%reg1
  11970. mov x,%reg2
  11971. test %reg1,%reg1
  11972. (%reg1 deallocated)
  11973. }
  11974. if (cs_opt_level3 in current_settings.optimizerswitches) and
  11975. (taicpu(p).oper[1]^.typ = top_reg) and
  11976. (
  11977. MatchOperand(taicpu(p).oper[0]^, taicpu(p).oper[1]^.reg) or
  11978. MatchOperand(taicpu(p).oper[0]^, -1)
  11979. ) and
  11980. GetNextInstruction(p, hp1) and
  11981. MatchInstruction(hp1, A_MOV, []) and
  11982. (taicpu(hp1).oper[1]^.typ = top_reg) and
  11983. SuperRegistersEqual(taicpu(hp1).oper[1]^.reg, taicpu(p).oper[1]^.reg) then
  11984. begin
  11985. TargetReg := taicpu(p).oper[1]^.reg;
  11986. { Now look backwards to find a simple commutative operation: ADD,
  11987. IMUL (2-register version), OR, AND or XOR - whose destination
  11988. register is the same as TEST }
  11989. hp2 := p;
  11990. while GetLastInstruction(hp2, hp2) and (hp2.typ = ait_instruction) do
  11991. if RegInInstruction(TargetReg, hp2) then
  11992. begin
  11993. if MatchInstruction(hp2, [A_ADD, A_IMUL, A_OR, A_AND, A_XOR], [taicpu(p).opsize]) and
  11994. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  11995. (taicpu(hp2).oper[1]^.reg = TargetReg) and
  11996. (taicpu(hp2).oper[0]^.reg <> TargetReg) then
  11997. begin
  11998. SourceReg := taicpu(hp2).oper[0]^.reg;
  11999. if
  12000. { Make sure the MOV doesn't use the other register }
  12001. not RegInOp(SourceReg, taicpu(hp1).oper[0]^) and
  12002. { And make sure the source register is not used afterwards }
  12003. not RegInUsedRegs(SourceReg, UsedRegs) then
  12004. begin
  12005. DebugMsg(SPeepholeOptimization + 'OpTest2OpTest (register swap) done', hp2);
  12006. taicpu(hp2).oper[0]^.reg := TargetReg;
  12007. taicpu(hp2).oper[1]^.reg := SourceReg;
  12008. if taicpu(p).oper[0]^.typ = top_reg then
  12009. taicpu(p).oper[0]^.reg := SourceReg;
  12010. taicpu(p).oper[1]^.reg := SourceReg;
  12011. IncludeRegInUsedRegs(SourceReg, UsedRegs);
  12012. AllocRegBetween(SourceReg, hp2, p, UsedRegs);
  12013. Include(OptsToCheck, aoc_ForceNewIteration);
  12014. { We can still check the following optimisations since
  12015. the instruction is still a TEST }
  12016. end;
  12017. end;
  12018. Break;
  12019. end;
  12020. end;
  12021. { Search ahead3 for CMOV instructions }
  12022. if (cs_opt_level2 in current_settings.optimizerswitches) then
  12023. begin
  12024. hp1 := p;
  12025. hp2 := p;
  12026. pCond := nil; { To prevent compiler warnings }
  12027. { For TryCmpCMOVOpts, try to insert MOVs before the allocation of
  12028. DEFAULTFLAGS }
  12029. if not SetAndTest(FindRegAllocBackward(NR_DEFAULTFLAGS, p), pCond) or
  12030. (tai_regalloc(pCond).ratype = ra_dealloc) then
  12031. pCond := p;
  12032. while GetNextInstruction(hp1, hp1) and (hp1 <> BlockEnd) do
  12033. begin
  12034. if (hp1.typ <> ait_instruction) then
  12035. { Break out on markers and labels etc. }
  12036. Break;
  12037. case taicpu(hp1).opcode of
  12038. A_MOV:
  12039. { Ignore regular MOVs unless they are obviously not related
  12040. to a CMOV block }
  12041. if taicpu(hp1).oper[1]^.typ <> top_reg then
  12042. Break;
  12043. A_CMOVcc:
  12044. if TryCmpCMovOpts(pCond, hp1) then
  12045. begin
  12046. hp1 := hp2;
  12047. { p itself isn't changed, and we're still inside a
  12048. while loop to catch subsequent CMOVs, so just flag
  12049. a new iteration }
  12050. Include(OptsToCheck, aoc_ForceNewIteration);
  12051. Continue;
  12052. end;
  12053. else
  12054. { Drop out if we find anything else }
  12055. Break;
  12056. end;
  12057. hp2 := hp1;
  12058. end;
  12059. end;
  12060. end;
  12061. function TX86AsmOptimizer.OptPass2Jmp(var p : tai) : boolean;
  12062. var
  12063. hp1: tai;
  12064. Count: Integer;
  12065. OrigLabel: TAsmLabel;
  12066. begin
  12067. result := False;
  12068. { Sometimes, the optimisations below can permit this }
  12069. RemoveDeadCodeAfterJump(p);
  12070. if (taicpu(p).oper[0]^.typ=top_ref) and (taicpu(p).oper[0]^.ref^.refaddr=addr_full) and (taicpu(p).oper[0]^.ref^.base=NR_NO) and
  12071. (taicpu(p).oper[0]^.ref^.index=NR_NO) and (taicpu(p).oper[0]^.ref^.symbol is tasmlabel) then
  12072. begin
  12073. OrigLabel := TAsmLabel(taicpu(p).oper[0]^.ref^.symbol);
  12074. { Also a side-effect of optimisations }
  12075. if CollapseZeroDistJump(p, OrigLabel) then
  12076. begin
  12077. Result := True;
  12078. Exit;
  12079. end;
  12080. hp1 := GetLabelWithSym(OrigLabel);
  12081. if (taicpu(p).condition=C_None) and assigned(hp1) and SkipLabels(hp1,hp1) and (hp1.typ = ait_instruction) then
  12082. begin
  12083. if taicpu(hp1).opcode = A_RET then
  12084. begin
  12085. {
  12086. change
  12087. jmp .L1
  12088. ...
  12089. .L1:
  12090. ret
  12091. into
  12092. ret
  12093. }
  12094. begin
  12095. ConvertJumpToRET(p, hp1);
  12096. result:=true;
  12097. end;
  12098. end
  12099. else if (cs_opt_level3 in current_settings.optimizerswitches) and
  12100. not (cs_opt_size in current_settings.optimizerswitches) and
  12101. CheckJumpMovTransferOpt(p, hp1, 0, Count) then
  12102. begin
  12103. Result := True;
  12104. Exit;
  12105. end;
  12106. end;
  12107. end;
  12108. end;
  12109. class function TX86AsmOptimizer.CanBeCMOV(p, cond_p: tai; var RefModified: Boolean) : boolean;
  12110. begin
  12111. Result := assigned(p) and
  12112. MatchInstruction(p,A_MOV,[S_W,S_L,S_Q]) and
  12113. (taicpu(p).oper[1]^.typ = top_reg) and
  12114. (
  12115. (taicpu(p).oper[0]^.typ = top_reg) or
  12116. { allow references, but only pure symbols or got rel. addressing with RIP as based,
  12117. it is not expected that this can cause a seg. violation }
  12118. (
  12119. (taicpu(p).oper[0]^.typ = top_ref) and
  12120. { TODO: Can we detect which references become constants at this
  12121. stage so we don't have to do a blanket ban? }
  12122. (taicpu(p).oper[0]^.ref^.refaddr <> addr_full) and
  12123. (
  12124. IsRefSafe(taicpu(p).oper[0]^.ref) or
  12125. (
  12126. { Don't use the reference in the condition if one of its registers got modified by a previous MOV }
  12127. not RefModified and
  12128. { If the reference also appears in the condition, then we know it's safe, otherwise
  12129. any kind of access violation would have occurred already }
  12130. Assigned(cond_p) and
  12131. { Make sure the sizes match too so we're reading and writing the same number of bytes }
  12132. (cond_p.typ = ait_instruction) and
  12133. (taicpu(cond_p).opsize = taicpu(p).opsize) and
  12134. { Just consider 2-operand comparison instructions for now to be safe }
  12135. (taicpu(cond_p).ops = 2) and
  12136. (
  12137. ((taicpu(cond_p).oper[1]^.typ = top_ref) and RefsEqual(taicpu(cond_p).oper[1]^.ref^, taicpu(p).oper[0]^.ref^)) or
  12138. (
  12139. (taicpu(cond_p).oper[0]^.typ = top_ref) and
  12140. { Don't risk identical registers but different offsets, as we may have constructs
  12141. such as buffer streams with things like length fields that indicate whether
  12142. any more data follows. And there are probably some contrived examples where
  12143. writing to offsets behind the one being read also lead to access violations }
  12144. RefsEqual(taicpu(cond_p).oper[0]^.ref^, taicpu(p).oper[0]^.ref^) and
  12145. (
  12146. { Check that we're not modifying a register that appears in the reference }
  12147. (InsProp[taicpu(cond_p).opcode].Ch * [Ch_Mop2, Ch_RWop2, Ch_Wop2] = []) or
  12148. (taicpu(cond_p).oper[1]^.typ <> top_reg) or
  12149. not RegInRef(taicpu(cond_p).oper[1]^.reg, taicpu(cond_p).oper[0]^.ref^)
  12150. )
  12151. )
  12152. )
  12153. )
  12154. )
  12155. )
  12156. );
  12157. end;
  12158. class procedure TX86AsmOptimizer.UpdateIntRegsNoDealloc(var AUsedRegs: TAllUsedRegs; p: Tai);
  12159. begin
  12160. { Update integer registers, ignoring deallocations }
  12161. repeat
  12162. while assigned(p) and
  12163. ((p.typ in (SkipInstr - [ait_RegAlloc])) or
  12164. (p.typ = ait_label) or
  12165. ((p.typ = ait_marker) and
  12166. (tai_Marker(p).Kind in [mark_AsmBlockEnd,mark_NoLineInfoStart,mark_NoLineInfoEnd]))) do
  12167. p := tai(p.next);
  12168. while assigned(p) and
  12169. (p.typ=ait_RegAlloc) Do
  12170. begin
  12171. if (getregtype(tai_regalloc(p).reg) = R_INTREGISTER) then
  12172. begin
  12173. case tai_regalloc(p).ratype of
  12174. ra_alloc :
  12175. IncludeRegInUsedRegs(tai_regalloc(p).reg, AUsedRegs);
  12176. else
  12177. ;
  12178. end;
  12179. end;
  12180. p := tai(p.next);
  12181. end;
  12182. until not(assigned(p)) or
  12183. (not(p.typ in SkipInstr) and
  12184. not((p.typ = ait_label) and
  12185. labelCanBeSkipped(tai_label(p))));
  12186. end;
  12187. {$ifndef 8086}
  12188. function TCMOVTracking.InitialiseBlock(BlockStart, OneBeforeBlock: tai; out BlockStop: tai; out EndJump: tai): Boolean;
  12189. begin
  12190. Result := False;
  12191. EndJump := nil;
  12192. BlockStop := nil;
  12193. while (BlockStart <> fOptimizer.BlockEnd) and
  12194. { stop on labels }
  12195. (BlockStart.typ <> ait_label) do
  12196. begin
  12197. { Keep track of all integer registers that are used }
  12198. fOptimizer.UpdateIntRegsNoDealloc(RegisterTracking, tai(OneBeforeBlock.Next));
  12199. if BlockStart.typ = ait_instruction then
  12200. begin
  12201. if (taicpu(BlockStart).opcode = A_JMP) then
  12202. begin
  12203. if not IsJumpToLabel(taicpu(BlockStart)) or
  12204. (JumpTargetOp(taicpu(BlockStart))^.ref^.index <> NR_NO) then
  12205. Exit;
  12206. EndJump := BlockStart;
  12207. Break;
  12208. end
  12209. { Check to see if we have a valid MOV instruction instead }
  12210. else if (taicpu(BlockStart).opcode <> A_MOV) or
  12211. (taicpu(BlockStart).oper[1]^.typ <> top_reg) or
  12212. not (taicpu(BlockStart).opsize in [S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  12213. begin
  12214. Exit;
  12215. end
  12216. else
  12217. { This will be a valid MOV }
  12218. fAllocationRange := BlockStart;
  12219. end;
  12220. OneBeforeBlock := BlockStart;
  12221. fOptimizer.GetNextInstruction(BlockStart, BlockStart);
  12222. end;
  12223. if (BlockStart = fOptimizer.BlockEnd) then
  12224. Exit;
  12225. BlockStop := BlockStart;
  12226. Result := True;
  12227. end;
  12228. function TCMOVTracking.AnalyseMOVBlock(BlockStart, BlockStop, SearchStart: tai): LongInt;
  12229. var
  12230. hp1: tai;
  12231. RefModified: Boolean;
  12232. begin
  12233. Result := 0;
  12234. hp1 := BlockStart;
  12235. RefModified := False; { As long as the condition is inverted, this can be reset }
  12236. while assigned(hp1) and
  12237. (hp1 <> BlockStop) do
  12238. begin
  12239. case hp1.typ of
  12240. ait_instruction:
  12241. if MatchInstruction(hp1, A_MOV, [S_W, S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  12242. begin
  12243. if fOptimizer.CanBeCMOV(hp1, fCondition, RefModified) then
  12244. begin
  12245. Inc(Result);
  12246. if { Make sure the sizes match too so we're reading and writing the same number of bytes }
  12247. Assigned(fCondition) and
  12248. { Will have 2 operands }
  12249. (
  12250. (
  12251. (taicpu(fCondition).oper[0]^.typ = top_ref) and
  12252. fOptimizer.RegInRef(taicpu(hp1).oper[1]^.reg, taicpu(fCondition).oper[0]^.ref^)
  12253. ) or
  12254. (
  12255. (taicpu(fCondition).oper[1]^.typ = top_ref) and
  12256. fOptimizer.RegInRef(taicpu(hp1).oper[1]^.reg, taicpu(fCondition).oper[1]^.ref^)
  12257. )
  12258. ) then
  12259. { It is no longer safe to use the reference in the condition.
  12260. this prevents problems such as:
  12261. mov (%reg),%reg
  12262. mov (%reg),...
  12263. When the comparison is cmp (%reg),0 and guarding against a null pointer deallocation
  12264. (fixes #40165)
  12265. Note: "mov (%reg1),%reg2; mov (%reg2),..." won't be optimised this way since
  12266. at least one of (%reg1) and (%reg2) won't be in the condition and is hence unsafe.
  12267. }
  12268. RefModified := True;
  12269. end
  12270. else if not (cs_opt_size in current_settings.optimizerswitches) and
  12271. { CMOV with constants grows the code size }
  12272. TryCMOVConst(hp1, SearchStart, BlockStop, Result) then
  12273. begin
  12274. { Register was reserved by TryCMOVConst and
  12275. stored on ConstRegs }
  12276. end
  12277. else
  12278. begin
  12279. Result := -1;
  12280. Exit;
  12281. end;
  12282. end
  12283. else
  12284. begin
  12285. Result := -1;
  12286. Exit;
  12287. end;
  12288. else
  12289. { Most likely an align };
  12290. end;
  12291. fOptimizer.GetNextInstruction(hp1, hp1);
  12292. end;
  12293. end;
  12294. constructor TCMOVTracking.Init(Optimizer: TX86AsmOptimizer; var p_initialjump, p_initialmov: tai; var AFirstLabel: TAsmLabel);
  12295. { For the tsBranching type, increase the weighting score to account for the new conditional jump
  12296. (this is done as a separate stage because the double types are extensions of the branching type,
  12297. but we can't discount the conditional jump until the last step) }
  12298. procedure EvaluateBranchingType;
  12299. begin
  12300. Inc(CMOVScore);
  12301. if (CMOVScore > MAX_CMOV_INSTRUCTIONS) then
  12302. { Too many instructions to be worthwhile }
  12303. fState := tsInvalid;
  12304. end;
  12305. var
  12306. hp1: tai;
  12307. Count: Integer;
  12308. begin
  12309. { Table of valid CMOV block types
  12310. Block type 2nd Jump Mid-label 2nd MOVs 3rd Jump End-label
  12311. ---------- --------- --------- --------- --------- ---------
  12312. tsSimple X Yes X X X
  12313. tsDetour = 1st X X X X
  12314. tsBranching <> Mid Yes X X X
  12315. tsDouble End-label Yes * Yes X Yes
  12316. tsDoubleBranchSame <> Mid Yes * Yes = 2nd X
  12317. tsDoubleBranchDifferent <> Mid Yes * Yes <> 2nd X
  12318. tsDoubleSecondBranching End-label Yes * Yes <> 2nd Yes
  12319. * Only one reference allowed
  12320. }
  12321. hp1 := nil; { To prevent compiler warnings }
  12322. Optimizer.CopyUsedRegs(RegisterTracking);
  12323. fOptimizer := Optimizer;
  12324. fLabel := AFirstLabel;
  12325. CMOVScore := 0;
  12326. ConstCount := 0;
  12327. { Initialise RegWrites, ConstRegs, ConstVals, ConstSizes, ConstWriteSizes and ConstMovs }
  12328. FillChar(RegWrites[0], MAX_CMOV_INSTRUCTIONS * 2 * SizeOf(TRegister), 0);
  12329. FillChar(ConstRegs[0], MAX_CMOV_REGISTERS * SizeOf(TRegister), 0);
  12330. FillChar(ConstVals[0], MAX_CMOV_REGISTERS * SizeOf(TCGInt), 0);
  12331. FillChar(ConstSizes[0], MAX_CMOV_REGISTERS * SizeOf(TSubRegister), 0);
  12332. FillChar(ConstWriteSizes[0], first_int_imreg * SizeOf(TOpSize), 0);
  12333. FillChar(ConstMovs[0], MAX_CMOV_REGISTERS * SizeOf(taicpu), 0);
  12334. fInsertionPoint := p_initialjump;
  12335. fCondition := nil;
  12336. fInitialJump := p_initialjump;
  12337. fFirstMovBlock := p_initialmov;
  12338. fFirstMovBlockStop := nil;
  12339. fSecondJump := nil;
  12340. fSecondMovBlock := nil;
  12341. fSecondMovBlockStop := nil;
  12342. fMidLabel := nil;
  12343. fSecondJump := nil;
  12344. fSecondMovBlock := nil;
  12345. fEndLabel := nil;
  12346. fAllocationRange := nil;
  12347. { Assume it all goes horribly wrong! }
  12348. fState := tsInvalid;
  12349. { Look backwards at the comparisons to get an accurate picture of register usage and a better position for any MOV const,reg insertions }
  12350. if Optimizer.GetLastInstruction(p_initialjump, fCondition) and
  12351. MatchInstruction(fCondition, [A_CMP, A_TEST, A_BSR, A_BSF, A_COMISS, A_COMISD, A_UCOMISS, A_UCOMISD, A_VCOMISS, A_VCOMISD, A_VUCOMISS, A_VUCOMISD], []) then
  12352. begin
  12353. { Mark all the registers in the comparison as 'in use', even if they've just been deallocated }
  12354. for Count := 0 to 1 do
  12355. with taicpu(fCondition).oper[Count]^ do
  12356. case typ of
  12357. top_reg:
  12358. if getregtype(reg) = R_INTREGISTER then
  12359. Optimizer.IncludeRegInUsedRegs(reg, RegisterTracking);
  12360. top_ref:
  12361. begin
  12362. if
  12363. {$ifdef x86_64}
  12364. (ref^.base <> NR_RIP) and
  12365. {$endif x86_64}
  12366. (ref^.base <> NR_NO) then
  12367. Optimizer.IncludeRegInUsedRegs(ref^.base, RegisterTracking);
  12368. if (ref^.index <> NR_NO) then
  12369. Optimizer.IncludeRegInUsedRegs(ref^.index, RegisterTracking);
  12370. end
  12371. else
  12372. ;
  12373. end;
  12374. { When inserting instructions before hp_prev, try to insert them
  12375. before the allocation of the FLAGS register }
  12376. if not SetAndTest(Optimizer.FindRegAllocBackward(NR_DEFAULTFLAGS, tai(fCondition.Previous)), fInsertionPoint) or
  12377. (tai_regalloc(fInsertionPoint).ratype = ra_dealloc) then
  12378. { If not found, set it equal to the condition so it's something sensible }
  12379. fInsertionPoint := fCondition;
  12380. { When dealing with a comparison against zero, take note of the
  12381. instruction before it to see if we can move instructions further
  12382. back in order to benefit PostPeepholeOptTestOr.
  12383. }
  12384. if (
  12385. (
  12386. (taicpu(fCondition).opcode = A_CMP) and
  12387. MatchOperand(taicpu(fCondition).oper[0]^, 0)
  12388. ) or
  12389. (
  12390. (taicpu(fCondition).opcode = A_TEST) and
  12391. (
  12392. Optimizer.OpsEqual(taicpu(fCondition).oper[0]^, taicpu(fCondition).oper[1]^) or
  12393. MatchOperand(taicpu(fCondition).oper[0]^, -1)
  12394. )
  12395. )
  12396. ) and
  12397. Optimizer.GetLastInstruction(fCondition, hp1) then
  12398. begin
  12399. { These instructions set the zero flag if the result is zero }
  12400. if MatchInstruction(hp1, [A_ADD, A_SUB, A_OR, A_XOR, A_AND, A_POPCNT, A_LZCNT], []) then
  12401. begin
  12402. fInsertionPoint := hp1;
  12403. { Also mark all the registers in this previous instruction
  12404. as 'in use', even if they've just been deallocated }
  12405. for Count := 0 to 1 do
  12406. with taicpu(hp1).oper[Count]^ do
  12407. case typ of
  12408. top_reg:
  12409. if getregtype(reg) = R_INTREGISTER then
  12410. Optimizer.IncludeRegInUsedRegs(reg, RegisterTracking);
  12411. top_ref:
  12412. begin
  12413. if
  12414. {$ifdef x86_64}
  12415. (ref^.base <> NR_RIP) and
  12416. {$endif x86_64}
  12417. (ref^.base <> NR_NO) then
  12418. Optimizer.IncludeRegInUsedRegs(ref^.base, RegisterTracking);
  12419. if (ref^.index <> NR_NO) then
  12420. Optimizer.IncludeRegInUsedRegs(ref^.index, RegisterTracking);
  12421. end
  12422. else
  12423. ;
  12424. end;
  12425. end;
  12426. end;
  12427. end
  12428. else
  12429. fCondition := nil;
  12430. { When inserting instructions, try to insert them before the allocation of the FLAGS register }
  12431. if SetAndTest(Optimizer.FindRegAllocBackward(NR_DEFAULTFLAGS, tai(p_initialjump.Previous)), hp1) and
  12432. (tai_regalloc(hp1).ratype <> ra_dealloc) then
  12433. { If not found, set it equal to p so it's something sensible }
  12434. fInsertionPoint := hp1;
  12435. hp1 := p_initialmov;
  12436. if not InitialiseBlock(p_initialmov, p_initialjump, fFirstMovBlockStop, fSecondJump) then
  12437. Exit;
  12438. hp1 := fFirstMovBlockStop; { Will either be on a label or a jump }
  12439. if (hp1.typ <> ait_label) then { should be on a jump }
  12440. begin
  12441. if not Optimizer.GetNextInstruction(hp1, fMidLabel) or not (fMidLabel.typ = ait_label) then
  12442. { Need a label afterwards }
  12443. Exit;
  12444. end
  12445. else
  12446. fMidLabel := hp1;
  12447. if tai_label(fMidLabel).labsym <> AFirstLabel then
  12448. { Not the correct label }
  12449. fMidLabel := nil;
  12450. if not Assigned(fSecondJump) and not Assigned(fMidLabel) then
  12451. { If there's neither a 2nd jump nor correct label, then it's invalid
  12452. (see above table) }
  12453. Exit;
  12454. { Analyse the first block of MOVs more closely }
  12455. CMOVScore := AnalyseMOVBlock(fFirstMovBlock, fFirstMovBlockStop, fInsertionPoint);
  12456. if Assigned(fSecondJump) then
  12457. begin
  12458. if (JumpTargetOp(taicpu(fSecondJump))^.ref^.symbol = AFirstLabel) then
  12459. begin
  12460. fState := tsDetour
  12461. end
  12462. else
  12463. begin
  12464. { Need the correct mid-label for this one }
  12465. if not Assigned(fMidLabel) then
  12466. Exit;
  12467. fState := tsBranching;
  12468. end;
  12469. end
  12470. else
  12471. { No jump. but mid-label is present }
  12472. fState := tsSimple;
  12473. if (CMOVScore > MAX_CMOV_INSTRUCTIONS) or (CMOVScore <= 0) then
  12474. begin
  12475. { Invalid or too many instructions to be worthwhile }
  12476. fState := tsInvalid;
  12477. Exit;
  12478. end;
  12479. { check further for
  12480. jCC xxx
  12481. <several movs 1>
  12482. jmp yyy
  12483. xxx:
  12484. <several movs 2>
  12485. yyy:
  12486. etc.
  12487. }
  12488. if (fState = tsBranching) and
  12489. { Estimate for required savings for extra jump }
  12490. (CMOVScore <= MAX_CMOV_INSTRUCTIONS - 1) and
  12491. { Only one reference is allowed for double blocks }
  12492. (AFirstLabel.getrefs = 1) then
  12493. begin
  12494. Optimizer.GetNextInstruction(fMidLabel, hp1);
  12495. fSecondMovBlock := hp1;
  12496. if not InitialiseBlock(fSecondMovBlock, fMidLabel, fSecondMovBlockStop, fThirdJump) then
  12497. begin
  12498. EvaluateBranchingType;
  12499. Exit;
  12500. end;
  12501. hp1 := fSecondMovBlockStop; { Will either be on a label or a jump }
  12502. if (hp1.typ <> ait_label) then { should be on a jump }
  12503. begin
  12504. if not Optimizer.GetNextInstruction(hp1, fEndLabel) or not (fEndLabel.typ = ait_label) then
  12505. begin
  12506. { Need a label afterwards }
  12507. EvaluateBranchingType;
  12508. Exit;
  12509. end;
  12510. end
  12511. else
  12512. fEndLabel := hp1;
  12513. if tai_label(fEndLabel).labsym <> JumpTargetOp(taicpu(fSecondJump))^.ref^.symbol then
  12514. { Second jump doesn't go to the end }
  12515. fEndLabel := nil;
  12516. if not Assigned(fThirdJump) and not Assigned(fEndLabel) then
  12517. begin
  12518. { If there's neither a 3rd jump nor correct end label, then it's
  12519. not a invalid double block, but is a valid single branching
  12520. block (see above table) }
  12521. EvaluateBranchingType;
  12522. Exit;
  12523. end;
  12524. Count := AnalyseMOVBlock(fSecondMovBlock, fSecondMovBlockStop, fMidLabel);
  12525. if (Count > MAX_CMOV_INSTRUCTIONS) or (Count <= 0) then
  12526. { Invalid or too many instructions to be worthwhile }
  12527. Exit;
  12528. Inc(CMOVScore, Count);
  12529. if Assigned(fThirdJump) then
  12530. begin
  12531. if not Assigned(fSecondJump) then
  12532. fState := tsDoubleSecondBranching
  12533. else if (JumpTargetOp(taicpu(fSecondJump))^.ref^.symbol = JumpTargetOp(taicpu(fThirdJump))^.ref^.symbol) then
  12534. fState := tsDoubleBranchSame
  12535. else
  12536. fState := tsDoubleBranchDifferent;
  12537. end
  12538. else
  12539. fState := tsDouble;
  12540. end;
  12541. if fState = tsBranching then
  12542. EvaluateBranchingType;
  12543. end;
  12544. { Tries to convert a mov const,%reg instruction into a CMOV by reserving a
  12545. new register to store the constant }
  12546. function TCMOVTracking.TryCMOVConst(p, start, stop: tai; var Count: LongInt): Boolean;
  12547. var
  12548. RegSize: TSubRegister;
  12549. CurrentVal: TCGInt;
  12550. ANewReg: TRegister;
  12551. X: ShortInt;
  12552. begin
  12553. Result := False;
  12554. if not MatchOpType(taicpu(p), top_const, top_reg) then
  12555. Exit;
  12556. if ConstCount >= MAX_CMOV_REGISTERS then
  12557. { Arrays are full }
  12558. Exit;
  12559. { Remember that CMOV can't encode 8-bit registers }
  12560. case taicpu(p).opsize of
  12561. S_W:
  12562. RegSize := R_SUBW;
  12563. S_L:
  12564. RegSize := R_SUBD;
  12565. {$ifdef x86_64}
  12566. S_Q:
  12567. RegSize := R_SUBQ;
  12568. {$endif x86_64}
  12569. else
  12570. InternalError(2021100401);
  12571. end;
  12572. { See if the value has already been reserved for another CMOV instruction }
  12573. CurrentVal := taicpu(p).oper[0]^.val;
  12574. for X := 0 to ConstCount - 1 do
  12575. if ConstVals[X] = CurrentVal then
  12576. begin
  12577. ConstRegs[ConstCount] := ConstRegs[X];
  12578. ConstSizes[ConstCount] := RegSize;
  12579. ConstVals[ConstCount] := CurrentVal;
  12580. Inc(ConstCount);
  12581. Inc(Count);
  12582. Result := True;
  12583. Exit;
  12584. end;
  12585. ANewReg := fOptimizer.GetIntRegisterBetween(R_SUBWHOLE, RegisterTracking, start, stop, True);
  12586. if ANewReg = NR_NO then
  12587. { No free registers }
  12588. Exit;
  12589. { Reserve the register so subsequent TryCMOVConst calls don't all end
  12590. up vying for the same register }
  12591. fOptimizer.IncludeRegInUsedRegs(ANewReg, RegisterTracking);
  12592. ConstRegs[ConstCount] := ANewReg;
  12593. ConstSizes[ConstCount] := RegSize;
  12594. ConstVals[ConstCount] := CurrentVal;
  12595. Inc(ConstCount);
  12596. Inc(Count);
  12597. Result := True;
  12598. end;
  12599. destructor TCMOVTracking.Done;
  12600. begin
  12601. TAOptObj.ReleaseUsedRegs(RegisterTracking);
  12602. end;
  12603. procedure TCMOVTracking.Process(out new_p: tai);
  12604. var
  12605. Count, Writes: LongInt;
  12606. RegMatch: Boolean;
  12607. hp1, hp_new: tai;
  12608. inverted_condition, condition: TAsmCond;
  12609. begin
  12610. if (fState in [tsInvalid, tsProcessed]) then
  12611. InternalError(2023110701);
  12612. { Repurpose RegisterTracking to mark registers that we've defined }
  12613. RegisterTracking[R_INTREGISTER].Clear;
  12614. Count := 0;
  12615. Writes := 0;
  12616. condition := taicpu(fInitialJump).condition;
  12617. inverted_condition := inverse_cond(condition);
  12618. { Exclude tsDoubleBranchDifferent from this check, as the second block
  12619. doesn't get CMOVs in this case }
  12620. if (fState in [tsDouble, tsDoubleBranchSame, tsDoubleSecondBranching]) then
  12621. begin
  12622. { Include the jump in the flag tracking }
  12623. if Assigned(fThirdJump) then
  12624. begin
  12625. if (fState = tsDoubleBranchSame) then
  12626. begin
  12627. { Will be an unconditional jump, so track to the instruction before it }
  12628. if not fOptimizer.GetLastInstruction(fThirdJump, hp1) then
  12629. InternalError(2023110710);
  12630. end
  12631. else
  12632. hp1 := fThirdJump;
  12633. end
  12634. else
  12635. hp1 := fSecondMovBlockStop;
  12636. end
  12637. else
  12638. begin
  12639. { Include a conditional jump in the flag tracking }
  12640. if Assigned(fSecondJump) then
  12641. begin
  12642. if (fState = tsDetour) then
  12643. begin
  12644. { Will be an unconditional jump, so track to the instruction before it }
  12645. if not fOptimizer.GetLastInstruction(fSecondJump, hp1) then
  12646. InternalError(2023110711);
  12647. end
  12648. else
  12649. hp1 := fSecondJump;
  12650. end
  12651. else
  12652. hp1 := fFirstMovBlockStop;
  12653. end;
  12654. fOptimizer.AllocRegBetween(NR_DEFAULTFLAGS, fInitialJump, hp1, fOptimizer.UsedRegs);
  12655. { Process the second set of MOVs first, because if a destination
  12656. register is shared between the first and second MOV sets, it is more
  12657. efficient to turn the first one into a MOV instruction and place it
  12658. before the CMP if possible, but we won't know which registers are
  12659. shared until we've processed at least one list, so we might as well
  12660. make it the second one since that won't be modified again. }
  12661. if (fState in [tsDouble, tsDoubleBranchSame, tsDoubleBranchDifferent, tsDoubleSecondBranching]) then
  12662. begin
  12663. hp1 := fSecondMovBlock;
  12664. repeat
  12665. if not Assigned(hp1) then
  12666. InternalError(2018062902);
  12667. if (hp1.typ = ait_instruction) then
  12668. begin
  12669. { Extra safeguard }
  12670. if (taicpu(hp1).opcode <> A_MOV) then
  12671. InternalError(2018062903);
  12672. { Note: tsDoubleBranchDifferent is essentially identical to
  12673. tsBranching and the 2nd block is best left largely
  12674. untouched, but we need to evaluate which registers the MOVs
  12675. write to in order to track what would be complementary CMOV
  12676. pairs that can be further optimised. [Kit] }
  12677. if fState <> tsDoubleBranchDifferent then
  12678. begin
  12679. if taicpu(hp1).oper[0]^.typ = top_const then
  12680. begin
  12681. RegMatch := False;
  12682. for Count := 0 to ConstCount - 1 do
  12683. if (ConstVals[Count] = taicpu(hp1).oper[0]^.val) and
  12684. (getsubreg(taicpu(hp1).oper[1]^.reg) = ConstSizes[Count]) then
  12685. begin
  12686. RegMatch := True;
  12687. { If it's in RegisterTracking, then this register
  12688. is being used more than once and hence has
  12689. already had its value defined (it gets added to
  12690. UsedRegs through AllocRegBetween below) }
  12691. if not RegisterTracking[R_INTREGISTER].IsUsed(ConstRegs[Count]) then
  12692. begin
  12693. hp_new := taicpu.op_const_reg(A_MOV, subreg2opsize(R_SUBWHOLE), taicpu(hp1).oper[0]^.val, ConstRegs[Count]);
  12694. taicpu(hp_new).fileinfo := taicpu(fInitialJump).fileinfo;
  12695. fOptimizer.asml.InsertBefore(hp_new, fInsertionPoint);
  12696. fOptimizer.IncludeRegInUsedRegs(ConstRegs[Count], RegisterTracking);
  12697. ConstMovs[Count] := hp_new;
  12698. end
  12699. else
  12700. { We just need an instruction between hp_prev and hp1
  12701. where we know the register is marked as in use }
  12702. hp_new := fSecondMovBlock;
  12703. { Keep track of largest write for this register so it can be optimised later }
  12704. if (getsubreg(taicpu(hp1).oper[1]^.reg) > ConstWriteSizes[getsupreg(ConstRegs[Count])]) then
  12705. ConstWriteSizes[getsupreg(ConstRegs[Count])] := getsubreg(taicpu(hp1).oper[1]^.reg);
  12706. fOptimizer.AllocRegBetween(ConstRegs[Count], hp_new, hp1, fOptimizer.UsedRegs);
  12707. taicpu(hp1).loadreg(0, newreg(R_INTREGISTER, getsupreg(ConstRegs[Count]), ConstSizes[Count]));
  12708. Break;
  12709. end;
  12710. if not RegMatch then
  12711. InternalError(2021100411);
  12712. end;
  12713. taicpu(hp1).opcode := A_CMOVcc;
  12714. taicpu(hp1).condition := condition;
  12715. end;
  12716. { Store these writes to search for duplicates later on }
  12717. RegWrites[Writes] := taicpu(hp1).oper[1]^.reg;
  12718. Inc(Writes);
  12719. end;
  12720. fOptimizer.GetNextInstruction(hp1, hp1);
  12721. until (hp1 = fSecondMovBlockStop);
  12722. end;
  12723. { Now do the first set of MOVs }
  12724. hp1 := fFirstMovBlock;
  12725. repeat
  12726. if not Assigned(hp1) then
  12727. InternalError(2018062904);
  12728. if (hp1.typ = ait_instruction) then
  12729. begin
  12730. RegMatch := False;
  12731. { Extra safeguard }
  12732. if (taicpu(hp1).opcode <> A_MOV) then
  12733. InternalError(2018062905);
  12734. { Search through the RegWrites list to see if there are any
  12735. opposing CMOV pairs that write to the same register }
  12736. for Count := 0 to Writes - 1 do
  12737. if (RegWrites[Count] = taicpu(hp1).oper[1]^.reg) then
  12738. begin
  12739. { We have a match. Keep this as a MOV }
  12740. { Move ahead in preparation }
  12741. fOptimizer.GetNextInstruction(hp1, hp1);
  12742. RegMatch := True;
  12743. Break;
  12744. end;
  12745. if RegMatch then
  12746. Continue;
  12747. if taicpu(hp1).oper[0]^.typ = top_const then
  12748. begin
  12749. for Count := 0 to ConstCount - 1 do
  12750. if (ConstVals[Count] = taicpu(hp1).oper[0]^.val) and
  12751. (getsubreg(taicpu(hp1).oper[1]^.reg) = ConstSizes[Count]) then
  12752. begin
  12753. RegMatch := True;
  12754. { If it's in RegisterTracking, then this register is
  12755. being used more than once and hence has already had
  12756. its value defined (it gets added to UsedRegs through
  12757. AllocRegBetween below) }
  12758. if not RegisterTracking[R_INTREGISTER].IsUsed(ConstRegs[Count]) then
  12759. begin
  12760. hp_new := taicpu.op_const_reg(A_MOV, subreg2opsize(R_SUBWHOLE), taicpu(hp1).oper[0]^.val, ConstRegs[Count]);
  12761. taicpu(hp_new).fileinfo := taicpu(fInitialJump).fileinfo;
  12762. fOptimizer.asml.InsertBefore(hp_new, fInsertionPoint);
  12763. fOptimizer.IncludeRegInUsedRegs(ConstRegs[Count], RegisterTracking);
  12764. ConstMovs[Count] := hp_new;
  12765. end
  12766. else
  12767. { We just need an instruction between hp_prev and hp1
  12768. where we know the register is marked as in use }
  12769. hp_new := fFirstMovBlock;
  12770. { Keep track of largest write for this register so it can be optimised later }
  12771. if (getsubreg(taicpu(hp1).oper[1]^.reg) > ConstWriteSizes[getsupreg(ConstRegs[Count])]) then
  12772. ConstWriteSizes[getsupreg(ConstRegs[Count])] := getsubreg(taicpu(hp1).oper[1]^.reg);
  12773. fOptimizer.AllocRegBetween(ConstRegs[Count], hp_new, hp1, fOptimizer.UsedRegs);
  12774. taicpu(hp1).loadreg(0, newreg(R_INTREGISTER, getsupreg(ConstRegs[Count]), ConstSizes[Count]));
  12775. Break;
  12776. end;
  12777. if not RegMatch then
  12778. InternalError(2021100412);
  12779. end;
  12780. taicpu(hp1).opcode := A_CMOVcc;
  12781. taicpu(hp1).condition := inverted_condition;
  12782. if (fState = tsDoubleBranchDifferent) then
  12783. begin
  12784. { Store these writes to search for duplicates later on }
  12785. RegWrites[Writes] := taicpu(hp1).oper[1]^.reg;
  12786. Inc(Writes);
  12787. end;
  12788. end;
  12789. fOptimizer.GetNextInstruction(hp1, hp1);
  12790. until (hp1 = fFirstMovBlockStop);
  12791. { Update initialisation MOVs to the smallest possible size }
  12792. for Count := 0 to ConstCount - 1 do
  12793. if Assigned(ConstMovs[Count]) then
  12794. begin
  12795. taicpu(ConstMovs[Count]).opsize := subreg2opsize(ConstWriteSizes[Word(ConstRegs[Count])]);
  12796. setsubreg(taicpu(ConstMovs[Count]).oper[1]^.reg, ConstWriteSizes[Word(ConstRegs[Count])]);
  12797. end;
  12798. case fState of
  12799. tsSimple:
  12800. begin
  12801. fOptimizer.DebugMsg(SPeepholeOptimization + 'CMOV Block (Simple type)', fInitialJump);
  12802. { No branch to delete }
  12803. end;
  12804. tsDetour:
  12805. begin
  12806. fOptimizer.DebugMsg(SPeepholeOptimization + 'CMOV Block (Detour type)', fInitialJump);
  12807. { Preserve jump }
  12808. end;
  12809. tsBranching, tsDoubleBranchDifferent:
  12810. begin
  12811. if (fState = tsBranching) then
  12812. fOptimizer.DebugMsg(SPeepholeOptimization + 'CMOV Block (Branching type)', fInitialJump)
  12813. else
  12814. fOptimizer.DebugMsg(SPeepholeOptimization + 'CMOV Block (Double branching (different) type)', fInitialJump);
  12815. taicpu(fSecondJump).opcode := A_JCC;
  12816. taicpu(fSecondJump).condition := inverted_condition;
  12817. end;
  12818. tsDouble, tsDoubleBranchSame:
  12819. begin
  12820. if (fState = tsDouble) then
  12821. fOptimizer.DebugMsg(SPeepholeOptimization + 'CMOV Block (Double type)', fInitialJump)
  12822. else
  12823. fOptimizer.DebugMsg(SPeepholeOptimization + 'CMOV Block (Double branching (same) type)', fInitialJump);
  12824. { Delete second jump }
  12825. JumpTargetOp(taicpu(fSecondJump))^.ref^.symbol.decrefs;
  12826. fOptimizer.RemoveInstruction(fSecondJump);
  12827. end;
  12828. tsDoubleSecondBranching:
  12829. begin
  12830. fOptimizer.DebugMsg(SPeepholeOptimization + 'CMOV Block (Double, second branching type)', fInitialJump);
  12831. { Delete second jump, preserve third jump as conditional }
  12832. JumpTargetOp(taicpu(fSecondJump))^.ref^.symbol.decrefs;
  12833. fOptimizer.RemoveInstruction(fSecondJump);
  12834. taicpu(fThirdJump).opcode := A_JCC;
  12835. taicpu(fThirdJump).condition := condition;
  12836. end;
  12837. else
  12838. InternalError(2023110720);
  12839. end;
  12840. { Now we can safely decrement the reference count }
  12841. tasmlabel(fLabel).decrefs;
  12842. fOptimizer.UpdateUsedRegs(tai(fInitialJump.next));
  12843. { Remove the original jump }
  12844. fOptimizer.RemoveInstruction(fInitialJump); { Note, the choice to not use RemoveCurrentp is deliberate }
  12845. new_p := fFirstMovBlock; { Appears immediately after the initial jump }
  12846. fState := tsProcessed;
  12847. end;
  12848. {$endif 8086}
  12849. function TX86AsmOptimizer.OptPass2Jcc(var p : tai) : boolean;
  12850. var
  12851. hp1,hp2: tai;
  12852. carryadd_opcode : TAsmOp;
  12853. symbol: TAsmSymbol;
  12854. increg, tmpreg: TRegister;
  12855. {$ifndef i8086}
  12856. CMOVTracking: PCMOVTracking;
  12857. hp3,hp4,hp5: tai;
  12858. {$endif i8086}
  12859. TempBool: Boolean;
  12860. begin
  12861. if (aoc_DoPass2JccOpts in OptsToCheck) and
  12862. DoJumpOptimizations(p, TempBool) then
  12863. Exit(True);
  12864. result:=false;
  12865. if GetNextInstruction(p,hp1) then
  12866. begin
  12867. if (hp1.typ=ait_label) then
  12868. begin
  12869. Result := DoSETccLblRETOpt(p, tai_label(hp1));
  12870. Exit;
  12871. end
  12872. else if (hp1.typ<>ait_instruction) then
  12873. Exit;
  12874. symbol := TAsmLabel(taicpu(p).oper[0]^.ref^.symbol);
  12875. if (
  12876. (
  12877. ((Taicpu(hp1).opcode=A_ADD) or (Taicpu(hp1).opcode=A_SUB)) and
  12878. MatchOptype(Taicpu(hp1),top_const,top_reg) and
  12879. (Taicpu(hp1).oper[0]^.val=1)
  12880. ) or
  12881. ((Taicpu(hp1).opcode=A_INC) or (Taicpu(hp1).opcode=A_DEC))
  12882. ) and
  12883. GetNextInstruction(hp1,hp2) and
  12884. FindLabel(TAsmLabel(symbol), hp2) then
  12885. { jb @@1 cmc
  12886. inc/dec operand --> adc/sbb operand,0
  12887. @@1:
  12888. ... and ...
  12889. jnb @@1
  12890. inc/dec operand --> adc/sbb operand,0
  12891. @@1: }
  12892. begin
  12893. if Taicpu(p).condition in [C_NAE,C_B,C_C] then
  12894. begin
  12895. case taicpu(hp1).opcode of
  12896. A_INC,
  12897. A_ADD:
  12898. carryadd_opcode:=A_ADC;
  12899. A_DEC,
  12900. A_SUB:
  12901. carryadd_opcode:=A_SBB;
  12902. else
  12903. InternalError(2021011001);
  12904. end;
  12905. Taicpu(p).clearop(0);
  12906. Taicpu(p).ops:=0;
  12907. Taicpu(p).is_jmp:=false;
  12908. Taicpu(p).opcode:=A_CMC;
  12909. Taicpu(p).condition:=C_NONE;
  12910. DebugMsg(SPeepholeOptimization+'JccAdd/Inc/Dec2CmcAdc/Sbb',p);
  12911. Taicpu(hp1).ops:=2;
  12912. if (Taicpu(hp1).opcode=A_ADD) or (Taicpu(hp1).opcode=A_SUB) then
  12913. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[1]^)
  12914. else
  12915. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[0]^);
  12916. Taicpu(hp1).loadconst(0,0);
  12917. Taicpu(hp1).opcode:=carryadd_opcode;
  12918. result:=true;
  12919. exit;
  12920. end
  12921. else if Taicpu(p).condition in [C_AE,C_NB,C_NC] then
  12922. begin
  12923. case taicpu(hp1).opcode of
  12924. A_INC,
  12925. A_ADD:
  12926. carryadd_opcode:=A_ADC;
  12927. A_DEC,
  12928. A_SUB:
  12929. carryadd_opcode:=A_SBB;
  12930. else
  12931. InternalError(2021011002);
  12932. end;
  12933. Taicpu(hp1).ops:=2;
  12934. DebugMsg(SPeepholeOptimization+'JccAdd/Inc/Dec2Adc/Sbb',p);
  12935. if (Taicpu(hp1).opcode=A_ADD) or (Taicpu(hp1).opcode=A_SUB) then
  12936. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[1]^)
  12937. else
  12938. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[0]^);
  12939. Taicpu(hp1).loadconst(0,0);
  12940. Taicpu(hp1).opcode:=carryadd_opcode;
  12941. RemoveCurrentP(p, hp1);
  12942. result:=true;
  12943. exit;
  12944. end
  12945. {
  12946. jcc @@1 setcc tmpreg
  12947. inc/dec/add/sub operand -> (movzx tmpreg)
  12948. @@1: add/sub tmpreg,operand
  12949. While this increases code size slightly, it makes the code much faster if the
  12950. jump is unpredictable
  12951. }
  12952. else if not(cs_opt_size in current_settings.optimizerswitches) then
  12953. begin
  12954. { search for an available register which is volatile }
  12955. increg := GetIntRegisterBetween(R_SUBL, UsedRegs, p, hp1);
  12956. if increg <> NR_NO then
  12957. begin
  12958. { We don't need to check if tmpreg is in hp1 or not, because
  12959. it will be marked as in use at p (if not, this is
  12960. indictive of a compiler bug). }
  12961. TAsmLabel(symbol).decrefs;
  12962. Taicpu(p).clearop(0);
  12963. Taicpu(p).ops:=1;
  12964. Taicpu(p).is_jmp:=false;
  12965. Taicpu(p).opcode:=A_SETcc;
  12966. DebugMsg(SPeepholeOptimization+'JccAdd2SetccAdd',p);
  12967. Taicpu(p).condition:=inverse_cond(Taicpu(p).condition);
  12968. Taicpu(p).loadreg(0,increg);
  12969. if getsubreg(Taicpu(hp1).oper[1]^.reg)<>R_SUBL then
  12970. begin
  12971. case getsubreg(Taicpu(hp1).oper[1]^.reg) of
  12972. R_SUBW:
  12973. begin
  12974. tmpreg := newreg(R_INTREGISTER,getsupreg(increg),R_SUBW);
  12975. hp2:=Taicpu.op_reg_reg(A_MOVZX,S_BW,increg,tmpreg);
  12976. end;
  12977. R_SUBD:
  12978. begin
  12979. tmpreg := newreg(R_INTREGISTER,getsupreg(increg),R_SUBD);
  12980. hp2:=Taicpu.op_reg_reg(A_MOVZX,S_BL,increg,tmpreg);
  12981. end;
  12982. {$ifdef x86_64}
  12983. R_SUBQ:
  12984. begin
  12985. { MOVZX doesn't have a 64-bit variant, because
  12986. the 32-bit version implicitly zeroes the
  12987. upper 32-bits of the destination register }
  12988. tmpreg := newreg(R_INTREGISTER,getsupreg(increg),R_SUBD);
  12989. hp2:=Taicpu.op_reg_reg(A_MOVZX,S_BL,increg,tmpreg);
  12990. setsubreg(tmpreg, R_SUBQ);
  12991. end;
  12992. {$endif x86_64}
  12993. else
  12994. Internalerror(2020030601);
  12995. end;
  12996. taicpu(hp2).fileinfo:=taicpu(hp1).fileinfo;
  12997. asml.InsertAfter(hp2,p);
  12998. end
  12999. else
  13000. tmpreg := increg;
  13001. if (Taicpu(hp1).opcode=A_INC) or (Taicpu(hp1).opcode=A_DEC) then
  13002. begin
  13003. Taicpu(hp1).ops:=2;
  13004. Taicpu(hp1).loadoper(1,Taicpu(hp1).oper[0]^)
  13005. end;
  13006. Taicpu(hp1).loadreg(0,tmpreg);
  13007. AllocRegBetween(tmpreg,p,hp1,UsedRegs);
  13008. Result := True;
  13009. { p is no longer a Jcc instruction, so exit }
  13010. Exit;
  13011. end;
  13012. end;
  13013. end;
  13014. { Detect the following:
  13015. jmp<cond> @Lbl1
  13016. jmp @Lbl2
  13017. ...
  13018. @Lbl1:
  13019. ret
  13020. Change to:
  13021. jmp<inv_cond> @Lbl2
  13022. ret
  13023. }
  13024. if MatchInstruction(hp1,A_JMP,[]) and (taicpu(hp1).oper[0]^.ref^.refaddr=addr_full) then
  13025. begin
  13026. hp2:=getlabelwithsym(TAsmLabel(symbol));
  13027. if Assigned(hp2) and SkipLabels(hp2,hp2) and
  13028. MatchInstruction(hp2,A_RET,[S_NO]) then
  13029. begin
  13030. taicpu(p).condition := inverse_cond(taicpu(p).condition);
  13031. { Change label address to that of the unconditional jump }
  13032. taicpu(p).loadoper(0, taicpu(hp1).oper[0]^);
  13033. TAsmLabel(symbol).DecRefs;
  13034. taicpu(hp1).opcode := A_RET;
  13035. taicpu(hp1).is_jmp := false;
  13036. taicpu(hp1).ops := taicpu(hp2).ops;
  13037. DebugMsg(SPeepholeOptimization+'JccJmpRet2J!ccRet',p);
  13038. case taicpu(hp2).ops of
  13039. 0:
  13040. taicpu(hp1).clearop(0);
  13041. 1:
  13042. taicpu(hp1).loadconst(0,taicpu(hp2).oper[0]^.val);
  13043. else
  13044. internalerror(2016041302);
  13045. end;
  13046. end;
  13047. {$ifndef i8086}
  13048. end
  13049. {
  13050. convert
  13051. j<c> .L1
  13052. mov 1,reg
  13053. jmp .L2
  13054. .L1
  13055. mov 0,reg
  13056. .L2
  13057. into
  13058. mov 0,reg
  13059. set<not(c)> reg
  13060. take care of alignment and that the mov 0,reg is not converted into a xor as this
  13061. would destroy the flag contents
  13062. }
  13063. else if MatchInstruction(hp1,A_MOV,[]) and
  13064. MatchOpType(taicpu(hp1),top_const,top_reg) and
  13065. {$ifdef i386}
  13066. (
  13067. { Under i386, ESI, EDI, EBP and ESP
  13068. don't have an 8-bit representation }
  13069. not (getsupreg(taicpu(hp1).oper[1]^.reg) in [RS_ESI, RS_EDI, RS_EBP, RS_ESP])
  13070. ) and
  13071. {$endif i386}
  13072. (taicpu(hp1).oper[0]^.val=1) and
  13073. GetNextInstruction(hp1,hp2) and
  13074. MatchInstruction(hp2,A_JMP,[]) and (taicpu(hp2).oper[0]^.ref^.refaddr=addr_full) and
  13075. GetNextInstruction(hp2,hp3) and
  13076. (hp3.typ=ait_label) and
  13077. (tasmlabel(taicpu(p).oper[0]^.ref^.symbol)=tai_label(hp3).labsym) and
  13078. (tai_label(hp3).labsym.getrefs=1) and
  13079. GetNextInstruction(hp3,hp4) and
  13080. MatchInstruction(hp4,A_MOV,[]) and
  13081. MatchOpType(taicpu(hp4),top_const,top_reg) and
  13082. (taicpu(hp4).oper[0]^.val=0) and
  13083. MatchOperand(taicpu(hp1).oper[1]^,taicpu(hp4).oper[1]^) and
  13084. GetNextInstruction(hp4,hp5) and
  13085. (hp5.typ=ait_label) and
  13086. (tasmlabel(taicpu(hp2).oper[0]^.ref^.symbol)=tai_label(hp5).labsym) and
  13087. (tai_label(hp5).labsym.getrefs=1) then
  13088. begin
  13089. AllocRegBetween(NR_FLAGS,p,hp4,UsedRegs);
  13090. DebugMsg(SPeepholeOptimization+'JccMovJmpMov2MovSetcc',p);
  13091. { remove last label }
  13092. RemoveInstruction(hp5);
  13093. { remove second label }
  13094. RemoveInstruction(hp3);
  13095. { remove jmp }
  13096. RemoveInstruction(hp2);
  13097. if taicpu(hp1).opsize=S_B then
  13098. RemoveInstruction(hp1)
  13099. else
  13100. taicpu(hp1).loadconst(0,0);
  13101. taicpu(hp4).opcode:=A_SETcc;
  13102. taicpu(hp4).opsize:=S_B;
  13103. taicpu(hp4).condition:=inverse_cond(taicpu(p).condition);
  13104. taicpu(hp4).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(hp4).oper[1]^.reg),R_SUBL));
  13105. taicpu(hp4).opercnt:=1;
  13106. taicpu(hp4).ops:=1;
  13107. taicpu(hp4).freeop(1);
  13108. RemoveCurrentP(p);
  13109. Result:=true;
  13110. exit;
  13111. end
  13112. else if (CPUX86_HAS_CMOV in cpu_capabilities[current_settings.cputype]) and
  13113. MatchInstruction(hp1,A_MOV,[S_W,S_L{$ifdef x86_64},S_Q{$endif x86_64}]) then
  13114. begin
  13115. { check for
  13116. jCC xxx
  13117. <several movs>
  13118. xxx:
  13119. Also spot:
  13120. Jcc xxx
  13121. <several movs>
  13122. jmp xxx
  13123. Change to:
  13124. <several cmovs with inverted condition>
  13125. jmp xxx (only for the 2nd case)
  13126. }
  13127. CMOVTracking := New(PCMOVTracking, Init(Self, p, hp1, TAsmLabel(symbol)));
  13128. if CMOVTracking^.State <> tsInvalid then
  13129. begin
  13130. CMovTracking^.Process(p);
  13131. Result := True;
  13132. end;
  13133. CMOVTracking^.Done;
  13134. {$endif i8086}
  13135. end;
  13136. end;
  13137. end;
  13138. function TX86AsmOptimizer.OptPass1Movx(var p : tai) : boolean;
  13139. var
  13140. hp1,hp2,hp3: tai;
  13141. reg_and_hp1_is_instr, RegUsed, AndTest: Boolean;
  13142. NewSize: TOpSize;
  13143. NewRegSize: TSubRegister;
  13144. Limit: TCgInt;
  13145. SwapOper: POper;
  13146. begin
  13147. result:=false;
  13148. reg_and_hp1_is_instr:=(taicpu(p).oper[1]^.typ = top_reg) and
  13149. GetNextInstruction(p,hp1) and
  13150. (hp1.typ = ait_instruction);
  13151. if reg_and_hp1_is_instr and
  13152. (
  13153. (taicpu(hp1).opcode <> A_LEA) or
  13154. { If the LEA instruction can be converted into an arithmetic instruction,
  13155. it may be possible to then fold it. }
  13156. (
  13157. { If the flags register is in use, don't change the instruction
  13158. to an ADD otherwise this will scramble the flags. [Kit] }
  13159. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  13160. ConvertLEA(taicpu(hp1))
  13161. )
  13162. ) and
  13163. IsFoldableArithOp(taicpu(hp1),taicpu(p).oper[1]^.reg) and
  13164. GetNextInstruction(hp1,hp2) and
  13165. MatchInstruction(hp2,A_MOV,[]) and
  13166. (taicpu(hp2).oper[0]^.typ = top_reg) and
  13167. OpsEqual(taicpu(hp2).oper[1]^,taicpu(p).oper[0]^) and
  13168. ((taicpu(p).opsize in [S_BW,S_BL]) and (taicpu(hp2).opsize=S_B) or
  13169. (taicpu(p).opsize in [S_WL]) and (taicpu(hp2).opsize=S_W)) and
  13170. {$ifdef i386}
  13171. { not all registers have byte size sub registers on i386 }
  13172. ((taicpu(hp2).opsize<>S_B) or (getsupreg(taicpu(hp1).oper[0]^.reg) in [RS_EAX, RS_EBX, RS_ECX, RS_EDX])) and
  13173. {$endif i386}
  13174. (((taicpu(hp1).ops=2) and
  13175. (getsupreg(taicpu(hp2).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg))) or
  13176. ((taicpu(hp1).ops=1) and
  13177. (getsupreg(taicpu(hp2).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[0]^.reg)))) and
  13178. not(RegUsedAfterInstruction(taicpu(hp2).oper[0]^.reg,hp2,UsedRegs)) then
  13179. begin
  13180. { change movsX/movzX reg/ref, reg2
  13181. add/sub/or/... reg3/$const, reg2
  13182. mov reg2 reg/ref
  13183. to add/sub/or/... reg3/$const, reg/ref }
  13184. { by example:
  13185. movswl %si,%eax movswl %si,%eax p
  13186. decl %eax addl %edx,%eax hp1
  13187. movw %ax,%si movw %ax,%si hp2
  13188. ->
  13189. movswl %si,%eax movswl %si,%eax p
  13190. decw %eax addw %edx,%eax hp1
  13191. movw %ax,%si movw %ax,%si hp2
  13192. }
  13193. taicpu(hp1).changeopsize(taicpu(hp2).opsize);
  13194. {
  13195. ->
  13196. movswl %si,%eax movswl %si,%eax p
  13197. decw %si addw %dx,%si hp1
  13198. movw %ax,%si movw %ax,%si hp2
  13199. }
  13200. case taicpu(hp1).ops of
  13201. 1:
  13202. taicpu(hp1).loadoper(0,taicpu(hp2).oper[1]^);
  13203. 2:
  13204. begin
  13205. taicpu(hp1).loadoper(1,taicpu(hp2).oper[1]^);
  13206. if (taicpu(hp1).oper[0]^.typ = top_reg) then
  13207. setsubreg(taicpu(hp1).oper[0]^.reg,getsubreg(taicpu(hp2).oper[0]^.reg));
  13208. end;
  13209. else
  13210. internalerror(2008042702);
  13211. end;
  13212. {
  13213. ->
  13214. decw %si addw %dx,%si p
  13215. }
  13216. DebugMsg(SPeepholeOptimization + 'var3',p);
  13217. RemoveCurrentP(p, hp1);
  13218. RemoveInstruction(hp2);
  13219. Result := True;
  13220. Exit;
  13221. end;
  13222. if reg_and_hp1_is_instr and
  13223. (taicpu(hp1).opcode = A_MOV) and
  13224. MatchOpType(taicpu(hp1),top_reg,top_reg) and
  13225. (MatchOperand(taicpu(p).oper[1]^,taicpu(hp1).oper[0]^)
  13226. {$ifdef x86_64}
  13227. { check for implicit extension to 64 bit }
  13228. or
  13229. ((taicpu(p).opsize in [S_BL,S_WL]) and
  13230. (taicpu(hp1).opsize=S_Q) and
  13231. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.reg)
  13232. )
  13233. {$endif x86_64}
  13234. )
  13235. then
  13236. begin
  13237. { change
  13238. movx %reg1,%reg2
  13239. mov %reg2,%reg3
  13240. dealloc %reg2
  13241. into
  13242. movx %reg,%reg3
  13243. }
  13244. TransferUsedRegs(TmpUsedRegs);
  13245. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  13246. if not(RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs)) then
  13247. begin
  13248. DebugMsg(SPeepholeOptimization + 'MovxMov2Movx',p);
  13249. {$ifdef x86_64}
  13250. if (taicpu(p).opsize in [S_BL,S_WL]) and
  13251. (taicpu(hp1).opsize=S_Q) then
  13252. taicpu(p).loadreg(1,newreg(R_INTREGISTER,getsupreg(taicpu(hp1).oper[1]^.reg),R_SUBD))
  13253. else
  13254. {$endif x86_64}
  13255. taicpu(p).loadreg(1,taicpu(hp1).oper[1]^.reg);
  13256. RemoveInstruction(hp1);
  13257. Result := True;
  13258. Exit;
  13259. end;
  13260. end;
  13261. if reg_and_hp1_is_instr and
  13262. ((taicpu(hp1).opcode=A_MOV) or
  13263. (taicpu(hp1).opcode=A_ADD) or
  13264. (taicpu(hp1).opcode=A_SUB) or
  13265. (taicpu(hp1).opcode=A_CMP) or
  13266. (taicpu(hp1).opcode=A_OR) or
  13267. (taicpu(hp1).opcode=A_XOR) or
  13268. (taicpu(hp1).opcode=A_AND)
  13269. ) and
  13270. (taicpu(hp1).oper[1]^.typ = top_reg) then
  13271. begin
  13272. AndTest := (taicpu(hp1).opcode=A_AND) and
  13273. GetNextInstruction(hp1, hp2) and
  13274. (hp2.typ = ait_instruction) and
  13275. (
  13276. (
  13277. (taicpu(hp2).opcode=A_TEST) and
  13278. (
  13279. MatchOperand(taicpu(hp2).oper[0]^, taicpu(hp1).oper[1]^.reg) or
  13280. MatchOperand(taicpu(hp2).oper[0]^, -1) or
  13281. (
  13282. { If the AND and TEST instructions share a constant, this is also valid }
  13283. (taicpu(hp1).oper[0]^.typ = top_const) and
  13284. MatchOperand(taicpu(hp2).oper[0]^, taicpu(hp1).oper[0]^.val)
  13285. )
  13286. ) and
  13287. MatchOperand(taicpu(hp2).oper[1]^, taicpu(hp1).oper[1]^.reg)
  13288. ) or
  13289. (
  13290. (taicpu(hp2).opcode=A_CMP) and
  13291. MatchOperand(taicpu(hp2).oper[0]^, 0) and
  13292. MatchOperand(taicpu(hp2).oper[1]^, taicpu(hp1).oper[1]^.reg)
  13293. )
  13294. );
  13295. { change
  13296. movx (oper),%reg2
  13297. and $x,%reg2
  13298. test %reg2,%reg2
  13299. dealloc %reg2
  13300. into
  13301. op %reg1,%reg3
  13302. if the second op accesses only the bits stored in reg1
  13303. }
  13304. if ((taicpu(p).oper[0]^.typ=top_reg) or
  13305. ((taicpu(p).oper[0]^.typ=top_ref) and (taicpu(p).oper[0]^.ref^.refaddr<>addr_full))) and
  13306. (taicpu(hp1).oper[0]^.typ = top_const) and
  13307. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  13308. AndTest then
  13309. begin
  13310. { Check if the AND constant is in range }
  13311. case taicpu(p).opsize of
  13312. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  13313. begin
  13314. NewSize := S_B;
  13315. Limit := $FF;
  13316. end;
  13317. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  13318. begin
  13319. NewSize := S_W;
  13320. Limit := $FFFF;
  13321. end;
  13322. {$ifdef x86_64}
  13323. S_LQ:
  13324. begin
  13325. NewSize := S_L;
  13326. Limit := $FFFFFFFF;
  13327. end;
  13328. {$endif x86_64}
  13329. else
  13330. InternalError(2021120303);
  13331. end;
  13332. if (
  13333. ((taicpu(hp1).oper[0]^.val and Limit) = taicpu(hp1).oper[0]^.val) or
  13334. { Check for negative operands }
  13335. (((not taicpu(hp1).oper[0]^.val) and Limit) = (not taicpu(hp1).oper[0]^.val))
  13336. ) and
  13337. GetNextInstruction(hp2,hp3) and
  13338. MatchInstruction(hp3,A_Jcc,A_Setcc,A_CMOVcc,[]) and
  13339. (taicpu(hp3).condition in [C_E,C_NE]) then
  13340. begin
  13341. TransferUsedRegs(TmpUsedRegs);
  13342. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  13343. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  13344. if not(RegUsedAfterInstruction(taicpu(hp2).oper[1]^.reg, hp2, TmpUsedRegs)) then
  13345. begin
  13346. DebugMsg(SPeepholeOptimization + 'MovxAndTest2Test done',p);
  13347. taicpu(hp1).loadoper(1, taicpu(p).oper[0]^);
  13348. taicpu(hp1).opcode := A_TEST;
  13349. taicpu(hp1).opsize := NewSize;
  13350. RemoveInstruction(hp2);
  13351. RemoveCurrentP(p, hp1);
  13352. Result:=true;
  13353. exit;
  13354. end;
  13355. end;
  13356. end;
  13357. if (taicpu(hp1).oper[0]^.typ = top_reg) and
  13358. (((taicpu(p).opsize in [S_BW,S_BL,S_WL{$ifdef x86_64},S_BQ,S_WQ,S_LQ{$endif x86_64}]) and
  13359. (taicpu(hp1).opsize=S_B)) or
  13360. ((taicpu(p).opsize in [S_WL{$ifdef x86_64},S_WQ,S_LQ{$endif x86_64}]) and
  13361. (taicpu(hp1).opsize=S_W))
  13362. {$ifdef x86_64}
  13363. or ((taicpu(p).opsize=S_LQ) and
  13364. (taicpu(hp1).opsize=S_L))
  13365. {$endif x86_64}
  13366. ) and
  13367. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[0]^.reg) then
  13368. begin
  13369. { change
  13370. movx %reg1,%reg2
  13371. op %reg2,%reg3
  13372. dealloc %reg2
  13373. into
  13374. op %reg1,%reg3
  13375. if the second op accesses only the bits stored in reg1
  13376. }
  13377. TransferUsedRegs(TmpUsedRegs);
  13378. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  13379. if AndTest then
  13380. begin
  13381. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  13382. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs);
  13383. end
  13384. else
  13385. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs);
  13386. if not RegUsed then
  13387. begin
  13388. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 1',p);
  13389. if taicpu(p).oper[0]^.typ=top_reg then
  13390. begin
  13391. case taicpu(hp1).opsize of
  13392. S_B:
  13393. taicpu(hp1).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[0]^.reg),R_SUBL));
  13394. S_W:
  13395. taicpu(hp1).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[0]^.reg),R_SUBW));
  13396. S_L:
  13397. taicpu(hp1).loadreg(0,newreg(R_INTREGISTER,getsupreg(taicpu(p).oper[0]^.reg),R_SUBD));
  13398. else
  13399. Internalerror(2020102301);
  13400. end;
  13401. AllocRegBetween(taicpu(hp1).oper[0]^.reg,p,hp1,UsedRegs);
  13402. end
  13403. else
  13404. taicpu(hp1).loadref(0,taicpu(p).oper[0]^.ref^);
  13405. RemoveCurrentP(p);
  13406. if AndTest then
  13407. RemoveInstruction(hp2);
  13408. result:=true;
  13409. exit;
  13410. end;
  13411. end
  13412. else if (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg) and
  13413. (
  13414. { Bitwise operations only }
  13415. (taicpu(hp1).opcode=A_AND) or
  13416. (taicpu(hp1).opcode=A_TEST) or
  13417. (
  13418. (taicpu(hp1).oper[0]^.typ = top_const) and
  13419. (
  13420. (taicpu(hp1).opcode=A_OR) or
  13421. (taicpu(hp1).opcode=A_XOR)
  13422. )
  13423. )
  13424. ) and
  13425. (
  13426. (taicpu(hp1).oper[0]^.typ = top_const) or
  13427. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) or
  13428. not RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^)
  13429. ) then
  13430. begin
  13431. { change
  13432. movx %reg2,%reg2
  13433. op const,%reg2
  13434. into
  13435. op const,%reg2 (smaller version)
  13436. movx %reg2,%reg2
  13437. also change
  13438. movx %reg1,%reg2
  13439. and/test (oper),%reg2
  13440. dealloc %reg2
  13441. into
  13442. and/test (oper),%reg1
  13443. }
  13444. case taicpu(p).opsize of
  13445. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  13446. begin
  13447. NewSize := S_B;
  13448. NewRegSize := R_SUBL;
  13449. Limit := $FF;
  13450. end;
  13451. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  13452. begin
  13453. NewSize := S_W;
  13454. NewRegSize := R_SUBW;
  13455. Limit := $FFFF;
  13456. end;
  13457. {$ifdef x86_64}
  13458. S_LQ:
  13459. begin
  13460. NewSize := S_L;
  13461. NewRegSize := R_SUBD;
  13462. Limit := $FFFFFFFF;
  13463. end;
  13464. {$endif x86_64}
  13465. else
  13466. Internalerror(2021120302);
  13467. end;
  13468. TransferUsedRegs(TmpUsedRegs);
  13469. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  13470. if AndTest then
  13471. begin
  13472. UpdateUsedRegs(TmpUsedRegs, tai(hp1.next));
  13473. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp2,TmpUsedRegs);
  13474. end
  13475. else
  13476. RegUsed := RegUsedAfterInstruction(taicpu(p).oper[1]^.reg,hp1,TmpUsedRegs);
  13477. if
  13478. (
  13479. (taicpu(p).opcode = A_MOVZX) and
  13480. (
  13481. (taicpu(hp1).opcode=A_AND) or
  13482. (taicpu(hp1).opcode=A_TEST)
  13483. ) and
  13484. not (
  13485. { If both are references, then the final instruction will have
  13486. both operands as references, which is not allowed }
  13487. (taicpu(p).oper[0]^.typ = top_ref) and
  13488. (taicpu(hp1).oper[0]^.typ = top_ref)
  13489. ) and
  13490. not RegUsed
  13491. ) or
  13492. (
  13493. (
  13494. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) or
  13495. not RegUsed
  13496. ) and
  13497. (taicpu(p).oper[0]^.typ = top_reg) and
  13498. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  13499. (taicpu(hp1).oper[0]^.typ = top_const) and
  13500. ((taicpu(hp1).oper[0]^.val and Limit) = taicpu(hp1).oper[0]^.val)
  13501. ) then
  13502. begin
  13503. {$if defined(i386) or defined(i8086)}
  13504. { If the target size is 8-bit, make sure we can actually encode it }
  13505. if (NewRegSize = R_SUBL) and (taicpu(hp1).oper[0]^.typ = top_reg) and not (GetSupReg(taicpu(hp1).oper[0]^.reg) in [RS_EAX,RS_EBX,RS_ECX,RS_EDX]) then
  13506. Exit;
  13507. {$endif i386 or i8086}
  13508. DebugMsg(SPeepholeOptimization + 'MovxOp2Op 2',p);
  13509. taicpu(hp1).opsize := NewSize;
  13510. taicpu(hp1).loadoper(1, taicpu(p).oper[0]^);
  13511. if AndTest then
  13512. begin
  13513. RemoveInstruction(hp2);
  13514. if not RegUsed then
  13515. begin
  13516. taicpu(hp1).opcode := A_TEST;
  13517. if (taicpu(hp1).oper[0]^.typ = top_ref) then
  13518. begin
  13519. { Make sure the reference is the second operand }
  13520. SwapOper := taicpu(hp1).oper[0];
  13521. taicpu(hp1).oper[0] := taicpu(hp1).oper[1];
  13522. taicpu(hp1).oper[1] := SwapOper;
  13523. end;
  13524. end;
  13525. end;
  13526. case taicpu(hp1).oper[0]^.typ of
  13527. top_reg:
  13528. setsubreg(taicpu(hp1).oper[0]^.reg, NewRegSize);
  13529. top_const:
  13530. { For the AND/TEST case }
  13531. taicpu(hp1).oper[0]^.val := taicpu(hp1).oper[0]^.val and Limit;
  13532. else
  13533. ;
  13534. end;
  13535. if RegUsed then
  13536. begin
  13537. AsmL.Remove(p);
  13538. AsmL.InsertAfter(p, hp1);
  13539. p := hp1;
  13540. end
  13541. else
  13542. RemoveCurrentP(p, hp1);
  13543. result:=true;
  13544. exit;
  13545. end;
  13546. end;
  13547. end;
  13548. if reg_and_hp1_is_instr and
  13549. (taicpu(p).oper[0]^.typ = top_reg) and
  13550. (
  13551. (taicpu(hp1).opcode = A_SHL) or (taicpu(hp1).opcode = A_SAL)
  13552. ) and
  13553. (taicpu(hp1).oper[0]^.typ = top_const) and
  13554. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  13555. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  13556. { Minimum shift value allowed is the bit difference between the sizes }
  13557. (taicpu(hp1).oper[0]^.val >=
  13558. { Multiply by 8 because tcgsize2size returns bytes, not bits }
  13559. 8 * (
  13560. tcgsize2size[reg_cgsize(taicpu(p).oper[1]^.reg)] -
  13561. tcgsize2size[reg_cgsize(taicpu(p).oper[0]^.reg)]
  13562. )
  13563. ) then
  13564. begin
  13565. { For:
  13566. movsx/movzx %reg1,%reg1 (same register, just different sizes)
  13567. shl/sal ##, %reg1
  13568. Remove the movsx/movzx instruction if the shift overwrites the
  13569. extended bits of the register (e.g. movslq %eax,%rax; shlq $32,%rax
  13570. }
  13571. DebugMsg(SPeepholeOptimization + 'MovxShl2Shl',p);
  13572. RemoveCurrentP(p, hp1);
  13573. Result := True;
  13574. Exit;
  13575. end
  13576. else if reg_and_hp1_is_instr and
  13577. (taicpu(p).oper[0]^.typ = top_reg) and
  13578. (
  13579. ((taicpu(hp1).opcode = A_SHR) and (taicpu(p).opcode = A_MOVZX)) or
  13580. ((taicpu(hp1).opcode = A_SAR) and (taicpu(p).opcode <> A_MOVZX))
  13581. ) and
  13582. (taicpu(hp1).oper[0]^.typ = top_const) and
  13583. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  13584. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  13585. { Minimum shift value allowed is the bit size of the smallest register - 1 }
  13586. (taicpu(hp1).oper[0]^.val <
  13587. { Multiply by 8 because tcgsize2size returns bytes, not bits }
  13588. 8 * (
  13589. tcgsize2size[reg_cgsize(taicpu(p).oper[0]^.reg)]
  13590. )
  13591. ) then
  13592. begin
  13593. { For:
  13594. movsx %reg1,%reg1 movzx %reg1,%reg1 (same register, just different sizes)
  13595. sar ##, %reg1 shr ##, %reg1
  13596. Move the shift to before the movx instruction if the shift value
  13597. is not too large.
  13598. }
  13599. asml.Remove(hp1);
  13600. asml.InsertBefore(hp1, p);
  13601. taicpu(hp1).oper[1]^.reg := taicpu(p).oper[0]^.reg;
  13602. case taicpu(p).opsize of
  13603. s_BW, S_BL{$ifdef x86_64}, S_BQ{$endif}:
  13604. taicpu(hp1).opsize := S_B;
  13605. S_WL{$ifdef x86_64}, S_WQ{$endif}:
  13606. taicpu(hp1).opsize := S_W;
  13607. {$ifdef x86_64}
  13608. S_LQ:
  13609. taicpu(hp1).opsize := S_L;
  13610. {$endif}
  13611. else
  13612. InternalError(2020112401);
  13613. end;
  13614. if (taicpu(hp1).opcode = A_SHR) then
  13615. DebugMsg(SPeepholeOptimization + 'MovzShr2ShrMovz', hp1)
  13616. else
  13617. DebugMsg(SPeepholeOptimization + 'MovsSar2SarMovs', hp1);
  13618. Result := True;
  13619. end;
  13620. if reg_and_hp1_is_instr and
  13621. (taicpu(p).oper[0]^.typ = top_reg) and
  13622. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  13623. (
  13624. (taicpu(hp1).opcode = taicpu(p).opcode)
  13625. or ((taicpu(p).opcode = A_MOVZX) and ((taicpu(hp1).opcode = A_MOVSX){$ifdef x86_64} or (taicpu(hp1).opcode = A_MOVSXD){$endif x86_64}))
  13626. {$ifdef x86_64}
  13627. or ((taicpu(p).opcode = A_MOVSX) and (taicpu(hp1).opcode = A_MOVSXD))
  13628. {$endif x86_64}
  13629. ) then
  13630. begin
  13631. if MatchOpType(taicpu(hp1), top_reg, top_reg) and
  13632. (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[0]^.reg) and
  13633. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg) then
  13634. begin
  13635. {
  13636. For example:
  13637. movzbw %al,%ax
  13638. movzwl %ax,%eax
  13639. Compress into:
  13640. movzbl %al,%eax
  13641. }
  13642. RegUsed := False;
  13643. case taicpu(p).opsize of
  13644. S_BW:
  13645. case taicpu(hp1).opsize of
  13646. S_WL:
  13647. begin
  13648. taicpu(p).opsize := S_BL;
  13649. RegUsed := True;
  13650. end;
  13651. {$ifdef x86_64}
  13652. S_WQ:
  13653. begin
  13654. if taicpu(p).opcode = A_MOVZX then
  13655. begin
  13656. taicpu(p).opsize := S_BL;
  13657. { 64-bit zero extension is implicit, so change to the 32-bit register }
  13658. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  13659. end
  13660. else
  13661. taicpu(p).opsize := S_BQ;
  13662. RegUsed := True;
  13663. end;
  13664. {$endif x86_64}
  13665. else
  13666. ;
  13667. end;
  13668. {$ifdef x86_64}
  13669. S_BL:
  13670. case taicpu(hp1).opsize of
  13671. S_LQ:
  13672. begin
  13673. if taicpu(p).opcode = A_MOVZX then
  13674. begin
  13675. taicpu(p).opsize := S_BL;
  13676. { 64-bit zero extension is implicit, so change to the 32-bit register }
  13677. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  13678. end
  13679. else
  13680. taicpu(p).opsize := S_BQ;
  13681. RegUsed := True;
  13682. end;
  13683. else
  13684. ;
  13685. end;
  13686. S_WL:
  13687. case taicpu(hp1).opsize of
  13688. S_LQ:
  13689. begin
  13690. if taicpu(p).opcode = A_MOVZX then
  13691. begin
  13692. taicpu(p).opsize := S_WL;
  13693. { 64-bit zero extension is implicit, so change to the 32-bit register }
  13694. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  13695. end
  13696. else
  13697. taicpu(p).opsize := S_WQ;
  13698. RegUsed := True;
  13699. end;
  13700. else
  13701. ;
  13702. end;
  13703. {$endif x86_64}
  13704. else
  13705. ;
  13706. end;
  13707. if RegUsed then
  13708. begin
  13709. DebugMsg(SPeepholeOptimization + 'MovxMovx2Movx', p);
  13710. taicpu(p).oper[1]^.reg := taicpu(hp1).oper[1]^.reg;
  13711. RemoveInstruction(hp1);
  13712. Result := True;
  13713. Exit;
  13714. end;
  13715. end;
  13716. if (taicpu(hp1).opsize = taicpu(p).opsize) and
  13717. not RegInInstruction(taicpu(p).oper[1]^.reg, hp1) and
  13718. GetNextInstruction(hp1, hp2) and
  13719. MatchInstruction(hp2, [A_AND, A_OR, A_XOR, A_TEST], []) and
  13720. (
  13721. ((taicpu(hp2).opsize = S_W) and (taicpu(p).opsize = S_BW)) or
  13722. ((taicpu(hp2).opsize = S_L) and (taicpu(p).opsize in [S_BL, S_WL]))
  13723. {$ifdef x86_64}
  13724. or ((taicpu(hp2).opsize = S_Q) and (taicpu(p).opsize in [S_BL, S_BQ, S_WL, S_WQ, S_LQ]))
  13725. {$endif x86_64}
  13726. ) and
  13727. MatchOpType(taicpu(hp2), top_reg, top_reg) and
  13728. (
  13729. (
  13730. (taicpu(hp2).oper[0]^.reg = taicpu(hp1).oper[1]^.reg) and
  13731. (taicpu(hp2).oper[1]^.reg = taicpu(p).oper[1]^.reg)
  13732. ) or
  13733. (
  13734. { Only allow the operands in reverse order for TEST instructions }
  13735. (taicpu(hp2).opcode = A_TEST) and
  13736. (taicpu(hp2).oper[0]^.reg = taicpu(p).oper[1]^.reg) and
  13737. (taicpu(hp2).oper[1]^.reg = taicpu(hp1).oper[1]^.reg)
  13738. )
  13739. ) then
  13740. begin
  13741. {
  13742. For example:
  13743. movzbl %al,%eax
  13744. movzbl (ref),%edx
  13745. andl %edx,%eax
  13746. (%edx deallocated)
  13747. Change to:
  13748. andb (ref),%al
  13749. movzbl %al,%eax
  13750. Rules are:
  13751. - First two instructions have the same opcode and opsize
  13752. - First instruction's operands are the same super-register
  13753. - Second instruction operates on a different register
  13754. - Third instruction is AND, OR, XOR or TEST
  13755. - Third instruction's operands are the destination registers of the first two instructions
  13756. - Third instruction writes to the destination register of the first instruction (except with TEST)
  13757. - Second instruction's destination register is deallocated afterwards
  13758. }
  13759. TransferUsedRegs(TmpUsedRegs);
  13760. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  13761. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  13762. if not RegUsedAfterInstruction(taicpu(hp1).oper[1]^.reg, hp2, TmpUsedRegs) then
  13763. begin
  13764. case taicpu(p).opsize of
  13765. S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  13766. NewSize := S_B;
  13767. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  13768. NewSize := S_W;
  13769. {$ifdef x86_64}
  13770. S_LQ:
  13771. NewSize := S_L;
  13772. {$endif x86_64}
  13773. else
  13774. InternalError(2021120301);
  13775. end;
  13776. taicpu(hp2).loadoper(0, taicpu(hp1).oper[0]^);
  13777. taicpu(hp2).loadreg(1, taicpu(p).oper[0]^.reg);
  13778. taicpu(hp2).opsize := NewSize;
  13779. RemoveInstruction(hp1);
  13780. { With TEST, it's best to keep the MOVX instruction at the top }
  13781. if (taicpu(hp2).opcode <> A_TEST) then
  13782. begin
  13783. DebugMsg(SPeepholeOptimization + 'MovxMovxTest2MovxTest', p);
  13784. asml.Remove(p);
  13785. { If the third instruction uses the flags, the MOVX instruction won't modify then }
  13786. asml.InsertAfter(p, hp2);
  13787. p := hp2;
  13788. end
  13789. else
  13790. DebugMsg(SPeepholeOptimization + 'MovxMovxOp2OpMovx', p);
  13791. Result := True;
  13792. Exit;
  13793. end;
  13794. end;
  13795. end;
  13796. if taicpu(p).opcode=A_MOVZX then
  13797. begin
  13798. { removes superfluous And's after movzx's }
  13799. if reg_and_hp1_is_instr and
  13800. (taicpu(hp1).opcode = A_AND) and
  13801. MatchOpType(taicpu(hp1),top_const,top_reg) and
  13802. ((taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg)
  13803. {$ifdef x86_64}
  13804. { check for implicit extension to 64 bit }
  13805. or
  13806. ((taicpu(p).opsize in [S_BL,S_WL]) and
  13807. (taicpu(hp1).opsize=S_Q) and
  13808. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^.reg)
  13809. )
  13810. {$endif x86_64}
  13811. )
  13812. then
  13813. begin
  13814. case taicpu(p).opsize Of
  13815. S_BL, S_BW{$ifdef x86_64}, S_BQ{$endif x86_64}:
  13816. if (taicpu(hp1).oper[0]^.val = $ff) then
  13817. begin
  13818. DebugMsg(SPeepholeOptimization + 'MovzAnd2Movz1',p);
  13819. RemoveInstruction(hp1);
  13820. Result:=true;
  13821. exit;
  13822. end;
  13823. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  13824. if (taicpu(hp1).oper[0]^.val = $ffff) then
  13825. begin
  13826. DebugMsg(SPeepholeOptimization + 'MovzAnd2Movz2',p);
  13827. RemoveInstruction(hp1);
  13828. Result:=true;
  13829. exit;
  13830. end;
  13831. {$ifdef x86_64}
  13832. S_LQ:
  13833. if (taicpu(hp1).oper[0]^.val = $ffffffff) then
  13834. begin
  13835. DebugMsg(SPeepholeOptimization + 'MovzAnd2Movz3',p);
  13836. RemoveInstruction(hp1);
  13837. Result:=true;
  13838. exit;
  13839. end;
  13840. {$endif x86_64}
  13841. else
  13842. ;
  13843. end;
  13844. { we cannot get rid of the and, but can we get rid of the movz ?}
  13845. if SuperRegistersEqual(taicpu(p).oper[0]^.reg,taicpu(p).oper[1]^.reg) then
  13846. begin
  13847. case taicpu(p).opsize Of
  13848. S_BL, S_BW{$ifdef x86_64}, S_BQ{$endif x86_64}:
  13849. if (taicpu(hp1).oper[0]^.val and $ff)=taicpu(hp1).oper[0]^.val then
  13850. begin
  13851. DebugMsg(SPeepholeOptimization + 'MovzAnd2And1',p);
  13852. RemoveCurrentP(p,hp1);
  13853. Result:=true;
  13854. exit;
  13855. end;
  13856. S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  13857. if (taicpu(hp1).oper[0]^.val and $ffff)=taicpu(hp1).oper[0]^.val then
  13858. begin
  13859. DebugMsg(SPeepholeOptimization + 'MovzAnd2And2',p);
  13860. RemoveCurrentP(p,hp1);
  13861. Result:=true;
  13862. exit;
  13863. end;
  13864. {$ifdef x86_64}
  13865. S_LQ:
  13866. if (taicpu(hp1).oper[0]^.val and $ffffffff)=taicpu(hp1).oper[0]^.val then
  13867. begin
  13868. DebugMsg(SPeepholeOptimization + 'MovzAnd2And3',p);
  13869. RemoveCurrentP(p,hp1);
  13870. Result:=true;
  13871. exit;
  13872. end;
  13873. {$endif x86_64}
  13874. else
  13875. ;
  13876. end;
  13877. end;
  13878. end;
  13879. { changes some movzx constructs to faster synonyms (all examples
  13880. are given with eax/ax, but are also valid for other registers)}
  13881. if MatchOpType(taicpu(p),top_reg,top_reg) then
  13882. begin
  13883. case taicpu(p).opsize of
  13884. { Technically, movzbw %al,%ax cannot be encoded in 32/64-bit mode
  13885. (the machine code is equivalent to movzbl %al,%eax), but the
  13886. code generator still generates that assembler instruction and
  13887. it is silently converted. This should probably be checked.
  13888. [Kit] }
  13889. S_BW:
  13890. begin
  13891. if (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) and
  13892. (
  13893. not IsMOVZXAcceptable
  13894. { and $0xff,%ax has a smaller encoding but risks a partial write penalty }
  13895. or (
  13896. (cs_opt_size in current_settings.optimizerswitches) and
  13897. (taicpu(p).oper[1]^.reg = NR_AX)
  13898. )
  13899. ) then
  13900. {Change "movzbw %al, %ax" to "andw $0x0ffh, %ax"}
  13901. begin
  13902. DebugMsg(SPeepholeOptimization + 'var7',p);
  13903. taicpu(p).opcode := A_AND;
  13904. taicpu(p).changeopsize(S_W);
  13905. taicpu(p).loadConst(0,$ff);
  13906. Result := True;
  13907. end
  13908. else if not IsMOVZXAcceptable and
  13909. GetNextInstruction(p, hp1) and
  13910. (tai(hp1).typ = ait_instruction) and
  13911. (taicpu(hp1).opcode = A_AND) and
  13912. MatchOpType(taicpu(hp1),top_const,top_reg) and
  13913. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  13914. { Change "movzbw %reg1, %reg2; andw $const, %reg2"
  13915. to "movw %reg1, reg2; andw $(const1 and $ff), %reg2"}
  13916. begin
  13917. DebugMsg(SPeepholeOptimization + 'var8',p);
  13918. taicpu(p).opcode := A_MOV;
  13919. taicpu(p).changeopsize(S_W);
  13920. setsubreg(taicpu(p).oper[0]^.reg,R_SUBW);
  13921. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  13922. Result := True;
  13923. end;
  13924. end;
  13925. {$ifndef i8086} { movzbl %al,%eax cannot be encoded in 16-bit mode (the machine code is equivalent to movzbw %al,%ax }
  13926. S_BL:
  13927. if not IsMOVZXAcceptable then
  13928. begin
  13929. if (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) then
  13930. { Change "movzbl %al, %eax" to "andl $0x0ffh, %eax" }
  13931. begin
  13932. DebugMsg(SPeepholeOptimization + 'var9',p);
  13933. taicpu(p).opcode := A_AND;
  13934. taicpu(p).changeopsize(S_L);
  13935. taicpu(p).loadConst(0,$ff);
  13936. Result := True;
  13937. end
  13938. else if GetNextInstruction(p, hp1) and
  13939. (tai(hp1).typ = ait_instruction) and
  13940. (taicpu(hp1).opcode = A_AND) and
  13941. MatchOpType(taicpu(hp1),top_const,top_reg) and
  13942. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  13943. { Change "movzbl %reg1, %reg2; andl $const, %reg2"
  13944. to "movl %reg1, reg2; andl $(const1 and $ff), %reg2"}
  13945. begin
  13946. DebugMsg(SPeepholeOptimization + 'var10',p);
  13947. taicpu(p).opcode := A_MOV;
  13948. taicpu(p).changeopsize(S_L);
  13949. { do not use R_SUBWHOLE
  13950. as movl %rdx,%eax
  13951. is invalid in assembler PM }
  13952. setsubreg(taicpu(p).oper[0]^.reg, R_SUBD);
  13953. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  13954. Result := True;
  13955. end;
  13956. end;
  13957. {$endif i8086}
  13958. S_WL:
  13959. if not IsMOVZXAcceptable then
  13960. begin
  13961. if (getsupreg(taicpu(p).oper[0]^.reg)=getsupreg(taicpu(p).oper[1]^.reg)) then
  13962. { Change "movzwl %ax, %eax" to "andl $0x0ffffh, %eax" }
  13963. begin
  13964. DebugMsg(SPeepholeOptimization + 'var11',p);
  13965. taicpu(p).opcode := A_AND;
  13966. taicpu(p).changeopsize(S_L);
  13967. taicpu(p).loadConst(0,$ffff);
  13968. Result := True;
  13969. end
  13970. else if GetNextInstruction(p, hp1) and
  13971. (tai(hp1).typ = ait_instruction) and
  13972. (taicpu(hp1).opcode = A_AND) and
  13973. (taicpu(hp1).oper[0]^.typ = top_const) and
  13974. (taicpu(hp1).oper[1]^.typ = top_reg) and
  13975. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  13976. { Change "movzwl %reg1, %reg2; andl $const, %reg2"
  13977. to "movl %reg1, reg2; andl $(const1 and $ffff), %reg2"}
  13978. begin
  13979. DebugMsg(SPeepholeOptimization + 'var12',p);
  13980. taicpu(p).opcode := A_MOV;
  13981. taicpu(p).changeopsize(S_L);
  13982. { do not use R_SUBWHOLE
  13983. as movl %rdx,%eax
  13984. is invalid in assembler PM }
  13985. setsubreg(taicpu(p).oper[0]^.reg, R_SUBD);
  13986. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ffff);
  13987. Result := True;
  13988. end;
  13989. end;
  13990. else
  13991. InternalError(2017050705);
  13992. end;
  13993. end
  13994. else if not IsMOVZXAcceptable and (taicpu(p).oper[0]^.typ = top_ref) then
  13995. begin
  13996. if GetNextInstruction(p, hp1) and
  13997. (tai(hp1).typ = ait_instruction) and
  13998. (taicpu(hp1).opcode = A_AND) and
  13999. MatchOpType(taicpu(hp1),top_const,top_reg) and
  14000. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) then
  14001. begin
  14002. case taicpu(p).opsize Of
  14003. S_BL:
  14004. if (taicpu(hp1).opsize <> S_L) or
  14005. (taicpu(hp1).oper[0]^.val > $FF) then
  14006. begin
  14007. DebugMsg(SPeepholeOptimization + 'var13',p);
  14008. taicpu(hp1).changeopsize(S_L);
  14009. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  14010. Include(OptsToCheck, aoc_ForceNewIteration);
  14011. end;
  14012. S_WL:
  14013. if (taicpu(hp1).opsize <> S_L) or
  14014. (taicpu(hp1).oper[0]^.val > $FFFF) then
  14015. begin
  14016. DebugMsg(SPeepholeOptimization + 'var14',p);
  14017. taicpu(hp1).changeopsize(S_L);
  14018. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ffff);
  14019. Include(OptsToCheck, aoc_ForceNewIteration);
  14020. end;
  14021. S_BW:
  14022. if (taicpu(hp1).opsize <> S_W) or
  14023. (taicpu(hp1).oper[0]^.val > $FF) then
  14024. begin
  14025. DebugMsg(SPeepholeOptimization + 'var15',p);
  14026. taicpu(hp1).changeopsize(S_W);
  14027. taicpu(hp1).loadConst(0,taicpu(hp1).oper[0]^.val and $ff);
  14028. Include(OptsToCheck, aoc_ForceNewIteration);
  14029. end;
  14030. else
  14031. Internalerror(2017050704)
  14032. end;
  14033. end;
  14034. end;
  14035. end;
  14036. end;
  14037. {$ifdef x86_64}
  14038. function TX86AsmOptimizer.DoZeroUpper32Opt(var mov_p: tai; var and_p: tai): Boolean;
  14039. var
  14040. hp1, old_hp1: tai;
  14041. FullSourceReg, FullTargetReg: TRegister;
  14042. begin
  14043. if (mov_p.typ<>ait_instruction) or
  14044. (taicpu(mov_p).opsize<>S_L) or
  14045. not MatchOpType(taicpu(mov_p),top_reg,top_reg) then
  14046. InternalError(2025062801);
  14047. Result:=False;
  14048. FullSourceReg:=taicpu(mov_p).oper[0]^.reg; setsubreg(FullSourceReg, R_SUBQ);
  14049. FullTargetReg:=taicpu(mov_p).oper[1]^.reg; setsubreg(FullTargetReg, R_SUBQ);
  14050. { Mark the registers in the MOV command as "used" }
  14051. IncludeRegInUsedRegs(FullSourceReg,UsedRegs);
  14052. IncludeRegInUsedRegs(FullTargetReg,UsedRegs);
  14053. { This is a little hack to get DeepMOVOpt to replace the full 64-bit
  14054. registers. The MOV instruction will be put back as it was afterwards
  14055. (unless it got removed). }
  14056. taicpu(mov_p).oper[0]^.reg:=FullSourceReg;
  14057. taicpu(mov_p).oper[1]^.reg:=FullTargetReg;
  14058. { Start after the and_p otherwise that instruction will be considered
  14059. to have modified the source register }
  14060. old_hp1:=and_p;
  14061. while GetNextInstructionUsingReg(old_hp1,hp1,FullTargetReg) and
  14062. (hp1.typ=ait_instruction) do
  14063. begin
  14064. if RegReadByInstruction(FullTargetReg,hp1) and
  14065. not RegModifiedBetween(FullSourceReg,old_hp1,hp1) and
  14066. DeepMOVOpt(taicpu(mov_p),taicpu(hp1)) then
  14067. begin
  14068. { A change has occurred, just not in mov_p }
  14069. Include(OptsToCheck, aoc_ForceNewIteration);
  14070. TransferUsedRegs(TmpUsedRegs);
  14071. UpdateUsedRegsBetween(TmpUsedRegs,tai(mov_p.Next), hp1);
  14072. if not RegUsedAfterInstruction(FullTargetReg,hp1,TmpUsedRegs) and
  14073. { Just in case something didn't get modified (e.g. an
  14074. implicit register) }
  14075. not RegReadByInstruction(FullTargetReg,hp1) then
  14076. begin
  14077. { We can remove the original MOV }
  14078. DebugMsg(SPeepholeOptimization + 'Mov2Nop 3d done',mov_p);
  14079. RemoveCurrentP(mov_p);
  14080. Result := True;
  14081. Exit;
  14082. end;
  14083. end
  14084. else
  14085. Break;
  14086. old_hp1:=hp1;
  14087. end;
  14088. { Put the MOV instruction back as it was }
  14089. setsubreg(taicpu(mov_p).oper[0]^.reg,R_SUBD);
  14090. setsubreg(taicpu(mov_p).oper[1]^.reg,R_SUBD);
  14091. end;
  14092. {$endif x86_64}
  14093. function TX86AsmOptimizer.OptPass1AND(var p : tai) : boolean;
  14094. var
  14095. hp1, hp2 : tai;
  14096. MaskLength : Cardinal;
  14097. MaskedBits : TCgInt;
  14098. ActiveReg : TRegister;
  14099. begin
  14100. Result:=false;
  14101. { There are no optimisations for reference targets }
  14102. if (taicpu(p).oper[1]^.typ <> top_reg) then
  14103. Exit;
  14104. { Saves on a bunch of dereferences }
  14105. ActiveReg := taicpu(p).oper[1]^.reg;
  14106. while GetNextInstruction(p, hp1) and
  14107. (hp1.typ = ait_instruction) do
  14108. begin
  14109. if (taicpu(p).oper[0]^.typ = top_const) then
  14110. begin
  14111. case taicpu(hp1).opcode of
  14112. A_AND:
  14113. if MatchOpType(taicpu(hp1),top_const,top_reg) and
  14114. (getsupreg(taicpu(p).oper[1]^.reg) = getsupreg(taicpu(hp1).oper[1]^.reg)) and
  14115. { the second register must contain the first one, so compare their subreg types }
  14116. (getsubreg(taicpu(p).oper[1]^.reg)<=getsubreg(taicpu(hp1).oper[1]^.reg)) and
  14117. (abs(taicpu(p).oper[0]^.val and taicpu(hp1).oper[0]^.val)<$80000000) then
  14118. { change
  14119. and const1, reg
  14120. and const2, reg
  14121. to
  14122. and (const1 and const2), reg
  14123. }
  14124. begin
  14125. taicpu(hp1).loadConst(0, taicpu(p).oper[0]^.val and taicpu(hp1).oper[0]^.val);
  14126. DebugMsg(SPeepholeOptimization + 'AndAnd2And done',hp1);
  14127. RemoveCurrentP(p, hp1);
  14128. Result:=true;
  14129. exit;
  14130. end;
  14131. A_CMP:
  14132. if (PopCnt(DWord(taicpu(p).oper[0]^.val)) = 1) and { Only 1 bit set }
  14133. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[0]^.val) and
  14134. MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg) and
  14135. { Just check that the condition on the next instruction is compatible }
  14136. GetNextInstruction(hp1, hp2) and
  14137. (hp2.typ = ait_instruction) and
  14138. (taicpu(hp2).condition in [C_Z, C_E, C_NZ, C_NE])
  14139. then
  14140. { change
  14141. and 2^n, reg
  14142. cmp 2^n, reg
  14143. j(c) / set(c) / cmov(c) (c is equal or not equal)
  14144. to
  14145. and 2^n, reg
  14146. test reg, reg
  14147. j(~c) / set(~c) / cmov(~c)
  14148. }
  14149. begin
  14150. { Keep TEST instruction in, rather than remove it, because
  14151. it may trigger other optimisations such as MovAndTest2Test }
  14152. taicpu(hp1).loadreg(0, taicpu(hp1).oper[1]^.reg);
  14153. taicpu(hp1).opcode := A_TEST;
  14154. DebugMsg(SPeepholeOptimization + 'AND/CMP/J(c) -> AND/J(~c) with power of 2 constant', p);
  14155. taicpu(hp2).condition := inverse_cond(taicpu(hp2).condition);
  14156. Result := True;
  14157. Exit;
  14158. end
  14159. else if ((taicpu(p).oper[0]^.val=$ff) or (taicpu(p).oper[0]^.val=$ffff) or (taicpu(p).oper[0]^.val=$ffffffff)) and
  14160. MatchOpType(taicpu(hp1),top_const,top_reg) and
  14161. (taicpu(p).oper[0]^.val>=taicpu(hp1).oper[0]^.val) and
  14162. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^.reg) then
  14163. { change
  14164. and $ff/$ff/$ffff, reg
  14165. cmp val<=$ff/val<=$ffff/val<=$ffffffff, reg
  14166. dealloc reg
  14167. to
  14168. cmp val<=$ff/val<=$ffff/val<=$ffffffff, resized reg
  14169. }
  14170. begin
  14171. TransferUsedRegs(TmpUsedRegs);
  14172. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  14173. if not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) then
  14174. begin
  14175. DebugMsg(SPeepholeOptimization + 'AND/CMP -> CMP', p);
  14176. case taicpu(p).oper[0]^.val of
  14177. $ff:
  14178. begin
  14179. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBL);
  14180. taicpu(hp1).opsize:=S_B;
  14181. end;
  14182. $ffff:
  14183. begin
  14184. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBW);
  14185. taicpu(hp1).opsize:=S_W;
  14186. end;
  14187. $ffffffff:
  14188. begin
  14189. setsubreg(taicpu(hp1).oper[1]^.reg, R_SUBD);
  14190. taicpu(hp1).opsize:=S_L;
  14191. end;
  14192. else
  14193. Internalerror(2023030401);
  14194. end;
  14195. RemoveCurrentP(p);
  14196. Result := True;
  14197. Exit;
  14198. end;
  14199. end;
  14200. A_MOVZX:
  14201. if MatchOpType(taicpu(hp1),top_reg,top_reg) and
  14202. SuperRegistersEqual(taicpu(p).oper[1]^.reg,taicpu(hp1).oper[1]^.reg) and
  14203. (getsupreg(taicpu(hp1).oper[0]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg)) and
  14204. (
  14205. (
  14206. (taicpu(p).opsize=S_W) and
  14207. (taicpu(hp1).opsize=S_BW)
  14208. ) or
  14209. (
  14210. (taicpu(p).opsize=S_L) and
  14211. (taicpu(hp1).opsize in [S_WL,S_BL{$ifdef x86_64},S_BQ,S_WQ{$endif x86_64}])
  14212. )
  14213. {$ifdef x86_64}
  14214. or
  14215. (
  14216. (taicpu(p).opsize=S_Q) and
  14217. (taicpu(hp1).opsize in [S_BQ,S_WQ,S_BL,S_WL])
  14218. )
  14219. {$endif x86_64}
  14220. ) then
  14221. begin
  14222. if (((taicpu(hp1).opsize) in [S_BW,S_BL{$ifdef x86_64},S_BQ{$endif x86_64}]) and
  14223. ((taicpu(p).oper[0]^.val and $ff)=taicpu(p).oper[0]^.val)
  14224. ) or
  14225. (((taicpu(hp1).opsize) in [S_WL{$ifdef x86_64},S_WQ{$endif x86_64}]) and
  14226. ((taicpu(p).oper[0]^.val and $ffff)=taicpu(p).oper[0]^.val))
  14227. then
  14228. begin
  14229. { Unlike MOVSX, MOVZX doesn't actually have a version that zero-extends a
  14230. 32-bit register to a 64-bit register, or even a version called MOVZXD, so
  14231. code that tests for the presence of AND 0xffffffff followed by MOVZX is
  14232. wasted, and is indictive of a compiler bug if it were triggered. [Kit]
  14233. NOTE: To zero-extend from 32 bits to 64 bits, simply use the standard MOV.
  14234. }
  14235. DebugMsg(SPeepholeOptimization + 'AndMovzToAnd done',p);
  14236. RemoveInstruction(hp1);
  14237. { See if there are other optimisations possible }
  14238. Continue;
  14239. end;
  14240. end;
  14241. A_SHL:
  14242. if MatchOpType(taicpu(hp1),top_const,top_reg) and
  14243. (getsupreg(taicpu(p).oper[1]^.reg)=getsupreg(taicpu(hp1).oper[1]^.reg)) then
  14244. begin
  14245. {$ifopt R+}
  14246. {$define RANGE_WAS_ON}
  14247. {$R-}
  14248. {$endif}
  14249. { get length of potential and mask }
  14250. MaskLength:=SizeOf(taicpu(p).oper[0]^.val)*8-BsrQWord(taicpu(p).oper[0]^.val)-1;
  14251. { really a mask? }
  14252. {$ifdef RANGE_WAS_ON}
  14253. {$R+}
  14254. {$endif}
  14255. if (((QWord(1) shl MaskLength)-1)=taicpu(p).oper[0]^.val) and
  14256. { unmasked part shifted out? }
  14257. ((MaskLength+taicpu(hp1).oper[0]^.val)>=topsize2memsize[taicpu(hp1).opsize]) then
  14258. begin
  14259. DebugMsg(SPeepholeOptimization + 'AndShlToShl done',p);
  14260. RemoveCurrentP(p, hp1);
  14261. Result:=true;
  14262. exit;
  14263. end;
  14264. end;
  14265. A_SHR:
  14266. if MatchOpType(taicpu(hp1),top_const,top_reg) and
  14267. (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg) and
  14268. (taicpu(hp1).oper[0]^.val <= 63) then
  14269. begin
  14270. { Does SHR combined with the AND cover all the bits?
  14271. e.g. for "andb $252,%reg; shrb $2,%reg" - the "and" can be removed }
  14272. MaskedBits := taicpu(p).oper[0]^.val or ((TCgInt(1) shl taicpu(hp1).oper[0]^.val) - 1);
  14273. if ((taicpu(p).opsize = S_B) and ((MaskedBits and $FF) = $FF)) or
  14274. ((taicpu(p).opsize = S_W) and ((MaskedBits and $FFFF) = $FFFF)) or
  14275. ((taicpu(p).opsize = S_L) and ((MaskedBits and $FFFFFFFF) = $FFFFFFFF)) then
  14276. begin
  14277. DebugMsg(SPeepholeOptimization + 'AndShrToShr done', p);
  14278. RemoveCurrentP(p, hp1);
  14279. Result := True;
  14280. Exit;
  14281. end;
  14282. end;
  14283. A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  14284. if (taicpu(hp1).oper[0]^.typ = top_reg) and
  14285. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg) then
  14286. begin
  14287. if SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) and
  14288. (
  14289. (
  14290. (taicpu(hp1).opsize in [S_BW,S_BL{$ifdef x86_64},S_BQ{$endif x86_64}]) and
  14291. ((taicpu(p).oper[0]^.val and $7F) = taicpu(p).oper[0]^.val)
  14292. ) or (
  14293. (taicpu(hp1).opsize in [S_WL{$ifdef x86_64},S_WQ{$endif x86_64}]) and
  14294. ((taicpu(p).oper[0]^.val and $7FFF) = taicpu(p).oper[0]^.val)
  14295. {$ifdef x86_64}
  14296. ) or (
  14297. (taicpu(hp1).opsize = S_LQ) and
  14298. ((taicpu(p).oper[0]^.val and $7fffffff) = taicpu(p).oper[0]^.val)
  14299. {$endif x86_64}
  14300. )
  14301. ) then
  14302. begin
  14303. if (taicpu(p).oper[1]^.reg = taicpu(hp1).oper[1]^.reg){$ifdef x86_64} or (taicpu(hp1).opsize = S_LQ){$endif x86_64} then
  14304. begin
  14305. DebugMsg(SPeepholeOptimization + 'AndMovsxToAnd',p);
  14306. RemoveInstruction(hp1);
  14307. { See if there are other optimisations possible }
  14308. Continue;
  14309. end;
  14310. { The super-registers are the same though.
  14311. Note that this change by itself doesn't improve
  14312. code speed, but it opens up other optimisations. }
  14313. {$ifdef x86_64}
  14314. { Convert 64-bit register to 32-bit }
  14315. case taicpu(hp1).opsize of
  14316. S_BQ:
  14317. begin
  14318. taicpu(hp1).opsize := S_BL;
  14319. taicpu(hp1).oper[1]^.reg := newreg(R_INTREGISTER, getsupreg(taicpu(hp1).oper[1]^.reg), R_SUBD);
  14320. end;
  14321. S_WQ:
  14322. begin
  14323. taicpu(hp1).opsize := S_WL;
  14324. taicpu(hp1).oper[1]^.reg := newreg(R_INTREGISTER, getsupreg(taicpu(hp1).oper[1]^.reg), R_SUBD);
  14325. end
  14326. else
  14327. ;
  14328. end;
  14329. {$endif x86_64}
  14330. DebugMsg(SPeepholeOptimization + 'AndMovsxToAndMovzx', hp1);
  14331. taicpu(hp1).opcode := A_MOVZX;
  14332. { See if there are other optimisations possible }
  14333. Continue;
  14334. end;
  14335. end;
  14336. else
  14337. ;
  14338. end;
  14339. end
  14340. else if MatchOperand(taicpu(p).oper[0]^, taicpu(p).oper[1]^.reg) and
  14341. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  14342. begin
  14343. {$ifdef x86_64}
  14344. if (taicpu(p).opsize = S_Q) then
  14345. begin
  14346. { Never necessary }
  14347. DebugMsg(SPeepholeOptimization + 'Andq2Nop', p);
  14348. RemoveCurrentP(p, hp1);
  14349. Result := True;
  14350. Exit;
  14351. end;
  14352. {$endif x86_64}
  14353. { Forward check to determine necessity of and %reg,%reg }
  14354. TransferUsedRegs(TmpUsedRegs);
  14355. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  14356. case taicpu(hp1).opcode of
  14357. A_MOV, A_MOVZX, A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif x86_64}:
  14358. if (
  14359. (taicpu(hp1).oper[0]^.typ <> top_ref) or
  14360. not RegInRef(ActiveReg, taicpu(hp1).oper[0]^.ref^)
  14361. ) and
  14362. (
  14363. (taicpu(hp1).opcode <> A_MOV) or
  14364. (taicpu(hp1).oper[1]^.typ <> top_ref) or
  14365. not RegInRef(ActiveReg, taicpu(hp1).oper[1]^.ref^)
  14366. ) and
  14367. not (
  14368. { If mov %reg,%reg is present, remove that instruction instead in OptPass1MOV }
  14369. (taicpu(hp1).opcode = A_MOV) and
  14370. MatchOperand(taicpu(hp1).oper[0]^, ActiveReg) and
  14371. MatchOperand(taicpu(hp1).oper[1]^, ActiveReg)
  14372. ) and
  14373. (
  14374. (
  14375. (taicpu(hp1).oper[0]^.typ = top_reg) and
  14376. (taicpu(hp1).oper[0]^.reg = ActiveReg) and
  14377. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^.reg)
  14378. ) or
  14379. (
  14380. {$ifdef x86_64}
  14381. (
  14382. { If we read from the register, make sure it's not dependent on the upper 32 bits }
  14383. (taicpu(hp1).oper[0]^.typ <> top_reg) or
  14384. not SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, ActiveReg) or
  14385. (GetSubReg(taicpu(hp1).oper[0]^.reg) <> R_SUBQ)
  14386. ) and
  14387. {$endif x86_64}
  14388. not RegUsedAfterInstruction(ActiveReg, hp1, TmpUsedRegs)
  14389. )
  14390. ) then
  14391. begin
  14392. DebugMsg(SPeepholeOptimization + 'AndMovx2Movx', p);
  14393. RemoveCurrentP(p, hp1);
  14394. Result := True;
  14395. Exit;
  14396. end;
  14397. A_ADD,
  14398. A_AND,
  14399. A_BSF,
  14400. A_BSR,
  14401. A_BTC,
  14402. A_BTR,
  14403. A_BTS,
  14404. A_OR,
  14405. A_SUB,
  14406. A_XOR:
  14407. { Register is written to, so this will clear the upper 32 bits (2-operand instructions) }
  14408. if (
  14409. (taicpu(hp1).oper[0]^.typ <> top_ref) or
  14410. not RegInRef(ActiveReg, taicpu(hp1).oper[0]^.ref^)
  14411. ) and
  14412. MatchOperand(taicpu(hp1).oper[1]^, ActiveReg) then
  14413. begin
  14414. DebugMsg(SPeepholeOptimization + 'AndOp2Op 2', p);
  14415. RemoveCurrentP(p, hp1);
  14416. Result := True;
  14417. Exit;
  14418. end;
  14419. A_CMP,
  14420. A_TEST:
  14421. if (
  14422. (taicpu(hp1).oper[0]^.typ <> top_ref) or
  14423. not RegInRef(ActiveReg, taicpu(hp1).oper[0]^.ref^)
  14424. ) and
  14425. MatchOperand(taicpu(hp1).oper[1]^, ActiveReg) and
  14426. not RegUsedAfterInstruction(ActiveReg, hp1, TmpUsedRegs) then
  14427. begin
  14428. DebugMsg(SPeepholeOptimization + 'AND; CMP/TEST -> CMP/TEST', p);
  14429. RemoveCurrentP(p, hp1);
  14430. Result := True;
  14431. Exit;
  14432. end;
  14433. A_BSWAP,
  14434. A_NEG,
  14435. A_NOT:
  14436. { Register is written to, so this will clear the upper 32 bits (1-operand instructions) }
  14437. if MatchOperand(taicpu(hp1).oper[0]^, ActiveReg) then
  14438. begin
  14439. DebugMsg(SPeepholeOptimization + 'AndOp2Op 1', p);
  14440. RemoveCurrentP(p, hp1);
  14441. Result := True;
  14442. Exit;
  14443. end;
  14444. else
  14445. ;
  14446. end;
  14447. end;
  14448. if (taicpu(hp1).is_jmp) and
  14449. (taicpu(hp1).opcode<>A_JMP) and
  14450. not(RegInUsedRegs(taicpu(p).oper[1]^.reg,UsedRegs)) then
  14451. begin
  14452. { change
  14453. and x, reg
  14454. jxx
  14455. to
  14456. test x, reg
  14457. jxx
  14458. if reg is deallocated before the
  14459. jump, but only if it's a conditional jump (PFV)
  14460. }
  14461. DebugMsg(SPeepholeOptimization + 'AndJcc2TestJcc', p);
  14462. taicpu(p).opcode := A_TEST;
  14463. Exit;
  14464. end;
  14465. Break;
  14466. end;
  14467. { Lone AND tests }
  14468. if (taicpu(p).oper[0]^.typ = top_const) then
  14469. begin
  14470. {
  14471. - Convert and $0xFF,reg to and reg,reg if reg is 8-bit
  14472. - Convert and $0xFFFF,reg to and reg,reg if reg is 16-bit
  14473. - Convert and $0xFFFFFFFF,reg to and reg,reg if reg is 32-bit
  14474. }
  14475. if ((taicpu(p).oper[0]^.val = $FF) and (taicpu(p).opsize = S_B)) or
  14476. ((taicpu(p).oper[0]^.val = $FFFF) and (taicpu(p).opsize = S_W)) or
  14477. ((taicpu(p).oper[0]^.val = $FFFFFFFF) and (taicpu(p).opsize = S_L)) then
  14478. begin
  14479. taicpu(p).loadreg(0, taicpu(p).oper[1]^.reg);
  14480. if taicpu(p).opsize = S_L then
  14481. begin
  14482. Include(OptsToCheck,aoc_MovAnd2Mov_3);
  14483. Result := True;
  14484. end;
  14485. end;
  14486. end;
  14487. { Backward check to determine necessity of and %reg,%reg }
  14488. if (taicpu(p).oper[0]^.typ = top_reg) and
  14489. (taicpu(p).oper[0]^.reg = taicpu(p).oper[1]^.reg) and
  14490. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  14491. begin
  14492. hp2:=p;
  14493. while GetLastInstruction(hp2, hp2) and
  14494. (cs_opt_level3 in current_settings.optimizerswitches) and
  14495. (hp2.typ=ait_instruction) and
  14496. not RegModifiedByInstruction(ActiveReg,hp2) do { loop };
  14497. if Assigned(hp2) and
  14498. RegModifiedByInstruction(ActiveReg,hp2) and { Also checks if hp2 is an instruction }
  14499. { Check size of instruction to determine if the AND is effectively
  14500. a null operation }
  14501. (
  14502. (taicpu(p).opsize = taicpu(hp2).opsize) or
  14503. { Note: Don't include S_Q }
  14504. ((taicpu(p).opsize = S_L) and (taicpu(hp2).opsize in [S_BL, S_WL])) or
  14505. ((taicpu(p).opsize = S_W) and (taicpu(hp2).opsize in [S_BW, S_BL, S_WL, S_L])) or
  14506. ((taicpu(p).opsize = S_B) and (taicpu(hp2).opsize in [S_BW, S_BL, S_WL, S_W, S_L]))
  14507. ) then
  14508. begin
  14509. { AND %reg,%reg is unnecessary to zero the upper 32 bits. }
  14510. DebugMsg(SPeepholeOptimization + 'AND %reg,%reg proven unnecessary after backward search (And2Nop)', p);
  14511. RemoveCurrentP(p, hp1);
  14512. Result:=True;
  14513. Exit;
  14514. end;
  14515. end;
  14516. end;
  14517. function TX86AsmOptimizer.OptPass2ADD(var p : tai) : boolean;
  14518. var
  14519. hp1, hp2: tai;
  14520. NewRef: TReference;
  14521. Distance: Cardinal;
  14522. TempTracking: TAllUsedRegs;
  14523. DoAddMov2Lea: Boolean;
  14524. { This entire nested function is used in an if-statement below, but we
  14525. want to avoid all the used reg transfers and GetNextInstruction calls
  14526. until we really have to check }
  14527. function MemRegisterNotUsedLater: Boolean; inline;
  14528. var
  14529. hp2: tai;
  14530. begin
  14531. TransferUsedRegs(TmpUsedRegs);
  14532. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14533. UpdateUsedRegsBetween(TmpUsedRegs, p, hp1)
  14534. else
  14535. { p and hp1 will be adjacent }
  14536. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  14537. Result := not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs);
  14538. end;
  14539. begin
  14540. Result := False;
  14541. if (taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif}]) and
  14542. (taicpu(p).oper[1]^.typ = top_reg) then
  14543. begin
  14544. Distance := GetNextInstructionUsingRegCount(p, hp1, taicpu(p).oper[1]^.reg);
  14545. if (Distance = 0) or (Distance > 3) { Likely too far to make a meaningful difference } or
  14546. (hp1.typ <> ait_instruction) or
  14547. not
  14548. (
  14549. (cs_opt_level3 in current_settings.optimizerswitches) or
  14550. { GetNextInstructionUsingRegCount just returns the next valid instruction under -O2 and under }
  14551. RegInInstruction(taicpu(p).oper[1]^.reg, hp1)
  14552. ) then
  14553. Exit;
  14554. { Some of the MOV optimisations are much more in-depth. For example, if we have:
  14555. addq $x, %rax
  14556. movq %rax, %rdx
  14557. sarq $63, %rdx
  14558. (%rax still in use)
  14559. ...letting OptPass2ADD run its course (and without -Os) will produce:
  14560. leaq $x(%rax),%rdx
  14561. addq $x, %rax
  14562. sarq $63, %rdx
  14563. ...which is okay since it breaks the dependency chain between
  14564. addq and movq, but if OptPass2MOV is called first:
  14565. addq $x, %rax
  14566. cqto
  14567. ...which is better in all ways, taking only 2 cycles to execute
  14568. and much smaller in code size.
  14569. }
  14570. { The extra register tracking is quite strenuous }
  14571. if (cs_opt_level2 in current_settings.optimizerswitches) and
  14572. MatchInstruction(hp1, A_MOV, []) then
  14573. begin
  14574. { Update the register tracking to the MOV instruction }
  14575. CopyUsedRegs(TempTracking);
  14576. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14577. UpdateUsedRegsBetween(UsedRegs, p, hp1)
  14578. else
  14579. { p and hp1 will be adjacent }
  14580. UpdateUsedRegs(UsedRegs, tai(p.Next));
  14581. hp2 := hp1;
  14582. if OptPass2MOV(hp1) then
  14583. Include(OptsToCheck, aoc_ForceNewIteration);
  14584. { Reset the tracking to the current instruction }
  14585. RestoreUsedRegs(TempTracking);
  14586. ReleaseUsedRegs(TempTracking);
  14587. { if hp1 <> hp2 after the call, then hp1 got removed, so let
  14588. OptPass2ADD get called again }
  14589. if (hp1 <> hp2) then
  14590. begin
  14591. Result := True;
  14592. Exit;
  14593. end;
  14594. end;
  14595. { Change:
  14596. add %reg2,%reg1
  14597. (%reg2 not modified in between)
  14598. mov/s/z #(%reg1),%reg1 (%reg1 superregisters must be the same)
  14599. To:
  14600. mov/s/z #(%reg1,%reg2),%reg1
  14601. }
  14602. if (taicpu(p).oper[0]^.typ = top_reg) and
  14603. MatchInstruction(hp1, [A_MOV, A_MOVZX, A_MOVSX{$ifdef x86_64}, A_MOVSXD{$endif}], []) and
  14604. MatchOpType(taicpu(hp1), top_ref, top_reg) and
  14605. (taicpu(hp1).oper[0]^.ref^.scalefactor <= 1) and
  14606. (
  14607. (
  14608. (taicpu(hp1).oper[0]^.ref^.base = taicpu(p).oper[1]^.reg) and
  14609. (taicpu(hp1).oper[0]^.ref^.index = NR_NO) and
  14610. { r/esp cannot be an index }
  14611. (taicpu(p).oper[0]^.reg<>NR_STACK_POINTER_REG)
  14612. ) or (
  14613. (taicpu(hp1).oper[0]^.ref^.index = taicpu(p).oper[1]^.reg) and
  14614. (taicpu(hp1).oper[0]^.ref^.base = NR_NO)
  14615. )
  14616. ) and (
  14617. Reg1WriteOverwritesReg2Entirely(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) or
  14618. (
  14619. { If the super registers ARE equal, then this MOV/S/Z does a partial write }
  14620. not SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) and
  14621. MemRegisterNotUsedLater
  14622. )
  14623. ) then
  14624. begin
  14625. if (
  14626. { Instructions are guaranteed to be adjacent on -O2 and under }
  14627. (cs_opt_level3 in current_settings.optimizerswitches) and
  14628. RegModifiedBetween(taicpu(p).oper[0]^.reg, p, hp1)
  14629. ) then
  14630. begin
  14631. { If the other register is used in between, move the MOV
  14632. instruction to right after the ADD instruction so a
  14633. saving can still be made }
  14634. Asml.Remove(hp1);
  14635. Asml.InsertAfter(hp1, p);
  14636. taicpu(hp1).oper[0]^.ref^.base := taicpu(p).oper[1]^.reg;
  14637. taicpu(hp1).oper[0]^.ref^.index := taicpu(p).oper[0]^.reg;
  14638. DebugMsg(SPeepholeOptimization + 'AddMov2Mov done (instruction moved)', p);
  14639. RemoveCurrentp(p, hp1);
  14640. end
  14641. else
  14642. begin
  14643. AllocRegBetween(taicpu(p).oper[0]^.reg, p, hp1, UsedRegs);
  14644. taicpu(hp1).oper[0]^.ref^.base := taicpu(p).oper[1]^.reg;
  14645. taicpu(hp1).oper[0]^.ref^.index := taicpu(p).oper[0]^.reg;
  14646. DebugMsg(SPeepholeOptimization + 'AddMov2Mov done', p);
  14647. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14648. { hp1 may not be the immediate next instruction under -O3 }
  14649. RemoveCurrentp(p)
  14650. else
  14651. RemoveCurrentp(p, hp1);
  14652. end;
  14653. Result := True;
  14654. Exit;
  14655. end;
  14656. { Change:
  14657. addl/q $x,%reg1
  14658. movl/q %reg1,%reg2
  14659. To:
  14660. leal/q $x(%reg1),%reg2
  14661. addl/q $x,%reg1 (can be removed if %reg1 or the flags are not used afterwards)
  14662. Breaks the dependency chain.
  14663. }
  14664. if (taicpu(p).oper[0]^.typ = top_const) and
  14665. MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  14666. (taicpu(hp1).oper[1]^.typ = top_reg) and
  14667. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) and
  14668. (
  14669. { Instructions are guaranteed to be adjacent on -O2 and under }
  14670. not (cs_opt_level3 in current_settings.optimizerswitches) or
  14671. (
  14672. { If the flags are used, don't make the optimisation,
  14673. otherwise they will be scrambled. Fixes #41148 }
  14674. (
  14675. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) or
  14676. not RegUsedBetween(NR_DEFAULTFLAGS, p, hp1)
  14677. ) and
  14678. not RegUsedBetween(taicpu(hp1).oper[1]^.reg, p, hp1)
  14679. )
  14680. ) then
  14681. begin
  14682. TransferUsedRegs(TmpUsedRegs);
  14683. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14684. UpdateUsedRegsBetween(TmpUsedRegs, p, hp1)
  14685. else
  14686. { p and hp1 will be adjacent }
  14687. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  14688. if (
  14689. SetAndTest(
  14690. (
  14691. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) and
  14692. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)
  14693. ),
  14694. DoAddMov2Lea
  14695. ) or
  14696. { Don't do AddMov2LeaAdd under -Os, but do allow AddMov2Lea }
  14697. not (cs_opt_size in current_settings.optimizerswitches)
  14698. ) then
  14699. begin
  14700. { Change the MOV instruction to a LEA instruction, and update the
  14701. first operand }
  14702. reference_reset(NewRef, 1, []);
  14703. NewRef.base := taicpu(p).oper[1]^.reg;
  14704. NewRef.scalefactor := 1;
  14705. NewRef.offset := asizeint(taicpu(p).oper[0]^.val);
  14706. taicpu(hp1).opcode := A_LEA;
  14707. taicpu(hp1).loadref(0, NewRef);
  14708. if DoAddMov2Lea then
  14709. begin
  14710. { Since %reg1 or the flags aren't used afterwards, we can delete p completely }
  14711. DebugMsg(SPeepholeOptimization + 'AddMov2Lea', hp1);
  14712. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14713. { hp1 may not be the immediate next instruction under -O3 }
  14714. RemoveCurrentp(p)
  14715. else
  14716. RemoveCurrentp(p, hp1);
  14717. end
  14718. else
  14719. begin
  14720. hp2 := tai(hp1.Next); { for the benefit of AllocRegBetween }
  14721. { Move what is now the LEA instruction to before the ADD instruction }
  14722. Asml.Remove(hp1);
  14723. Asml.InsertBefore(hp1, p);
  14724. AllocRegBetween(taicpu(hp1).oper[1]^.reg, hp1, hp2, UsedRegs);
  14725. DebugMsg(SPeepholeOptimization + 'AddMov2LeaAdd', p);
  14726. p := hp1;
  14727. end;
  14728. Result := True;
  14729. end;
  14730. end;
  14731. end;
  14732. end;
  14733. function TX86AsmOptimizer.OptPass2Lea(var p : tai) : Boolean;
  14734. var
  14735. SubReg: TSubRegister;
  14736. hp1, hp2: tai;
  14737. CallJmp: Boolean;
  14738. begin
  14739. Result := False;
  14740. CallJmp := False;
  14741. SubReg := getsubreg(taicpu(p).oper[1]^.reg);
  14742. if not (RegInUsedRegs(NR_DEFAULTFLAGS,UsedRegs)) then
  14743. with taicpu(p).oper[0]^.ref^ do
  14744. if not Assigned(symbol) and not Assigned(relsymbol) and (index <> NR_NO) then
  14745. if (offset = 0) then
  14746. begin
  14747. if (scalefactor <= 1) and SuperRegistersEqual(base, taicpu(p).oper[1]^.reg) then
  14748. begin
  14749. taicpu(p).loadreg(0, newreg(R_INTREGISTER, getsupreg(index), SubReg));
  14750. taicpu(p).opcode := A_ADD;
  14751. DebugMsg(SPeepholeOptimization + 'Lea2AddBase done',p);
  14752. Result := True;
  14753. end
  14754. else if SuperRegistersEqual(index, taicpu(p).oper[1]^.reg) then
  14755. begin
  14756. if (base <> NR_NO) then
  14757. begin
  14758. if (scalefactor <= 1) then
  14759. begin
  14760. taicpu(p).loadreg(0, newreg(R_INTREGISTER, getsupreg(base), SubReg));
  14761. taicpu(p).opcode := A_ADD;
  14762. DebugMsg(SPeepholeOptimization + 'Lea2AddIndex done',p);
  14763. Result := True;
  14764. end;
  14765. end
  14766. else
  14767. { Convert lea (%reg,2^x),%reg to shl x,%reg }
  14768. if (scalefactor in [2, 4, 8]) then
  14769. begin
  14770. { BsrByte is, in essence, the base-2 logarithm of the scale factor }
  14771. taicpu(p).loadconst(0, BsrByte(scalefactor));
  14772. taicpu(p).opcode := A_SHL;
  14773. DebugMsg(SPeepholeOptimization + 'Lea2Shl done',p);
  14774. Result := True;
  14775. end;
  14776. end;
  14777. end
  14778. { lea x(%reg1,%reg2),%reg3 and lea x(symbol,%reg2),%reg3 have a
  14779. lot of latency, so break off the offset if %reg3 is used soon
  14780. afterwards }
  14781. else if not (cs_opt_size in current_settings.optimizerswitches) and
  14782. { If 3-component addresses don't have additional latency, don't
  14783. perform this optimisation }
  14784. not (CPUX86_HINT_FAST_3COMP_ADDR in cpu_optimization_hints[current_settings.optimizecputype]) and
  14785. GetNextInstruction(p, hp1) and
  14786. (hp1.typ = ait_instruction) and
  14787. (
  14788. (
  14789. { Permit jumps and calls since they have a larger degree of overhead }
  14790. (
  14791. not SetAndTest(is_calljmp(taicpu(hp1).opcode), CallJmp) or
  14792. (
  14793. { ... unless the register specifies the location }
  14794. (taicpu(hp1).ops > 0) and
  14795. RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[0]^)
  14796. )
  14797. ) and
  14798. (
  14799. not CallJmp and { Use the Boolean result to avoid calling "is_calljmp" twice }
  14800. RegInInstruction(taicpu(p).oper[1]^.reg, hp1)
  14801. )
  14802. )
  14803. or
  14804. (
  14805. { Check up to two instructions ahead }
  14806. GetNextInstruction(hp1, hp2) and
  14807. (hp2.typ = ait_instruction) and
  14808. (
  14809. not SetAndTest(is_calljmp(taicpu(hp2).opcode), CallJmp) or
  14810. (
  14811. { Same as above }
  14812. (taicpu(hp2).ops > 0) and
  14813. RegInOp(taicpu(p).oper[1]^.reg, taicpu(hp2).oper[0]^)
  14814. )
  14815. ) and
  14816. (
  14817. not CallJmp and { Use the Boolean result to avoid calling "is_calljmp" twice }
  14818. RegInInstruction(taicpu(p).oper[1]^.reg, hp2)
  14819. )
  14820. )
  14821. ) then
  14822. begin
  14823. { Offset will be a 32-bit signed integer, so it's safe to use in the 64-bit version of ADD }
  14824. hp2 := taicpu.op_const_reg(A_ADD, taicpu(p).opsize, offset, taicpu(p).oper[1]^.reg);
  14825. taicpu(hp2).fileinfo := taicpu(p).fileinfo;
  14826. offset := 0;
  14827. if Assigned(symbol) or Assigned(relsymbol) then
  14828. DebugMsg(SPeepholeOptimization + 'lea x(sym,%reg1),%reg2 -> lea(sym,%reg1),%reg2; add $x,%reg2 to minimise instruction latency (Lea2LeaAdd)', p)
  14829. else
  14830. DebugMsg(SPeepholeOptimization + 'lea x(%reg1,%reg2),%reg3 -> lea(%reg1,%reg2),%reg3; add $x,%reg3 to minimise instruction latency (Lea2LeaAdd)', p);
  14831. { Inserting before the next instruction rather than after the
  14832. current instruction gives more accurate register tracking }
  14833. asml.InsertBefore(hp2, hp1);
  14834. AllocRegBetween(taicpu(p).oper[1]^.reg, p, hp2, UsedRegs);
  14835. Result := True;
  14836. end;
  14837. end;
  14838. function TX86AsmOptimizer.OptPass2SUB(var p: tai): Boolean;
  14839. var
  14840. hp1, hp2: tai;
  14841. NewRef: TReference;
  14842. Distance: Cardinal;
  14843. TempTracking: TAllUsedRegs;
  14844. DoSubMov2Lea: Boolean;
  14845. begin
  14846. Result := False;
  14847. if (taicpu(p).opsize in [S_L{$ifdef x86_64}, S_Q{$endif}]) and
  14848. MatchOpType(taicpu(p),top_const,top_reg) then
  14849. begin
  14850. Distance := GetNextInstructionUsingRegCount(p, hp1, taicpu(p).oper[1]^.reg);
  14851. if (Distance = 0) or (Distance > 3) { Likely too far to make a meaningful difference } or
  14852. (hp1.typ <> ait_instruction) or
  14853. not
  14854. (
  14855. (cs_opt_level3 in current_settings.optimizerswitches) or
  14856. { GetNextInstructionUsingRegCount just returns the next valid instruction under -O2 and under }
  14857. RegInInstruction(taicpu(p).oper[1]^.reg, hp1)
  14858. ) then
  14859. Exit;
  14860. { Some of the MOV optimisations are much more in-depth. For example, if we have:
  14861. subq $x, %rax
  14862. movq %rax, %rdx
  14863. sarq $63, %rdx
  14864. (%rax still in use)
  14865. ...letting OptPass2SUB run its course (and without -Os) will produce:
  14866. leaq $-x(%rax),%rdx
  14867. movq $x, %rax
  14868. sarq $63, %rdx
  14869. ...which is okay since it breaks the dependency chain between
  14870. subq and movq, but if OptPass2MOV is called first:
  14871. subq $x, %rax
  14872. cqto
  14873. ...which is better in all ways, taking only 2 cycles to execute
  14874. and much smaller in code size.
  14875. }
  14876. { The extra register tracking is quite strenuous }
  14877. if (cs_opt_level2 in current_settings.optimizerswitches) and
  14878. MatchInstruction(hp1, A_MOV, []) then
  14879. begin
  14880. { Update the register tracking to the MOV instruction }
  14881. CopyUsedRegs(TempTracking);
  14882. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14883. UpdateUsedRegsBetween(UsedRegs, p, hp1)
  14884. else
  14885. { p and hp1 will be adjacent }
  14886. UpdateUsedRegs(UsedRegs, tai(p.Next));
  14887. hp2 := hp1;
  14888. if OptPass2MOV(hp1) then
  14889. Include(OptsToCheck, aoc_ForceNewIteration);
  14890. { Reset the tracking to the current instruction }
  14891. RestoreUsedRegs(TempTracking);
  14892. ReleaseUsedRegs(TempTracking);
  14893. { if hp1 <> hp2 after the call, then hp1 got removed, so let
  14894. OptPass2SUB get called again }
  14895. if (hp1 <> hp2) then
  14896. begin
  14897. Result := True;
  14898. Exit;
  14899. end;
  14900. end;
  14901. { Change:
  14902. subl/q $x,%reg1
  14903. movl/q %reg1,%reg2
  14904. To:
  14905. leal/q $-x(%reg1),%reg2
  14906. subl/q $x,%reg1 (can be removed if %reg1 or the flags are not used afterwards)
  14907. Breaks the dependency chain and potentially permits the removal of
  14908. a CMP instruction if one follows.
  14909. }
  14910. if MatchInstruction(hp1, A_MOV, [taicpu(p).opsize]) and
  14911. (taicpu(hp1).oper[1]^.typ = top_reg) and
  14912. MatchOperand(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^.reg) and
  14913. (
  14914. { Instructions are guaranteed to be adjacent on -O2 and under }
  14915. not (cs_opt_level3 in current_settings.optimizerswitches) or
  14916. (
  14917. { If the flags are used, don't make the optimisation,
  14918. otherwise they will be scrambled. Fixes #41148 }
  14919. (
  14920. not RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) or
  14921. not RegUsedBetween(NR_DEFAULTFLAGS, p, hp1)
  14922. ) and
  14923. not RegUsedBetween(taicpu(hp1).oper[1]^.reg, p, hp1)
  14924. )
  14925. ) then
  14926. begin
  14927. TransferUsedRegs(TmpUsedRegs);
  14928. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14929. UpdateUsedRegsBetween(TmpUsedRegs, p, hp1)
  14930. else
  14931. { p and hp1 will be adjacent }
  14932. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  14933. if (
  14934. SetAndTest(
  14935. (
  14936. not RegUsedAfterInstruction(taicpu(p).oper[1]^.reg, hp1, TmpUsedRegs) and
  14937. not RegUsedAfterInstruction(NR_DEFAULTFLAGS, hp1, TmpUsedRegs)
  14938. ),
  14939. DoSubMov2Lea
  14940. ) or
  14941. { Don't do SubMov2LeaSub under -Os, but do allow SubMov2Lea }
  14942. not (cs_opt_size in current_settings.optimizerswitches)
  14943. ) then
  14944. begin
  14945. { Change the MOV instruction to a LEA instruction, and update the
  14946. first operand }
  14947. reference_reset(NewRef, 1, []);
  14948. NewRef.base := taicpu(p).oper[1]^.reg;
  14949. NewRef.scalefactor := 1;
  14950. NewRef.offset := -taicpu(p).oper[0]^.val;
  14951. taicpu(hp1).opcode := A_LEA;
  14952. taicpu(hp1).loadref(0, NewRef);
  14953. if DoSubMov2Lea then
  14954. begin
  14955. { Since %reg1 or the flags aren't used afterwards, we can delete p completely }
  14956. DebugMsg(SPeepholeOptimization + 'SubMov2Lea', hp1);
  14957. if (cs_opt_level3 in current_settings.optimizerswitches) then
  14958. { hp1 may not be the immediate next instruction under -O3 }
  14959. RemoveCurrentp(p)
  14960. else
  14961. RemoveCurrentp(p, hp1);
  14962. end
  14963. else
  14964. begin
  14965. hp2 := tai(hp1.Next); { for the benefit of AllocRegBetween }
  14966. { Move what is now the LEA instruction to before the SUB instruction }
  14967. Asml.Remove(hp1);
  14968. Asml.InsertBefore(hp1, p);
  14969. AllocRegBetween(taicpu(hp1).oper[1]^.reg, hp1, hp2, UsedRegs);
  14970. DebugMsg(SPeepholeOptimization + 'SubMov2LeaSub', p);
  14971. p := hp1;
  14972. end;
  14973. Result := True;
  14974. end;
  14975. end;
  14976. end;
  14977. end;
  14978. function TX86AsmOptimizer.SkipSimpleInstructions(var hp1 : tai) : Boolean;
  14979. begin
  14980. { we can skip all instructions not messing with the stack pointer }
  14981. while assigned(hp1) and {MatchInstruction(hp1,[A_LEA,A_MOV,A_MOVQ,A_MOVSQ,A_MOVSX,A_MOVSXD,A_MOVZX,
  14982. A_AND,A_OR,A_XOR,A_ADD,A_SHR,A_SHL,A_IMUL,A_SETcc,A_SAR,A_SUB,A_TEST,A_CMOVcc,
  14983. A_MOVSS,A_MOVSD,A_MOVAPS,A_MOVUPD,A_MOVAPD,A_MOVUPS,
  14984. A_VMOVSS,A_VMOVSD,A_VMOVAPS,A_VMOVUPD,A_VMOVAPD,A_VMOVUPS],[]) and}
  14985. ({(taicpu(hp1).ops=0) or }
  14986. ({(MatchOpType(taicpu(hp1),top_reg,top_reg) or MatchOpType(taicpu(hp1),top_const,top_reg) or
  14987. (MatchOpType(taicpu(hp1),top_ref,top_reg))
  14988. ) and }
  14989. not(RegInInstruction(NR_STACK_POINTER_REG,hp1)) { and not(RegInInstruction(NR_FRAME_POINTER_REG,hp1))}
  14990. )
  14991. ) do
  14992. GetNextInstruction(hp1,hp1);
  14993. Result:=assigned(hp1);
  14994. end;
  14995. function TX86AsmOptimizer.PostPeepholeOptLea(var p : tai) : Boolean;
  14996. var
  14997. hp1, hp2, hp3, hp4, hp5, hp6, hp7, hp8: tai;
  14998. begin
  14999. Result:=false;
  15000. {$ifdef x86_64}
  15001. { Change:
  15002. lea x(%reg1d,%reg2d),%reg3d
  15003. To:
  15004. lea x(%reg1q,%reg2q),%reg3d
  15005. Reduces the number of bytes of machine code
  15006. }
  15007. if (getsubreg(taicpu(p).oper[1]^.reg)=R_SUBD) and
  15008. (
  15009. (getsubreg(taicpu(p).oper[0]^.ref^.base)=R_SUBD) or
  15010. (getsubreg(taicpu(p).oper[0]^.ref^.index)=R_SUBD)
  15011. ) then
  15012. begin
  15013. DebugMsg(SPeepholeOptimization + 'Changed 32-bit registers in reference to 64-bit (reduces instruction size)', p);
  15014. if (getsubreg(taicpu(p).oper[0]^.ref^.base)=R_SUBD) then
  15015. setsubreg(taicpu(p).oper[0]^.ref^.base,R_SUBQ);
  15016. if (getsubreg(taicpu(p).oper[0]^.ref^.index)=R_SUBD) then
  15017. setsubreg(taicpu(p).oper[0]^.ref^.index,R_SUBQ);
  15018. { No reason to set Result to true }
  15019. end;
  15020. {$endif x86_64}
  15021. hp5:=nil;
  15022. hp6:=nil;
  15023. hp7:=nil;
  15024. hp8:=nil;
  15025. { replace
  15026. leal(q) x(<stackpointer>),<stackpointer>
  15027. <optional .seh_stackalloc ...>
  15028. <optional .seh_endprologue ...>
  15029. call procname
  15030. <optional NOP>
  15031. leal(q) -x(<stackpointer>),<stackpointer>
  15032. <optional VZEROUPPER>
  15033. ret
  15034. by
  15035. jmp procname
  15036. but do it only on level 4 because it destroys stack back traces
  15037. }
  15038. if (cs_opt_level4 in current_settings.optimizerswitches) and
  15039. (taicpu(p).oper[1]^.reg=NR_STACK_POINTER_REG) and
  15040. (taicpu(p).oper[0]^.ref^.base=NR_STACK_POINTER_REG) and
  15041. (taicpu(p).oper[0]^.ref^.index=NR_NO) and
  15042. { the -8, -24, -40 are not required, but bail out early if possible,
  15043. higher values are unlikely }
  15044. ((taicpu(p).oper[0]^.ref^.offset=-8) or
  15045. (taicpu(p).oper[0]^.ref^.offset=-24) or
  15046. (taicpu(p).oper[0]^.ref^.offset=-40)) and
  15047. (taicpu(p).oper[0]^.ref^.symbol=nil) and
  15048. (taicpu(p).oper[0]^.ref^.relsymbol=nil) and
  15049. GetNextInstruction(p, hp1) and
  15050. { Take a copy of hp1 }
  15051. SetAndTest(hp1, hp4) and
  15052. { trick to skip label }
  15053. ((hp1.typ=ait_instruction) or (SetAndTest(hp1, hp7) and GetNextInstruction(hp1, hp1))) and
  15054. { skip directives, .seh_stackalloc and .seh_endprologue on windows
  15055. ((hp1.typ=ait_instruction) or (SetAndTest(hp1, hp7) and GetNextInstruction(hp1, hp1))) and
  15056. ((hp1.typ=ait_instruction) or (SetAndTest(hp1, hp8) and GetNextInstruction(hp1, hp1))) and }
  15057. SkipSimpleInstructions(hp1) and
  15058. MatchInstruction(hp1,A_CALL,[S_NO]) and
  15059. GetNextInstruction(hp1, hp2) and
  15060. (MatchInstruction(hp2,A_LEA,[taicpu(p).opsize]) or
  15061. { skip nop instruction on win64 }
  15062. (MatchInstruction(hp2,A_NOP,[S_NO]) and
  15063. SetAndTest(hp2,hp6) and
  15064. GetNextInstruction(hp2,hp2) and
  15065. MatchInstruction(hp2,A_LEA,[taicpu(p).opsize]))
  15066. ) and
  15067. (taicpu(hp2).oper[1]^.reg=NR_STACK_POINTER_REG) and
  15068. (taicpu(hp2).oper[0]^.ref^.offset=-taicpu(p).oper[0]^.ref^.offset) and
  15069. (taicpu(hp2).oper[0]^.ref^.base=NR_STACK_POINTER_REG) and
  15070. (taicpu(hp2).oper[0]^.ref^.index=NR_NO) and
  15071. (taicpu(hp2).oper[0]^.ref^.symbol=nil) and
  15072. (taicpu(hp2).oper[0]^.ref^.relsymbol=nil) and
  15073. { Segment register will be NR_NO }
  15074. GetNextInstruction(hp2, hp3) and
  15075. { trick to skip label }
  15076. ((hp3.typ=ait_instruction) or GetNextInstruction(hp3, hp3)) and
  15077. (MatchInstruction(hp3,A_RET,[S_NO]) or
  15078. (MatchInstruction(hp3,A_VZEROUPPER,[S_NO]) and
  15079. SetAndTest(hp3,hp5) and
  15080. GetNextInstruction(hp3,hp3) and
  15081. MatchInstruction(hp3,A_RET,[S_NO])
  15082. )
  15083. ) and
  15084. (taicpu(hp3).ops=0) then
  15085. begin
  15086. taicpu(hp1).opcode := A_JMP;
  15087. taicpu(hp1).is_jmp := true;
  15088. DebugMsg(SPeepholeOptimization + 'LeaCallLeaRet2Jmp done',p);
  15089. { search for the stackalloc directive and remove it }
  15090. hp7:=tai(p.next);
  15091. while assigned(hp7) and (tai(hp7).typ<>ait_instruction) do
  15092. begin
  15093. if (hp7.typ=ait_seh_directive) and (tai_seh_directive(hp7).kind=ash_stackalloc) then
  15094. begin
  15095. { sanity check }
  15096. if taicpu(p).oper[0]^.ref^.offset<>-tai_seh_directive(hp7).data.offset then
  15097. Internalerror(2024012201);
  15098. hp8:=tai(hp7.next);
  15099. RemoveInstruction(tai(hp7));
  15100. hp7:=hp8;
  15101. break;
  15102. end
  15103. else
  15104. hp7:=tai(hp7.next);
  15105. end;
  15106. RemoveCurrentP(p, hp4);
  15107. RemoveInstruction(hp2);
  15108. RemoveInstruction(hp3);
  15109. { if there is a vzeroupper instruction then move it before the jmp }
  15110. if Assigned(hp5) then
  15111. begin
  15112. AsmL.Remove(hp5);
  15113. ASmL.InsertBefore(hp5,hp1)
  15114. end;
  15115. { remove nop on win64 }
  15116. if Assigned(hp6) then
  15117. RemoveInstruction(hp6);
  15118. Result:=true;
  15119. end;
  15120. end;
  15121. function TX86AsmOptimizer.PostPeepholeOptPush(var p : tai) : Boolean;
  15122. {$ifdef x86_64}
  15123. var
  15124. hp1, hp2, hp3, hp4, hp5: tai;
  15125. {$endif x86_64}
  15126. begin
  15127. Result:=false;
  15128. {$ifdef x86_64}
  15129. hp5:=nil;
  15130. { replace
  15131. push %rax
  15132. call procname
  15133. pop %rcx
  15134. ret
  15135. by
  15136. jmp procname
  15137. but do it only on level 4 because it destroys stack back traces
  15138. It depends on the fact, that the sequence push rax/pop rcx is used for stack alignment as rcx is volatile
  15139. for all supported calling conventions
  15140. }
  15141. if (cs_opt_level4 in current_settings.optimizerswitches) and
  15142. MatchOpType(taicpu(p),top_reg) and
  15143. (taicpu(p).oper[0]^.reg=NR_RAX) and
  15144. GetNextInstruction(p, hp1) and
  15145. { Take a copy of hp1 }
  15146. SetAndTest(hp1, hp4) and
  15147. { trick to skip label }
  15148. ((hp1.typ=ait_instruction) or GetNextInstruction(hp1, hp1)) and
  15149. SkipSimpleInstructions(hp1) and
  15150. MatchInstruction(hp1,A_CALL,[S_NO]) and
  15151. GetNextInstruction(hp1, hp2) and
  15152. MatchInstruction(hp2,A_POP,[taicpu(p).opsize]) and
  15153. MatchOpType(taicpu(hp2),top_reg) and
  15154. (taicpu(hp2).oper[0]^.reg=NR_RCX) and
  15155. GetNextInstruction(hp2, hp3) and
  15156. { trick to skip label }
  15157. ((hp3.typ=ait_instruction) or GetNextInstruction(hp3, hp3)) and
  15158. (MatchInstruction(hp3,A_RET,[S_NO]) or
  15159. (MatchInstruction(hp3,A_VZEROUPPER,[S_NO]) and
  15160. SetAndTest(hp3,hp5) and
  15161. GetNextInstruction(hp3,hp3) and
  15162. MatchInstruction(hp3,A_RET,[S_NO])
  15163. )
  15164. ) and
  15165. (taicpu(hp3).ops=0) then
  15166. begin
  15167. taicpu(hp1).opcode := A_JMP;
  15168. taicpu(hp1).is_jmp := true;
  15169. DebugMsg(SPeepholeOptimization + 'PushCallPushRet2Jmp done',p);
  15170. RemoveCurrentP(p, hp4);
  15171. RemoveInstruction(hp2);
  15172. RemoveInstruction(hp3);
  15173. if Assigned(hp5) then
  15174. begin
  15175. AsmL.Remove(hp5);
  15176. ASmL.InsertBefore(hp5,hp1)
  15177. end;
  15178. Result:=true;
  15179. end;
  15180. {$endif x86_64}
  15181. end;
  15182. function TX86AsmOptimizer.PostPeepholeOptMov(var p : tai) : Boolean;
  15183. var
  15184. Value, RegName: string;
  15185. hp1: tai;
  15186. begin
  15187. Result:=false;
  15188. if (taicpu(p).oper[1]^.typ = top_reg) and (taicpu(p).oper[0]^.typ = top_const) then
  15189. begin
  15190. case taicpu(p).oper[0]^.val of
  15191. 0:
  15192. { Don't make this optimisation if the CPU flags are required, since XOR scrambles them }
  15193. if not RegInUsedRegs(NR_DEFAULTFLAGS,UsedRegs) or
  15194. (
  15195. { See if we can still convert the instruction }
  15196. GetNextInstructionUsingReg(p, hp1, NR_DEFAULTFLAGS) and
  15197. RegLoadedWithNewValue(NR_DEFAULTFLAGS, hp1)
  15198. ) then
  15199. begin
  15200. { change "mov $0,%reg" into "xor %reg,%reg" }
  15201. taicpu(p).opcode := A_XOR;
  15202. taicpu(p).loadReg(0,taicpu(p).oper[1]^.reg);
  15203. Result := True;
  15204. {$ifdef x86_64}
  15205. end
  15206. else if (taicpu(p).opsize = S_Q) then
  15207. begin
  15208. RegName := debug_regname(taicpu(p).oper[1]^.reg); { 64-bit register name }
  15209. { The actual optimization }
  15210. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  15211. taicpu(p).changeopsize(S_L);
  15212. DebugMsg(SPeepholeOptimization + 'movq $0,' + RegName + ' -> movl $0,' + debug_regname(taicpu(p).oper[1]^.reg) + ' (immediate can be represented with just 32 bits)', p);
  15213. Result := True;
  15214. end;
  15215. $1..$FFFFFFFF:
  15216. begin
  15217. { Code size reduction by J. Gareth "Kit" Moreton }
  15218. { change 64-bit register to 32-bit register to reduce code size (upper 32 bits will be set to zero) }
  15219. case taicpu(p).opsize of
  15220. S_Q:
  15221. begin
  15222. RegName := debug_regname(taicpu(p).oper[1]^.reg); { 64-bit register name }
  15223. Value := debug_tostr(taicpu(p).oper[0]^.val);
  15224. { The actual optimization }
  15225. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  15226. taicpu(p).changeopsize(S_L);
  15227. DebugMsg(SPeepholeOptimization + 'movq $' + Value + ',' + RegName + ' -> movl $' + Value + ',' + debug_regname(taicpu(p).oper[1]^.reg) + ' (immediate can be represented with just 32 bits)', p);
  15228. Result := True;
  15229. end;
  15230. else
  15231. { Do nothing };
  15232. end;
  15233. {$endif x86_64}
  15234. end;
  15235. -1:
  15236. { Don't make this optimisation if the CPU flags are required, since OR scrambles them }
  15237. if (cs_opt_size in current_settings.optimizerswitches) and
  15238. (taicpu(p).opsize <> S_B) and
  15239. (
  15240. not RegInUsedRegs(NR_DEFAULTFLAGS,UsedRegs) or
  15241. (
  15242. { See if we can still convert the instruction }
  15243. GetNextInstructionUsingReg(p, hp1, NR_DEFAULTFLAGS) and
  15244. RegLoadedWithNewValue(NR_DEFAULTFLAGS, hp1)
  15245. )
  15246. ) then
  15247. begin
  15248. { change "mov $-1,%reg" into "or $-1,%reg" }
  15249. { NOTES:
  15250. - No size saving is made when changing a Word-sized assignment unless the register is AX (smaller encoding)
  15251. - This operation creates a false dependency on the register, so only do it when optimising for size
  15252. - It is possible to set memory operands using this method, but this creates an even greater false dependency, so don't do this at all
  15253. }
  15254. taicpu(p).opcode := A_OR;
  15255. DebugMsg(SPeepholeOptimization + 'Mov-12Or-1',p);
  15256. Result := True;
  15257. end;
  15258. else
  15259. { Do nothing };
  15260. end;
  15261. end;
  15262. end;
  15263. { Returns true if the given logic instruction can be converted into a BTx instruction (BT not included) }
  15264. class function TX86AsmOptimizer.IsBTXAcceptable(p : tai) : boolean;
  15265. begin
  15266. Result := False;
  15267. if not (CPUX86_HAS_BTX in cpu_capabilities[current_settings.optimizecputype]) then
  15268. Exit;
  15269. { For sizes less than S_L, the byte size is equal or larger with BTx,
  15270. so don't bother optimising }
  15271. if not MatchInstruction(p, A_AND, A_OR, A_XOR, [S_L{$ifdef x86_64}, S_Q{$endif x86_64}]) then
  15272. Exit;
  15273. if (taicpu(p).oper[0]^.typ <> top_const) or
  15274. { If the value can fit into an 8-bit signed integer, a smaller
  15275. instruction can be encoded with AND/OR/XOR, so don't optimise if it
  15276. falls within this range }
  15277. (
  15278. (taicpu(p).oper[0]^.val > -128) and
  15279. (taicpu(p).oper[0]^.val <= 127)
  15280. ) then
  15281. Exit;
  15282. { If we're optimising for size, this is acceptable }
  15283. if (cs_opt_size in current_settings.optimizerswitches) then
  15284. Exit(True);
  15285. if (taicpu(p).oper[1]^.typ = top_reg) and
  15286. (CPUX86_HINT_FAST_BTX_REG_IMM in cpu_optimization_hints[current_settings.optimizecputype]) then
  15287. Exit(True);
  15288. if (taicpu(p).oper[1]^.typ <> top_reg) and
  15289. (CPUX86_HINT_FAST_BTX_MEM_IMM in cpu_optimization_hints[current_settings.optimizecputype]) then
  15290. Exit(True);
  15291. end;
  15292. function TX86AsmOptimizer.PostPeepholeOptAnd(var p : tai) : boolean;
  15293. var
  15294. hp1: tai;
  15295. Value: TCGInt;
  15296. begin
  15297. Result := False;
  15298. if MatchOpType(taicpu(p), top_const, top_reg) then
  15299. begin
  15300. { Detect:
  15301. andw x, %ax (0 <= x < $8000)
  15302. ...
  15303. movzwl %ax,%eax
  15304. Change movzwl %ax,%eax to cwtl (shorter encoding for movswl %ax,%eax)
  15305. }
  15306. if (taicpu(p).oper[1]^.reg = NR_AX) and { This is also enough to determine that opsize = S_W }
  15307. ((taicpu(p).oper[0]^.val and $7FFF) = taicpu(p).oper[0]^.val) and
  15308. GetNextInstructionUsingReg(p, hp1, NR_EAX) and
  15309. MatchInstruction(hp1, A_MOVZX, [S_WL]) and
  15310. MatchOperand(taicpu(hp1).oper[0]^, NR_AX) and
  15311. MatchOperand(taicpu(hp1).oper[1]^, NR_EAX) then
  15312. begin
  15313. DebugMsg(SPeepholeOptimization + 'Converted movzwl %ax,%eax to cwtl (via AndMovz2AndCwtl)', hp1);
  15314. taicpu(hp1).opcode := A_CWDE;
  15315. taicpu(hp1).clearop(0);
  15316. taicpu(hp1).clearop(1);
  15317. taicpu(hp1).ops := 0;
  15318. { A change was made, but not with p, so don't set Result, but
  15319. notify the compiler that a change was made }
  15320. Include(OptsToCheck, aoc_ForceNewIteration);
  15321. Exit; { and -> btr won't happen because an opsize of S_W won't be optimised anyway }
  15322. end;
  15323. end;
  15324. { If "not x" is a power of 2 (popcnt = 1), change:
  15325. and $x, %reg/ref
  15326. To:
  15327. btr lb(x), %reg/ref
  15328. }
  15329. if IsBTXAcceptable(p) and
  15330. (
  15331. { Make sure a TEST doesn't follow that plays with the register }
  15332. not GetNextInstruction(p, hp1) or
  15333. not MatchInstruction(hp1, A_TEST, A_CMP, [taicpu(p).opsize]) or
  15334. not MatchOperand(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^.reg)
  15335. ) then
  15336. begin
  15337. {$push}{$R-}{$Q-}
  15338. { Value is a sign-extended 32-bit integer - just correct it
  15339. if it's represented as an unsigned value. Also, IsBTXAcceptable
  15340. checks to see if this operand is an immediate. }
  15341. Value := not taicpu(p).oper[0]^.val;
  15342. {$pop}
  15343. {$ifdef x86_64}
  15344. if taicpu(p).opsize = S_L then
  15345. {$endif x86_64}
  15346. Value := Value and $FFFFFFFF;
  15347. if (PopCnt(QWord(Value)) = 1) then
  15348. begin
  15349. DebugMsg(SPeepholeOptimization + 'Changed AND (not $' + debug_hexstr(taicpu(p).oper[0]^.val) + ') to BTR $' + debug_tostr(BsrQWord(Value)) + ' to shrink instruction size (And2Btr)', p);
  15350. taicpu(p).opcode := A_BTR;
  15351. taicpu(p).oper[0]^.val := BsrQWord(Value); { Essentially the base 2 logarithm }
  15352. Result := True;
  15353. Exit;
  15354. end;
  15355. end;
  15356. end;
  15357. function TX86AsmOptimizer.PostPeepholeOptMOVSX(var p : tai) : boolean;
  15358. begin
  15359. Result := False;
  15360. if not MatchOpType(taicpu(p), top_reg, top_reg) then
  15361. Exit;
  15362. { Convert:
  15363. movswl %ax,%eax -> cwtl
  15364. movslq %eax,%rax -> cdqe
  15365. NOTE: Don't convert movswl %al,%ax to cbw, because cbw and cwde
  15366. refer to the same opcode and depends only on the assembler's
  15367. current operand-size attribute. [Kit]
  15368. }
  15369. with taicpu(p) do
  15370. case opsize of
  15371. S_WL:
  15372. if (oper[0]^.reg = NR_AX) and (oper[1]^.reg = NR_EAX) then
  15373. begin
  15374. DebugMsg(SPeepholeOptimization + 'Converted movswl %ax,%eax to cwtl', p);
  15375. opcode := A_CWDE;
  15376. clearop(0);
  15377. clearop(1);
  15378. ops := 0;
  15379. Result := True;
  15380. end;
  15381. {$ifdef x86_64}
  15382. S_LQ:
  15383. if (oper[0]^.reg = NR_EAX) and (oper[1]^.reg = NR_RAX) then
  15384. begin
  15385. DebugMsg(SPeepholeOptimization + 'Converted movslq %eax,%rax to cltq', p);
  15386. opcode := A_CDQE;
  15387. clearop(0);
  15388. clearop(1);
  15389. ops := 0;
  15390. Result := True;
  15391. end;
  15392. {$endif x86_64}
  15393. else
  15394. ;
  15395. end;
  15396. end;
  15397. function TX86AsmOptimizer.PostPeepholeOptShr(var p : tai) : boolean;
  15398. var
  15399. hp1: tai;
  15400. begin
  15401. Result := False;
  15402. { All these optimisations work on "shr const,%reg" }
  15403. if not MatchOpType(taicpu(p), top_const, top_reg) then
  15404. Exit;
  15405. if HandleSHRMerge(p, True) then
  15406. begin
  15407. Result := True;
  15408. Exit;
  15409. end;
  15410. { Detect the following (looking backwards):
  15411. shr %cl,%reg
  15412. shr x, %reg
  15413. Swap the two SHR instructions to minimise a pipeline stall.
  15414. }
  15415. if GetLastInstruction(p, hp1) and
  15416. MatchInstruction(hp1, A_SHR, [taicpu(p).opsize]) and
  15417. MatchOpType(taicpu(hp1), top_reg, top_reg) and
  15418. { First operand will be %cl }
  15419. (taicpu(hp1).oper[1]^.reg = taicpu(p).oper[1]^.reg) and
  15420. { Just to be sure }
  15421. (getsupreg(taicpu(hp1).oper[1]^.reg) <> RS_ECX) then
  15422. begin
  15423. DebugMsg(SPeepholeOptimization + 'Swapped variable and constant SHR instructions to minimise pipeline stall (ShrShr2ShrShr)', hp1);
  15424. { Moving the entries this way ensures the register tracking remains correct }
  15425. Asml.Remove(p);
  15426. Asml.InsertBefore(p, hp1);
  15427. p := hp1;
  15428. { Don't set Result to True because the current instruction is now
  15429. "shr %cl,%reg" and there's nothing more we can do with it }
  15430. end;
  15431. end;
  15432. function TX86AsmOptimizer.PostPeepholeOptADDSUB(var p : tai) : boolean;
  15433. var
  15434. hp1, hp2: tai;
  15435. Opposite, SecondOpposite: TAsmOp;
  15436. NewCond: TAsmCond;
  15437. begin
  15438. Result := False;
  15439. { Change:
  15440. add/sub 128,(dest)
  15441. To:
  15442. sub/add -128,(dest)
  15443. This generaally takes fewer bytes to encode because -128 can be stored
  15444. in a signed byte, whereas +128 cannot.
  15445. }
  15446. if (taicpu(p).opsize <> S_B) and MatchOperand(taicpu(p).oper[0]^, 128) then
  15447. begin
  15448. if taicpu(p).opcode = A_ADD then
  15449. Opposite := A_SUB
  15450. else
  15451. Opposite := A_ADD;
  15452. { Be careful if the flags are in use, because the CF flag inverts
  15453. when changing from ADD to SUB and vice versa }
  15454. if RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) and
  15455. GetNextInstruction(p, hp1) then
  15456. begin
  15457. TransferUsedRegs(TmpUsedRegs);
  15458. TmpUsedRegs[R_SPECIALREGISTER].Update(tai(p.Next), True);
  15459. hp2 := hp1;
  15460. { Scan ahead to check if everything's safe }
  15461. while Assigned(hp1) and RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) do
  15462. begin
  15463. if (hp1.typ <> ait_instruction) then
  15464. { Probably unsafe since the flags are still in use }
  15465. Exit;
  15466. if MatchInstruction(hp1, A_CALL, A_JMP, A_RET, []) then
  15467. { Stop searching at an unconditional jump }
  15468. Break;
  15469. if not
  15470. (
  15471. MatchInstruction(hp1, A_ADC, A_SBB, []) and
  15472. (taicpu(hp1).oper[0]^.typ = top_const) { We need to be able to invert a constant }
  15473. ) and
  15474. (taicpu(hp1).condition = C_None) and RegInInstruction(NR_DEFAULTFLAGS, hp1) then
  15475. { Instruction depends on FLAGS (and is not ADC or SBB); break out }
  15476. Exit;
  15477. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  15478. TmpUsedRegs[R_SPECIALREGISTER].Update(tai(hp1.Next), True);
  15479. { Move to the next instruction }
  15480. GetNextInstruction(hp1, hp1);
  15481. end;
  15482. while Assigned(hp2) and (hp2 <> hp1) do
  15483. begin
  15484. NewCond := C_None;
  15485. case taicpu(hp2).condition of
  15486. C_A, C_NBE:
  15487. NewCond := C_BE;
  15488. C_B, C_C, C_NAE:
  15489. NewCond := C_AE;
  15490. C_AE, C_NB, C_NC:
  15491. NewCond := C_B;
  15492. C_BE, C_NA:
  15493. NewCond := C_A;
  15494. else
  15495. { No change needed };
  15496. end;
  15497. if NewCond <> C_None then
  15498. begin
  15499. DebugMsg(SPeepholeOptimization + 'Condition changed from ' + cond2str[taicpu(hp2).condition] + ' to ' + cond2str[NewCond] +
  15500. ' to accommodate ' + debug_op2str(taicpu(p).opcode) + ' -> ' + debug_op2str(opposite) + ' above', hp2);
  15501. taicpu(hp2).condition := NewCond;
  15502. end
  15503. else
  15504. if MatchInstruction(hp2, A_ADC, A_SBB, []) then
  15505. begin
  15506. { Because of the flipping of the carry bit, to ensure
  15507. the operation remains equivalent, ADC becomes SBB
  15508. and vice versa, and the constant is not-inverted.
  15509. If multiple ADCs or SBBs appear in a row, each one
  15510. changed causes the carry bit to invert, so they all
  15511. need to be flipped }
  15512. if taicpu(hp2).opcode = A_ADC then
  15513. SecondOpposite := A_SBB
  15514. else
  15515. SecondOpposite := A_ADC;
  15516. if taicpu(hp2).oper[0]^.typ <> top_const then
  15517. { Should have broken out of this optimisation already }
  15518. InternalError(2021112901);
  15519. DebugMsg(SPeepholeOptimization + debug_op2str(taicpu(hp2).opcode) + debug_opsize2str(taicpu(hp2).opsize) + ' $' + debug_tostr(taicpu(hp2).oper[0]^.val) + ',' + debug_operstr(taicpu(hp2).oper[1]^) + ' -> ' +
  15520. debug_op2str(SecondOpposite) + debug_opsize2str(taicpu(hp2).opsize) + ' $' + debug_tostr(not taicpu(hp2).oper[0]^.val) + ',' + debug_operstr(taicpu(hp2).oper[1]^) + ' to accommodate inverted carry bit', hp2);
  15521. { Bit-invert the constant (effectively equivalent to "-1 - val") }
  15522. taicpu(hp2).opcode := SecondOpposite;
  15523. taicpu(hp2).oper[0]^.val := not taicpu(hp2).oper[0]^.val;
  15524. end;
  15525. { Move to the next instruction }
  15526. GetNextInstruction(hp2, hp2);
  15527. end;
  15528. if (hp2 <> hp1) then
  15529. InternalError(2021111501);
  15530. end;
  15531. DebugMsg(SPeepholeOptimization + debug_op2str(taicpu(p).opcode) + debug_opsize2str(taicpu(p).opsize) + ' $128,' + debug_operstr(taicpu(p).oper[1]^) + ' changed to ' +
  15532. debug_op2str(opposite) + debug_opsize2str(taicpu(p).opsize) + ' $-128,' + debug_operstr(taicpu(p).oper[1]^) + ' to reduce instruction size', p);
  15533. taicpu(p).opcode := Opposite;
  15534. taicpu(p).oper[0]^.val := -128;
  15535. { No further optimisations can be made on this instruction, so move
  15536. onto the next one to save time }
  15537. p := tai(p.Next);
  15538. UpdateUsedRegs(p);
  15539. Result := True;
  15540. Exit;
  15541. end;
  15542. { Detect:
  15543. add/sub %reg2,(dest)
  15544. add/sub x, (dest)
  15545. (dest can be a register or a reference)
  15546. Swap the instructions to minimise a pipeline stall. This reverses the
  15547. "Add swap" and "Sub swap" optimisations done in pass 1 if no new
  15548. optimisations could be made.
  15549. }
  15550. if (taicpu(p).oper[0]^.typ = top_reg) and
  15551. not RegInOp(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^) and
  15552. (
  15553. (
  15554. (taicpu(p).oper[1]^.typ = top_reg) and
  15555. { We can try searching further ahead if we're writing to a register }
  15556. GetNextInstructionUsingReg(p, hp1, taicpu(p).oper[1]^.reg)
  15557. ) or
  15558. (
  15559. (taicpu(p).oper[1]^.typ = top_ref) and
  15560. GetNextInstruction(p, hp1)
  15561. )
  15562. ) and
  15563. MatchInstruction(hp1, A_ADD, A_SUB, [taicpu(p).opsize]) and
  15564. (taicpu(hp1).oper[0]^.typ = top_const) and
  15565. MatchOperand(taicpu(p).oper[1]^, taicpu(hp1).oper[1]^) then
  15566. begin
  15567. { Make doubly sure the flags aren't in use because the order of additions may affect them }
  15568. TransferUsedRegs(TmpUsedRegs);
  15569. UpdateUsedRegs(TmpUsedRegs, tai(p.next));
  15570. hp2 := p;
  15571. while not (cs_opt_level3 in current_settings.optimizerswitches) and
  15572. GetNextInstruction(hp2, hp2) and (hp2 <> hp1) do
  15573. UpdateUsedRegs(TmpUsedRegs, tai(hp2.next));
  15574. if not RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  15575. begin
  15576. asml.remove(hp1);
  15577. asml.InsertBefore(hp1, p);
  15578. DebugMsg(SPeepholeOptimization + 'Add/Sub swap 2 done', hp1);
  15579. Result := True;
  15580. end;
  15581. end;
  15582. end;
  15583. function TX86AsmOptimizer.PostPeepholeOptCmp(var p : tai) : Boolean;
  15584. var
  15585. hp1: tai;
  15586. begin
  15587. Result:=false;
  15588. { Final check to see if CMP/MOV pairs can be changed to MOV/CMP }
  15589. while GetNextInstruction(p, hp1) and
  15590. TrySwapMovCmp(p, hp1) do
  15591. begin
  15592. if MatchInstruction(hp1, A_MOV, []) then
  15593. begin
  15594. if RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  15595. begin
  15596. { A little hacky, but since CMP doesn't read the flags, only
  15597. modify them, it's safe if they get scrambled by MOV -> XOR }
  15598. ExcludeRegFromUsedRegs(NR_DEFAULTFLAGS, UsedRegs);
  15599. Result := PostPeepholeOptMov(hp1);
  15600. {$ifdef x86_64}
  15601. if Result and MatchInstruction(hp1, A_XOR, [S_Q]) then
  15602. { Used to shrink instruction size }
  15603. PostPeepholeOptXor(hp1);
  15604. {$endif x86_64}
  15605. IncludeRegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs);
  15606. end
  15607. else
  15608. begin
  15609. Result := PostPeepholeOptMov(hp1);
  15610. {$ifdef x86_64}
  15611. if Result and MatchInstruction(hp1, A_XOR, [S_Q]) then
  15612. { Used to shrink instruction size }
  15613. PostPeepholeOptXor(hp1);
  15614. {$endif x86_64}
  15615. end;
  15616. end;
  15617. { Enabling this flag is actually a null operation, but it marks
  15618. the code as 'modified' during this pass }
  15619. Include(OptsToCheck, aoc_ForceNewIteration);
  15620. end;
  15621. { change "cmp $0, %reg" to "test %reg, %reg" }
  15622. if MatchOpType(taicpu(p),top_const,top_reg) and
  15623. (taicpu(p).oper[0]^.val = 0) then
  15624. begin
  15625. taicpu(p).opcode := A_TEST;
  15626. taicpu(p).loadreg(0,taicpu(p).oper[1]^.reg);
  15627. DebugMsg(SPeepholeOptimization + 'Cmp2Test', p);
  15628. Result:=true;
  15629. end;
  15630. end;
  15631. function TX86AsmOptimizer.PostPeepholeOptTestOr(var p : tai) : Boolean;
  15632. var
  15633. IsTestConstX, IsValid : Boolean;
  15634. hp1,hp2 : tai;
  15635. begin
  15636. Result:=false;
  15637. { Final check to see if TEST/MOV pairs can be changed to MOV/TEST }
  15638. if (taicpu(p).opcode = A_TEST) then
  15639. while GetNextInstruction(p, hp1) and
  15640. TrySwapMovCmp(p, hp1) do
  15641. begin
  15642. if MatchInstruction(hp1, A_MOV, []) then
  15643. begin
  15644. if RegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs) then
  15645. begin
  15646. { A little hacky, but since TEST doesn't read the flags, only
  15647. modify them, it's safe if they get scrambled by MOV -> XOR }
  15648. ExcludeRegFromUsedRegs(NR_DEFAULTFLAGS, UsedRegs);
  15649. Result := PostPeepholeOptMov(hp1);
  15650. {$ifdef x86_64}
  15651. if Result and MatchInstruction(hp1, A_XOR, [S_Q]) then
  15652. { Used to shrink instruction size }
  15653. PostPeepholeOptXor(hp1);
  15654. {$endif x86_64}
  15655. IncludeRegInUsedRegs(NR_DEFAULTFLAGS, UsedRegs);
  15656. end
  15657. else
  15658. begin
  15659. Result := PostPeepholeOptMov(hp1);
  15660. {$ifdef x86_64}
  15661. if Result and MatchInstruction(hp1, A_XOR, [S_Q]) then
  15662. { Used to shrink instruction size }
  15663. PostPeepholeOptXor(hp1);
  15664. {$endif x86_64}
  15665. end;
  15666. end;
  15667. { Enabling this flag is actually a null operation, but it marks
  15668. the code as 'modified' during this pass }
  15669. Include(OptsToCheck, aoc_ForceNewIteration);
  15670. end;
  15671. { If x is a power of 2 (popcnt = 1), change:
  15672. or $x, %reg/ref
  15673. To:
  15674. bts lb(x), %reg/ref
  15675. }
  15676. if (taicpu(p).opcode = A_OR) and
  15677. IsBTXAcceptable(p) and
  15678. { IsBTXAcceptable checks to see if oper[0] is an immediate }
  15679. (PopCnt(QWord(taicpu(p).oper[0]^.val)) = 1) and
  15680. (
  15681. { Don't optimise if a test instruction follows }
  15682. not GetNextInstruction(p, hp1) or
  15683. not MatchInstruction(hp1, A_TEST, [taicpu(p).opsize])
  15684. ) then
  15685. begin
  15686. DebugMsg(SPeepholeOptimization + 'Changed OR $' + debug_hexstr(taicpu(p).oper[0]^.val) + ' to BTS $' + debug_tostr(BsrQWord(taicpu(p).oper[0]^.val)) + ' to shrink instruction size (Or2Bts)', p);
  15687. taicpu(p).opcode := A_BTS;
  15688. taicpu(p).oper[0]^.val := BsrQWord(taicpu(p).oper[0]^.val); { Essentially the base 2 logarithm }
  15689. Result := True;
  15690. Exit;
  15691. end;
  15692. { If x is a power of 2 (popcnt = 1), change:
  15693. test $x, %reg/ref
  15694. je / sete / cmove (or jne / setne)
  15695. To:
  15696. bt lb(x), %reg/ref
  15697. jnc / setnc / cmovnc (or jc / setc / cmovnc)
  15698. }
  15699. if (taicpu(p).opcode = A_TEST) and
  15700. (CPUX86_HAS_BTX in cpu_capabilities[current_settings.optimizecputype]) and
  15701. (taicpu(p).oper[0]^.typ = top_const) and
  15702. (
  15703. (cs_opt_size in current_settings.optimizerswitches) or
  15704. (
  15705. (taicpu(p).oper[1]^.typ = top_reg) and
  15706. (CPUX86_HINT_FAST_BT_REG_IMM in cpu_optimization_hints[current_settings.optimizecputype])
  15707. ) or
  15708. (
  15709. (taicpu(p).oper[1]^.typ <> top_reg) and
  15710. (CPUX86_HINT_FAST_BT_MEM_IMM in cpu_optimization_hints[current_settings.optimizecputype])
  15711. )
  15712. ) and
  15713. (PopCnt(QWord(taicpu(p).oper[0]^.val)) = 1) and
  15714. { For sizes less than S_L, the byte size is equal or larger with BT,
  15715. so don't bother optimising }
  15716. (taicpu(p).opsize >= S_L) then
  15717. begin
  15718. IsValid := True;
  15719. { Check the next set of instructions, watching the FLAGS register
  15720. and the conditions used }
  15721. TransferUsedRegs(TmpUsedRegs);
  15722. UpdateUsedRegs(TmpUsedRegs, tai(p.Next));
  15723. hp1 := p;
  15724. hp2 := nil;
  15725. while GetNextInstruction(hp1, hp1) do
  15726. begin
  15727. if not Assigned(hp2) then
  15728. { The first instruction after TEST }
  15729. hp2 := hp1;
  15730. if (hp1.typ <> ait_instruction) then
  15731. begin
  15732. { If the flags are no longer in use, everything is fine }
  15733. if RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) then
  15734. IsValid := False;
  15735. Break;
  15736. end;
  15737. case taicpu(hp1).condition of
  15738. C_None:
  15739. begin
  15740. if RegInUsedRegs(NR_DEFAULTFLAGS, TmpUsedRegs) and
  15741. not RegLoadedWithNewValue(NR_DEFAULTFLAGS, hp1) then
  15742. { Something is not quite normal, so play safe and don't change }
  15743. IsValid := False;
  15744. Break;
  15745. end;
  15746. C_E, C_Z, C_NE, C_NZ:
  15747. { This is fine };
  15748. else
  15749. begin
  15750. { Unsupported condition }
  15751. IsValid := False;
  15752. Break;
  15753. end;
  15754. end;
  15755. UpdateUsedRegs(TmpUsedRegs, tai(hp1.Next));
  15756. end;
  15757. if IsValid then
  15758. begin
  15759. while hp2 <> hp1 do
  15760. begin
  15761. case taicpu(hp2).condition of
  15762. C_Z, C_E:
  15763. taicpu(hp2).condition := C_NC;
  15764. C_NZ, C_NE:
  15765. taicpu(hp2).condition := C_C;
  15766. else
  15767. { Should not get this by this point }
  15768. InternalError(2022110701);
  15769. end;
  15770. GetNextInstruction(hp2, hp2);
  15771. end;
  15772. DebugMsg(SPeepholeOptimization + 'Changed TEST $' + debug_hexstr(taicpu(p).oper[0]^.val) + ' to BT $' + debug_tostr(BsrQWord(taicpu(p).oper[0]^.val)) + ' to shrink instruction size (Test2Bt)', p);
  15773. taicpu(p).opcode := A_BT;
  15774. taicpu(p).oper[0]^.val := BsrQWord(taicpu(p).oper[0]^.val); { Essentially the base 2 logarithm }
  15775. Result := True;
  15776. Exit;
  15777. end;
  15778. end;
  15779. { removes the line marked with (x) from the sequence
  15780. and/or/xor/add/sub/... $x, %y
  15781. test/or %y, %y | test $-1, %y (x)
  15782. j(n)z _Label
  15783. as the first instruction already adjusts the ZF
  15784. %y operand may also be a reference }
  15785. IsTestConstX:=(taicpu(p).opcode=A_TEST) and
  15786. MatchOperand(taicpu(p).oper[0]^,-1);
  15787. if (OpsEqual(taicpu(p).oper[0]^,taicpu(p).oper[1]^) or IsTestConstX) and
  15788. GetLastInstruction(p, hp1) and
  15789. (tai(hp1).typ = ait_instruction) and
  15790. GetNextInstruction(p,hp2) and
  15791. MatchInstruction(hp2,A_SETcc,A_Jcc,A_CMOVcc,[]) then
  15792. case taicpu(hp1).opcode Of
  15793. A_ADD, A_SUB, A_OR, A_XOR, A_AND,
  15794. { These two instructions set the zero flag if the result is zero }
  15795. A_POPCNT, A_LZCNT:
  15796. begin
  15797. if (
  15798. { With POPCNT, an input of zero will set the zero flag
  15799. because the population count of zero is zero }
  15800. (taicpu(hp1).opcode = A_POPCNT) and
  15801. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) and
  15802. (
  15803. OpsEqual(taicpu(hp1).oper[0]^, taicpu(p).oper[1]^) or
  15804. { Faster than going through the second half of the 'or'
  15805. condition below }
  15806. OpsEqual(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^)
  15807. )
  15808. ) or (
  15809. OpsEqual(taicpu(hp1).oper[1]^, taicpu(p).oper[1]^) and
  15810. { does not work in case of overflow for G(E)/L(E)/C_O/C_NO }
  15811. { and in case of carry for A(E)/B(E)/C/NC }
  15812. (
  15813. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) or
  15814. (
  15815. (taicpu(hp1).opcode <> A_ADD) and
  15816. (taicpu(hp1).opcode <> A_SUB) and
  15817. (taicpu(hp1).opcode <> A_LZCNT)
  15818. )
  15819. )
  15820. ) then
  15821. begin
  15822. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (2-op) done', hp1);
  15823. RemoveCurrentP(p, hp2);
  15824. Result:=true;
  15825. Exit;
  15826. end;
  15827. end;
  15828. A_SHL, A_SAL, A_SHR, A_SAR:
  15829. begin
  15830. if OpsEqual(taicpu(hp1).oper[1]^,taicpu(p).oper[1]^) and
  15831. { SHL/SAL/SHR/SAR with a value of 0 do not change the flags }
  15832. { therefore, it's only safe to do this optimization for }
  15833. { shifts by a (nonzero) constant }
  15834. (taicpu(hp1).oper[0]^.typ = top_const) and
  15835. (taicpu(hp1).oper[0]^.val <> 0) and
  15836. { does not work in case of overflow for G(E)/L(E)/C_O/C_NO }
  15837. { and in case of carry for A(E)/B(E)/C/NC }
  15838. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) then
  15839. begin
  15840. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (shift) done', hp1);
  15841. RemoveCurrentP(p, hp2);
  15842. Result:=true;
  15843. Exit;
  15844. end;
  15845. end;
  15846. A_DEC, A_INC, A_NEG:
  15847. begin
  15848. if OpsEqual(taicpu(hp1).oper[0]^,taicpu(p).oper[1]^) and
  15849. { does not work in case of overflow for G(E)/L(E)/C_O/C_NO }
  15850. { and in case of carry for A(E)/B(E)/C/NC }
  15851. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) then
  15852. begin
  15853. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (1-op) done', hp1);
  15854. RemoveCurrentP(p, hp2);
  15855. Result:=true;
  15856. Exit;
  15857. end;
  15858. end;
  15859. A_ANDN, A_BZHI:
  15860. begin
  15861. if OpsEqual(taicpu(hp1).oper[2]^,taicpu(p).oper[1]^) and
  15862. { Only the zero and sign flags are consistent with what the result is }
  15863. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE,C_S,C_NS]) then
  15864. begin
  15865. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (ANDN/BZHI) done', hp1);
  15866. RemoveCurrentP(p, hp2);
  15867. Result:=true;
  15868. Exit;
  15869. end;
  15870. end;
  15871. A_BEXTR:
  15872. begin
  15873. if OpsEqual(taicpu(hp1).oper[2]^,taicpu(p).oper[1]^) and
  15874. { Only the zero flag is set }
  15875. (taicpu(hp2).condition in [C_Z,C_NZ,C_E,C_NE]) then
  15876. begin
  15877. DebugMsg(SPeepholeOptimization + 'OpTest/Or2Op (BEXTR) done', hp1);
  15878. RemoveCurrentP(p, hp2);
  15879. Result:=true;
  15880. Exit;
  15881. end;
  15882. end;
  15883. else
  15884. ;
  15885. end; { case }
  15886. { change "test $-1,%reg" into "test %reg,%reg" }
  15887. if IsTestConstX and (taicpu(p).oper[1]^.typ=top_reg) then
  15888. taicpu(p).loadoper(0,taicpu(p).oper[1]^);
  15889. { Change "or %reg,%reg" to "test %reg,%reg" as OR generates a false dependency }
  15890. if MatchInstruction(p, A_OR, []) and
  15891. { Can only match if they're both registers }
  15892. MatchOperand(taicpu(p).oper[0]^, taicpu(p).oper[1]^) then
  15893. begin
  15894. DebugMsg(SPeepholeOptimization + 'or %reg,%reg -> test %reg,%reg to remove false dependency (Or2Test)', p);
  15895. taicpu(p).opcode := A_TEST;
  15896. { No need to set Result to True, as we've done all the optimisations we can }
  15897. end;
  15898. end;
  15899. function TX86AsmOptimizer.PostPeepholeOptCall(var p : tai) : Boolean;
  15900. var
  15901. hp1,hp3 : tai;
  15902. {$ifndef x86_64}
  15903. hp2 : taicpu;
  15904. {$endif x86_64}
  15905. begin
  15906. Result:=false;
  15907. hp3:=nil;
  15908. {$ifndef x86_64}
  15909. { don't do this on modern CPUs, this really hurts them due to
  15910. broken call/ret pairing }
  15911. if (current_settings.optimizecputype < cpu_Pentium2) and
  15912. not(cs_create_pic in current_settings.moduleswitches) and
  15913. GetNextInstruction(p, hp1) and
  15914. MatchInstruction(hp1,A_JMP,[S_NO]) and
  15915. MatchOpType(taicpu(hp1),top_ref) and
  15916. (taicpu(hp1).oper[0]^.ref^.refaddr=addr_full) then
  15917. begin
  15918. hp2 := taicpu.Op_sym(A_PUSH,S_L,taicpu(hp1).oper[0]^.ref^.symbol);
  15919. taicpu(hp2).fileinfo := taicpu(p).fileinfo;
  15920. InsertLLItem(p.previous, p, hp2);
  15921. taicpu(p).opcode := A_JMP;
  15922. taicpu(p).is_jmp := true;
  15923. RemoveInstruction(hp1);
  15924. Result:=true;
  15925. end
  15926. else
  15927. {$endif x86_64}
  15928. { replace
  15929. call procname
  15930. ret
  15931. by
  15932. jmp procname
  15933. but do it only on level 4 because it destroys stack back traces
  15934. else if the subroutine is marked as no return, remove the ret
  15935. }
  15936. if ((cs_opt_level4 in current_settings.optimizerswitches) or
  15937. (po_noreturn in current_procinfo.procdef.procoptions)) and
  15938. GetNextInstruction(p, hp1) and
  15939. (MatchInstruction(hp1,A_RET,[S_NO]) or
  15940. (MatchInstruction(hp1,A_VZEROUPPER,[S_NO]) and
  15941. SetAndTest(hp1,hp3) and
  15942. GetNextInstruction(hp1,hp1) and
  15943. MatchInstruction(hp1,A_RET,[S_NO])
  15944. )
  15945. ) and
  15946. (taicpu(hp1).ops=0) then
  15947. begin
  15948. if (cs_opt_level4 in current_settings.optimizerswitches) and
  15949. { we might destroy stack alignment here if we do not do a call }
  15950. (target_info.stackalign<=sizeof(SizeUInt)) then
  15951. begin
  15952. taicpu(p).opcode := A_JMP;
  15953. taicpu(p).is_jmp := true;
  15954. DebugMsg(SPeepholeOptimization + 'CallRet2Jmp done',p);
  15955. end
  15956. else
  15957. DebugMsg(SPeepholeOptimization + 'CallRet2Call done',p);
  15958. RemoveInstruction(hp1);
  15959. if Assigned(hp3) then
  15960. begin
  15961. AsmL.Remove(hp3);
  15962. AsmL.InsertBefore(hp3,p)
  15963. end;
  15964. Result:=true;
  15965. end;
  15966. end;
  15967. function TX86AsmOptimizer.PostPeepholeOptMovzx(var p : tai) : Boolean;
  15968. function ConstInRange(const Val: TCGInt; const OpSize: TOpSize): Boolean;
  15969. begin
  15970. case OpSize of
  15971. S_B, S_BW, S_BL{$ifdef x86_64}, S_BQ{$endif x86_64}:
  15972. Result := (Val <= $FF) and (Val >= -128);
  15973. S_W, S_WL{$ifdef x86_64}, S_WQ{$endif x86_64}:
  15974. Result := (Val <= $FFFF) and (Val >= -32768);
  15975. S_L{$ifdef x86_64}, S_LQ{$endif x86_64}:
  15976. Result := (Val <= $FFFFFFFF) and (Val >= -2147483648);
  15977. else
  15978. Result := True;
  15979. end;
  15980. end;
  15981. var
  15982. hp1, hp2 : tai;
  15983. SizeChange: Boolean;
  15984. PreMessage: string;
  15985. begin
  15986. Result := False;
  15987. if (taicpu(p).oper[0]^.typ = top_reg) and
  15988. SuperRegistersEqual(taicpu(p).oper[0]^.reg, taicpu(p).oper[1]^.reg) and
  15989. GetNextInstruction(p, hp1) and (hp1.typ = ait_instruction) then
  15990. begin
  15991. { Change (using movzbl %al,%eax as an example):
  15992. movzbl %al, %eax movzbl %al, %eax
  15993. cmpl x, %eax testl %eax,%eax
  15994. To:
  15995. cmpb x, %al testb %al, %al (Move one back to avoid a false dependency)
  15996. movzbl %al, %eax movzbl %al, %eax
  15997. Smaller instruction and minimises pipeline stall as the CPU
  15998. doesn't have to wait for the register to get zero-extended. [Kit]
  15999. Also allow if the smaller of the two registers is being checked,
  16000. as this still removes the false dependency.
  16001. }
  16002. if
  16003. (
  16004. (
  16005. (taicpu(hp1).opcode = A_CMP) and MatchOpType(taicpu(hp1), top_const, top_reg) and
  16006. ConstInRange(taicpu(hp1).oper[0]^.val, taicpu(p).opsize)
  16007. ) or (
  16008. { If MatchOperand returns True, they must both be registers }
  16009. (taicpu(hp1).opcode = A_TEST) and MatchOperand(taicpu(hp1).oper[0]^, taicpu(hp1).oper[1]^)
  16010. )
  16011. ) and
  16012. (reg2opsize(taicpu(hp1).oper[1]^.reg) <= reg2opsize(taicpu(p).oper[1]^.reg)) and
  16013. SuperRegistersEqual(taicpu(p).oper[1]^.reg, taicpu(hp1).oper[1]^.reg) then
  16014. begin
  16015. PreMessage := debug_op2str(taicpu(hp1).opcode) + debug_opsize2str(taicpu(hp1).opsize) + ' ' + debug_operstr(taicpu(hp1).oper[0]^) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' -> ' + debug_op2str(taicpu(hp1).opcode);
  16016. asml.Remove(hp1);
  16017. asml.InsertBefore(hp1, p);
  16018. { Swap instructions in the case of cmp 0,%reg or test %reg,%reg }
  16019. if (taicpu(hp1).opcode = A_TEST) or (taicpu(hp1).oper[0]^.val = 0) then
  16020. begin
  16021. taicpu(hp1).opcode := A_TEST;
  16022. taicpu(hp1).loadreg(0, taicpu(p).oper[0]^.reg);
  16023. end;
  16024. taicpu(hp1).oper[1]^.reg := taicpu(p).oper[0]^.reg;
  16025. case taicpu(p).opsize of
  16026. S_BW, S_BL:
  16027. begin
  16028. SizeChange := taicpu(hp1).opsize <> S_B;
  16029. taicpu(hp1).changeopsize(S_B);
  16030. end;
  16031. S_WL:
  16032. begin
  16033. SizeChange := taicpu(hp1).opsize <> S_W;
  16034. taicpu(hp1).changeopsize(S_W);
  16035. end
  16036. else
  16037. InternalError(2020112701);
  16038. end;
  16039. UpdateUsedRegs(tai(p.Next));
  16040. { Check if the register is used aferwards - if not, we can
  16041. remove the movzx instruction completely }
  16042. if not RegUsedAfterInstruction(taicpu(hp1).oper[1]^.reg, p, UsedRegs) then
  16043. begin
  16044. { Hp1 is a better position than p for debugging purposes }
  16045. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 4a', hp1);
  16046. RemoveCurrentp(p, hp1);
  16047. Result := True;
  16048. end;
  16049. if SizeChange then
  16050. DebugMsg(SPeepholeOptimization + PreMessage +
  16051. debug_opsize2str(taicpu(hp1).opsize) + ' ' + debug_operstr(taicpu(hp1).oper[0]^) + ',' + debug_regname(taicpu(hp1).oper[1]^.reg) + ' (smaller and minimises pipeline stall - MovzxCmp2CmpMovzx)', hp1)
  16052. else
  16053. DebugMsg(SPeepholeOptimization + 'MovzxCmp2CmpMovzx', hp1);
  16054. Exit;
  16055. end;
  16056. { Change (using movzwl %ax,%eax as an example):
  16057. movzwl %ax, %eax
  16058. movb %al, (dest) (Register is smaller than read register in movz)
  16059. To:
  16060. movb %al, (dest) (Move one back to avoid a false dependency)
  16061. movzwl %ax, %eax
  16062. }
  16063. if (taicpu(hp1).opcode = A_MOV) and
  16064. (taicpu(hp1).oper[0]^.typ = top_reg) and
  16065. not RegInOp(taicpu(hp1).oper[0]^.reg, taicpu(hp1).oper[1]^) and
  16066. SuperRegistersEqual(taicpu(hp1).oper[0]^.reg, taicpu(p).oper[0]^.reg) and
  16067. (reg2opsize(taicpu(hp1).oper[0]^.reg) <= reg2opsize(taicpu(p).oper[0]^.reg)) then
  16068. begin
  16069. DebugMsg(SPeepholeOptimization + 'MovzxMov2MovMovzx', hp1);
  16070. hp2 := tai(hp1.Previous); { Effectively the old position of hp1 }
  16071. asml.Remove(hp1);
  16072. asml.InsertBefore(hp1, p);
  16073. if taicpu(hp1).oper[1]^.typ = top_reg then
  16074. AllocRegBetween(taicpu(hp1).oper[1]^.reg, hp1, hp2, UsedRegs);
  16075. { Check if the register is used aferwards - if not, we can
  16076. remove the movzx instruction completely }
  16077. if not RegUsedAfterInstruction(taicpu(hp1).oper[0]^.reg, p, UsedRegs) then
  16078. begin
  16079. { Hp1 is a better position than p for debugging purposes }
  16080. DebugMsg(SPeepholeOptimization + 'Movzx2Nop 4b', hp1);
  16081. RemoveCurrentp(p, hp1);
  16082. Result := True;
  16083. end;
  16084. Exit;
  16085. end;
  16086. end;
  16087. end;
  16088. function TX86AsmOptimizer.PostPeepholeOptXor(var p : tai) : Boolean;
  16089. var
  16090. hp1: tai;
  16091. {$ifdef x86_64}
  16092. PreMessage, RegName: string;
  16093. {$endif x86_64}
  16094. begin
  16095. Result := False;
  16096. { If x is a power of 2 (popcnt = 1), change:
  16097. xor $x, %reg/ref
  16098. To:
  16099. btc lb(x), %reg/ref
  16100. }
  16101. if IsBTXAcceptable(p) and
  16102. { IsBTXAcceptable checks to see if oper[0] is an immediate }
  16103. (PopCnt(QWord(taicpu(p).oper[0]^.val)) = 1) and
  16104. (
  16105. { Don't optimise if a test instruction follows }
  16106. not GetNextInstruction(p, hp1) or
  16107. not MatchInstruction(hp1, A_TEST, [taicpu(p).opsize])
  16108. ) then
  16109. begin
  16110. DebugMsg(SPeepholeOptimization + 'Changed XOR $' + debug_hexstr(taicpu(p).oper[0]^.val) + ' to BTC $' + debug_tostr(BsrQWord(taicpu(p).oper[0]^.val)) + ' to shrink instruction size (Xor2Btc)', p);
  16111. taicpu(p).opcode := A_BTC;
  16112. taicpu(p).oper[0]^.val := BsrQWord(taicpu(p).oper[0]^.val); { Essentially the base 2 logarithm }
  16113. Result := True;
  16114. Exit;
  16115. end;
  16116. {$ifdef x86_64}
  16117. { Code size reduction by J. Gareth "Kit" Moreton }
  16118. { change "xorq %reg,%reg" to "xorl %reg,%reg" for %rax, %rcx, %rdx, %rbx, %rsi, %rdi, %rbp and %rsp,
  16119. as this removes the REX prefix }
  16120. if not OpsEqual(taicpu(p).oper[0]^,taicpu(p).oper[1]^) then
  16121. Exit;
  16122. if taicpu(p).oper[0]^.typ <> top_reg then
  16123. { Should be impossible if both operands were equal, since one of XOR's operands must be a register }
  16124. InternalError(2018011500);
  16125. case taicpu(p).opsize of
  16126. S_Q:
  16127. begin
  16128. RegName := debug_regname(taicpu(p).oper[0]^.reg); { 64-bit register name }
  16129. PreMessage := 'xorq ' + RegName + ',' + RegName + ' -> xorl ';
  16130. { The actual optimization }
  16131. setsubreg(taicpu(p).oper[0]^.reg, R_SUBD);
  16132. setsubreg(taicpu(p).oper[1]^.reg, R_SUBD);
  16133. taicpu(p).changeopsize(S_L);
  16134. RegName := debug_regname(taicpu(p).oper[0]^.reg); { 32-bit register name }
  16135. DebugMsg(SPeepholeOptimization + PreMessage + RegName + ',' + RegName + ' (32-bit register recommended when zeroing 64-bit counterpart)', p);
  16136. end;
  16137. else
  16138. ;
  16139. end;
  16140. {$endif x86_64}
  16141. end;
  16142. function TX86AsmOptimizer.PostPeepholeOptVPXOR(var p : tai) : Boolean;
  16143. var
  16144. XReg: TRegister;
  16145. begin
  16146. Result := False;
  16147. { Turn "vpxor %ymmreg2,%ymmreg2,%ymmreg1" to "vpxor %xmmreg2,%xmmreg2,%xmmreg1"
  16148. Smaller encoding and slightly faster on some platforms (also works for
  16149. ZMM-sized registers) }
  16150. if (taicpu(p).opsize in [S_YMM, S_ZMM]) and
  16151. MatchOpType(taicpu(p), top_reg, top_reg, top_reg) then
  16152. begin
  16153. XReg := taicpu(p).oper[0]^.reg;
  16154. if (taicpu(p).oper[1]^.reg = XReg) then
  16155. begin
  16156. taicpu(p).changeopsize(S_XMM);
  16157. setsubreg(taicpu(p).oper[2]^.reg, R_SUBMMX);
  16158. if (cs_opt_size in current_settings.optimizerswitches) then
  16159. begin
  16160. { Change input registers to %xmm0 to reduce size. Note that
  16161. there's a risk of a false dependency doing this, so only
  16162. optimise for size here }
  16163. XReg := NR_XMM0;
  16164. DebugMsg(SPeepholeOptimization + 'Changed zero-setting vpxor from Y/ZMM to XMM and changed input registers to %xmm0 to reduce size', p);
  16165. end
  16166. else
  16167. begin
  16168. setsubreg(XReg, R_SUBMMX);
  16169. DebugMsg(SPeepholeOptimization + 'Changed zero-setting vpxor from Y/ZMM to XMM to reduce size and increase efficiency', p);
  16170. end;
  16171. taicpu(p).oper[0]^.reg := XReg;
  16172. taicpu(p).oper[1]^.reg := XReg;
  16173. Result := True;
  16174. end;
  16175. end;
  16176. end;
  16177. function TX86AsmOptimizer.PostPeepholeOptRET(var p: tai): Boolean;
  16178. var
  16179. hp1, p_new: tai;
  16180. begin
  16181. Result := False;
  16182. { Check for:
  16183. ret
  16184. .Lbl:
  16185. ret
  16186. Remove first 'ret'
  16187. }
  16188. if GetNextInstruction(p, hp1) and
  16189. { Remember where the label is }
  16190. SetAndTest(hp1, p_new) and
  16191. (hp1.typ in [ait_align, ait_label]) and
  16192. SkipLabels(hp1, hp1) and
  16193. MatchInstruction(hp1, A_RET, []) and
  16194. { To be safe, make sure the RET instructions are identical }
  16195. (taicpu(p).ops = taicpu(hp1).ops) and
  16196. (
  16197. (taicpu(p).ops = 0) or
  16198. (
  16199. (taicpu(p).ops = 1) and
  16200. MatchOperand(taicpu(p).oper[0]^, taicpu(hp1).oper[0]^)
  16201. )
  16202. ) then
  16203. begin
  16204. DebugMsg(SPeepholeOptimization + 'Removed superfluous RET', p);
  16205. UpdateUsedRegs(tai(p.Next));
  16206. RemoveCurrentP(p, p_new);
  16207. Result := True;
  16208. Exit;
  16209. end;
  16210. end;
  16211. class procedure TX86AsmOptimizer.OptimizeRefs(var p: taicpu);
  16212. var
  16213. OperIdx: Integer;
  16214. begin
  16215. for OperIdx := 0 to p.ops - 1 do
  16216. if p.oper[OperIdx]^.typ = top_ref then
  16217. optimize_ref(p.oper[OperIdx]^.ref^, False);
  16218. end;
  16219. end.