BeMCContext.cpp 526 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825108261082710828108291083010831108321083310834108351083610837108381083910840108411084210843108441084510846108471084810849108501085110852108531085410855108561085710858108591086010861108621086310864108651086610867108681086910870108711087210873108741087510876108771087810879108801088110882108831088410885108861088710888108891089010891108921089310894108951089610897108981089910900109011090210903109041090510906109071090810909109101091110912109131091410915109161091710918109191092010921109221092310924109251092610927109281092910930109311093210933109341093510936109371093810939109401094110942109431094410945109461094710948109491095010951109521095310954109551095610957109581095910960109611096210963109641096510966109671096810969109701097110972109731097410975109761097710978109791098010981109821098310984109851098610987109881098910990109911099210993109941099510996109971099810999110001100111002110031100411005110061100711008110091101011011110121101311014110151101611017110181101911020110211102211023110241102511026110271102811029110301103111032110331103411035110361103711038110391104011041110421104311044110451104611047110481104911050110511105211053110541105511056110571105811059110601106111062110631106411065110661106711068110691107011071110721107311074110751107611077110781107911080110811108211083110841108511086110871108811089110901109111092110931109411095110961109711098110991110011101111021110311104111051110611107111081110911110111111111211113111141111511116111171111811119111201112111122111231112411125111261112711128111291113011131111321113311134111351113611137111381113911140111411114211143111441114511146111471114811149111501115111152111531115411155111561115711158111591116011161111621116311164111651116611167111681116911170111711117211173111741117511176111771117811179111801118111182111831118411185111861118711188111891119011191111921119311194111951119611197111981119911200112011120211203112041120511206112071120811209112101121111212112131121411215112161121711218112191122011221112221122311224112251122611227112281122911230112311123211233112341123511236112371123811239112401124111242112431124411245112461124711248112491125011251112521125311254112551125611257112581125911260112611126211263112641126511266112671126811269112701127111272112731127411275112761127711278112791128011281112821128311284112851128611287112881128911290112911129211293112941129511296112971129811299113001130111302113031130411305113061130711308113091131011311113121131311314113151131611317113181131911320113211132211323113241132511326113271132811329113301133111332113331133411335113361133711338113391134011341113421134311344113451134611347113481134911350113511135211353113541135511356113571135811359113601136111362113631136411365113661136711368113691137011371113721137311374113751137611377113781137911380113811138211383113841138511386113871138811389113901139111392113931139411395113961139711398113991140011401114021140311404114051140611407114081140911410114111141211413114141141511416114171141811419114201142111422114231142411425114261142711428114291143011431114321143311434114351143611437114381143911440114411144211443114441144511446114471144811449114501145111452114531145411455114561145711458114591146011461114621146311464114651146611467114681146911470114711147211473114741147511476114771147811479114801148111482114831148411485114861148711488114891149011491114921149311494114951149611497114981149911500115011150211503115041150511506115071150811509115101151111512115131151411515115161151711518115191152011521115221152311524115251152611527115281152911530115311153211533115341153511536115371153811539115401154111542115431154411545115461154711548115491155011551115521155311554115551155611557115581155911560115611156211563115641156511566115671156811569115701157111572115731157411575115761157711578115791158011581115821158311584115851158611587115881158911590115911159211593115941159511596115971159811599116001160111602116031160411605116061160711608116091161011611116121161311614116151161611617116181161911620116211162211623116241162511626116271162811629116301163111632116331163411635116361163711638116391164011641116421164311644116451164611647116481164911650116511165211653116541165511656116571165811659116601166111662116631166411665116661166711668116691167011671116721167311674116751167611677116781167911680116811168211683116841168511686116871168811689116901169111692116931169411695116961169711698116991170011701117021170311704117051170611707117081170911710117111171211713117141171511716117171171811719117201172111722117231172411725117261172711728117291173011731117321173311734117351173611737117381173911740117411174211743117441174511746117471174811749117501175111752117531175411755117561175711758117591176011761117621176311764117651176611767117681176911770117711177211773117741177511776117771177811779117801178111782117831178411785117861178711788117891179011791117921179311794117951179611797117981179911800118011180211803118041180511806118071180811809118101181111812118131181411815118161181711818118191182011821118221182311824118251182611827118281182911830118311183211833118341183511836118371183811839118401184111842118431184411845118461184711848118491185011851118521185311854118551185611857118581185911860118611186211863118641186511866118671186811869118701187111872118731187411875118761187711878118791188011881118821188311884118851188611887118881188911890118911189211893118941189511896118971189811899119001190111902119031190411905119061190711908119091191011911119121191311914119151191611917119181191911920119211192211923119241192511926119271192811929119301193111932119331193411935119361193711938119391194011941119421194311944119451194611947119481194911950119511195211953119541195511956119571195811959119601196111962119631196411965119661196711968119691197011971119721197311974119751197611977119781197911980119811198211983119841198511986119871198811989119901199111992119931199411995119961199711998119991200012001120021200312004120051200612007120081200912010120111201212013120141201512016120171201812019120201202112022120231202412025120261202712028120291203012031120321203312034120351203612037120381203912040120411204212043120441204512046120471204812049120501205112052120531205412055120561205712058120591206012061120621206312064120651206612067120681206912070120711207212073120741207512076120771207812079120801208112082120831208412085120861208712088120891209012091120921209312094120951209612097120981209912100121011210212103121041210512106121071210812109121101211112112121131211412115121161211712118121191212012121121221212312124121251212612127121281212912130121311213212133121341213512136121371213812139121401214112142121431214412145121461214712148121491215012151121521215312154121551215612157121581215912160121611216212163121641216512166121671216812169121701217112172121731217412175121761217712178121791218012181121821218312184121851218612187121881218912190121911219212193121941219512196121971219812199122001220112202122031220412205122061220712208122091221012211122121221312214122151221612217122181221912220122211222212223122241222512226122271222812229122301223112232122331223412235122361223712238122391224012241122421224312244122451224612247122481224912250122511225212253122541225512256122571225812259122601226112262122631226412265122661226712268122691227012271122721227312274122751227612277122781227912280122811228212283122841228512286122871228812289122901229112292122931229412295122961229712298122991230012301123021230312304123051230612307123081230912310123111231212313123141231512316123171231812319123201232112322123231232412325123261232712328123291233012331123321233312334123351233612337123381233912340123411234212343123441234512346123471234812349123501235112352123531235412355123561235712358123591236012361123621236312364123651236612367123681236912370123711237212373123741237512376123771237812379123801238112382123831238412385123861238712388123891239012391123921239312394123951239612397123981239912400124011240212403124041240512406124071240812409124101241112412124131241412415124161241712418124191242012421124221242312424124251242612427124281242912430124311243212433124341243512436124371243812439124401244112442124431244412445124461244712448124491245012451124521245312454124551245612457124581245912460124611246212463124641246512466124671246812469124701247112472124731247412475124761247712478124791248012481124821248312484124851248612487124881248912490124911249212493124941249512496124971249812499125001250112502125031250412505125061250712508125091251012511125121251312514125151251612517125181251912520125211252212523125241252512526125271252812529125301253112532125331253412535125361253712538125391254012541125421254312544125451254612547125481254912550125511255212553125541255512556125571255812559125601256112562125631256412565125661256712568125691257012571125721257312574125751257612577125781257912580125811258212583125841258512586125871258812589125901259112592125931259412595125961259712598125991260012601126021260312604126051260612607126081260912610126111261212613126141261512616126171261812619126201262112622126231262412625126261262712628126291263012631126321263312634126351263612637126381263912640126411264212643126441264512646126471264812649126501265112652126531265412655126561265712658126591266012661126621266312664126651266612667126681266912670126711267212673126741267512676126771267812679126801268112682126831268412685126861268712688126891269012691126921269312694126951269612697126981269912700127011270212703127041270512706127071270812709127101271112712127131271412715127161271712718127191272012721127221272312724127251272612727127281272912730127311273212733127341273512736127371273812739127401274112742127431274412745127461274712748127491275012751127521275312754127551275612757127581275912760127611276212763127641276512766127671276812769127701277112772127731277412775127761277712778127791278012781127821278312784127851278612787127881278912790127911279212793127941279512796127971279812799128001280112802128031280412805128061280712808128091281012811128121281312814128151281612817128181281912820128211282212823128241282512826128271282812829128301283112832128331283412835128361283712838128391284012841128421284312844128451284612847128481284912850128511285212853128541285512856128571285812859128601286112862128631286412865128661286712868128691287012871128721287312874128751287612877128781287912880128811288212883128841288512886128871288812889128901289112892128931289412895128961289712898128991290012901129021290312904129051290612907129081290912910129111291212913129141291512916129171291812919129201292112922129231292412925129261292712928129291293012931129321293312934129351293612937129381293912940129411294212943129441294512946129471294812949129501295112952129531295412955129561295712958129591296012961129621296312964129651296612967129681296912970129711297212973129741297512976129771297812979129801298112982129831298412985129861298712988129891299012991129921299312994129951299612997129981299913000130011300213003130041300513006130071300813009130101301113012130131301413015130161301713018130191302013021130221302313024130251302613027130281302913030130311303213033130341303513036130371303813039130401304113042130431304413045130461304713048130491305013051130521305313054130551305613057130581305913060130611306213063130641306513066130671306813069130701307113072130731307413075130761307713078130791308013081130821308313084130851308613087130881308913090130911309213093130941309513096130971309813099131001310113102131031310413105131061310713108131091311013111131121311313114131151311613117131181311913120131211312213123131241312513126131271312813129131301313113132131331313413135131361313713138131391314013141131421314313144131451314613147131481314913150131511315213153131541315513156131571315813159131601316113162131631316413165131661316713168131691317013171131721317313174131751317613177131781317913180131811318213183131841318513186131871318813189131901319113192131931319413195131961319713198131991320013201132021320313204132051320613207132081320913210132111321213213132141321513216132171321813219132201322113222132231322413225132261322713228132291323013231132321323313234132351323613237132381323913240132411324213243132441324513246132471324813249132501325113252132531325413255132561325713258132591326013261132621326313264132651326613267132681326913270132711327213273132741327513276132771327813279132801328113282132831328413285132861328713288132891329013291132921329313294132951329613297132981329913300133011330213303133041330513306133071330813309133101331113312133131331413315133161331713318133191332013321133221332313324133251332613327133281332913330133311333213333133341333513336133371333813339133401334113342133431334413345133461334713348133491335013351133521335313354133551335613357133581335913360133611336213363133641336513366133671336813369133701337113372133731337413375133761337713378133791338013381133821338313384133851338613387133881338913390133911339213393133941339513396133971339813399134001340113402134031340413405134061340713408134091341013411134121341313414134151341613417134181341913420134211342213423134241342513426134271342813429134301343113432134331343413435134361343713438134391344013441134421344313444134451344613447134481344913450134511345213453134541345513456134571345813459134601346113462134631346413465134661346713468134691347013471134721347313474134751347613477134781347913480134811348213483134841348513486134871348813489134901349113492134931349413495134961349713498134991350013501135021350313504135051350613507135081350913510135111351213513135141351513516135171351813519135201352113522135231352413525135261352713528135291353013531135321353313534135351353613537135381353913540135411354213543135441354513546135471354813549135501355113552135531355413555135561355713558135591356013561135621356313564135651356613567135681356913570135711357213573135741357513576135771357813579135801358113582135831358413585135861358713588135891359013591135921359313594135951359613597135981359913600136011360213603136041360513606136071360813609136101361113612136131361413615136161361713618136191362013621136221362313624136251362613627136281362913630136311363213633136341363513636136371363813639136401364113642136431364413645136461364713648136491365013651136521365313654136551365613657136581365913660136611366213663136641366513666136671366813669136701367113672136731367413675136761367713678136791368013681136821368313684136851368613687136881368913690136911369213693136941369513696136971369813699137001370113702137031370413705137061370713708137091371013711137121371313714137151371613717137181371913720137211372213723137241372513726137271372813729137301373113732137331373413735137361373713738137391374013741137421374313744137451374613747137481374913750137511375213753137541375513756137571375813759137601376113762137631376413765137661376713768137691377013771137721377313774137751377613777137781377913780137811378213783137841378513786137871378813789137901379113792137931379413795137961379713798137991380013801138021380313804138051380613807138081380913810138111381213813138141381513816138171381813819138201382113822138231382413825138261382713828138291383013831138321383313834138351383613837138381383913840138411384213843138441384513846138471384813849138501385113852138531385413855138561385713858138591386013861138621386313864138651386613867138681386913870138711387213873138741387513876138771387813879138801388113882138831388413885138861388713888138891389013891138921389313894138951389613897138981389913900139011390213903139041390513906139071390813909139101391113912139131391413915139161391713918139191392013921139221392313924139251392613927139281392913930139311393213933139341393513936139371393813939139401394113942139431394413945139461394713948139491395013951139521395313954139551395613957139581395913960139611396213963139641396513966139671396813969139701397113972139731397413975139761397713978139791398013981139821398313984139851398613987139881398913990139911399213993139941399513996139971399813999140001400114002140031400414005140061400714008140091401014011140121401314014140151401614017140181401914020140211402214023140241402514026140271402814029140301403114032140331403414035140361403714038140391404014041140421404314044140451404614047140481404914050140511405214053140541405514056140571405814059140601406114062140631406414065140661406714068140691407014071140721407314074140751407614077140781407914080140811408214083140841408514086140871408814089140901409114092140931409414095140961409714098140991410014101141021410314104141051410614107141081410914110141111411214113141141411514116141171411814119141201412114122141231412414125141261412714128141291413014131141321413314134141351413614137141381413914140141411414214143141441414514146141471414814149141501415114152141531415414155141561415714158141591416014161141621416314164141651416614167141681416914170141711417214173141741417514176141771417814179141801418114182141831418414185141861418714188141891419014191141921419314194141951419614197141981419914200142011420214203142041420514206142071420814209142101421114212142131421414215142161421714218142191422014221142221422314224142251422614227142281422914230142311423214233142341423514236142371423814239142401424114242142431424414245142461424714248142491425014251142521425314254142551425614257142581425914260142611426214263142641426514266142671426814269142701427114272142731427414275142761427714278142791428014281142821428314284142851428614287142881428914290142911429214293142941429514296142971429814299143001430114302143031430414305143061430714308143091431014311143121431314314143151431614317143181431914320143211432214323143241432514326143271432814329143301433114332143331433414335143361433714338143391434014341143421434314344143451434614347143481434914350143511435214353143541435514356143571435814359143601436114362143631436414365143661436714368143691437014371143721437314374143751437614377143781437914380143811438214383143841438514386143871438814389143901439114392143931439414395143961439714398143991440014401144021440314404144051440614407144081440914410144111441214413144141441514416144171441814419144201442114422144231442414425144261442714428144291443014431144321443314434144351443614437144381443914440144411444214443144441444514446144471444814449144501445114452144531445414455144561445714458144591446014461144621446314464144651446614467144681446914470144711447214473144741447514476144771447814479144801448114482144831448414485144861448714488144891449014491144921449314494144951449614497144981449914500145011450214503145041450514506145071450814509145101451114512145131451414515145161451714518145191452014521145221452314524145251452614527145281452914530145311453214533145341453514536145371453814539145401454114542145431454414545145461454714548145491455014551145521455314554145551455614557145581455914560145611456214563145641456514566145671456814569145701457114572145731457414575145761457714578145791458014581145821458314584145851458614587145881458914590145911459214593145941459514596145971459814599146001460114602146031460414605146061460714608146091461014611146121461314614146151461614617146181461914620146211462214623146241462514626146271462814629146301463114632146331463414635146361463714638146391464014641146421464314644146451464614647146481464914650146511465214653146541465514656146571465814659146601466114662146631466414665146661466714668146691467014671146721467314674146751467614677146781467914680146811468214683146841468514686146871468814689146901469114692146931469414695146961469714698146991470014701147021470314704147051470614707147081470914710147111471214713147141471514716147171471814719147201472114722147231472414725147261472714728147291473014731147321473314734147351473614737147381473914740147411474214743147441474514746147471474814749147501475114752147531475414755147561475714758147591476014761147621476314764147651476614767147681476914770147711477214773147741477514776147771477814779147801478114782147831478414785147861478714788147891479014791147921479314794147951479614797147981479914800148011480214803148041480514806148071480814809148101481114812148131481414815148161481714818148191482014821148221482314824148251482614827148281482914830148311483214833148341483514836148371483814839148401484114842148431484414845148461484714848148491485014851148521485314854148551485614857148581485914860148611486214863148641486514866148671486814869148701487114872148731487414875148761487714878148791488014881148821488314884148851488614887148881488914890148911489214893148941489514896148971489814899149001490114902149031490414905149061490714908149091491014911149121491314914149151491614917149181491914920149211492214923149241492514926149271492814929149301493114932149331493414935149361493714938149391494014941149421494314944149451494614947149481494914950149511495214953149541495514956149571495814959149601496114962149631496414965149661496714968149691497014971149721497314974149751497614977149781497914980149811498214983149841498514986149871498814989149901499114992149931499414995149961499714998149991500015001150021500315004150051500615007150081500915010150111501215013150141501515016150171501815019150201502115022150231502415025150261502715028150291503015031150321503315034150351503615037150381503915040150411504215043150441504515046150471504815049150501505115052150531505415055150561505715058150591506015061150621506315064150651506615067150681506915070150711507215073150741507515076150771507815079150801508115082150831508415085150861508715088150891509015091150921509315094150951509615097150981509915100151011510215103151041510515106151071510815109151101511115112151131511415115151161511715118151191512015121151221512315124151251512615127151281512915130151311513215133151341513515136151371513815139151401514115142151431514415145151461514715148151491515015151151521515315154151551515615157151581515915160151611516215163151641516515166151671516815169151701517115172151731517415175151761517715178151791518015181151821518315184151851518615187151881518915190151911519215193151941519515196151971519815199152001520115202152031520415205152061520715208152091521015211152121521315214152151521615217152181521915220152211522215223152241522515226152271522815229152301523115232152331523415235152361523715238152391524015241152421524315244152451524615247152481524915250152511525215253152541525515256152571525815259152601526115262152631526415265152661526715268152691527015271152721527315274152751527615277152781527915280152811528215283152841528515286152871528815289152901529115292152931529415295152961529715298152991530015301153021530315304153051530615307153081530915310153111531215313153141531515316153171531815319153201532115322153231532415325153261532715328153291533015331153321533315334153351533615337153381533915340153411534215343153441534515346153471534815349153501535115352153531535415355153561535715358153591536015361153621536315364153651536615367153681536915370153711537215373153741537515376153771537815379153801538115382153831538415385153861538715388153891539015391153921539315394153951539615397153981539915400154011540215403154041540515406154071540815409154101541115412154131541415415154161541715418154191542015421154221542315424154251542615427154281542915430154311543215433154341543515436154371543815439154401544115442154431544415445154461544715448154491545015451154521545315454154551545615457154581545915460154611546215463154641546515466154671546815469154701547115472154731547415475154761547715478154791548015481154821548315484154851548615487154881548915490154911549215493154941549515496154971549815499155001550115502155031550415505155061550715508155091551015511155121551315514155151551615517155181551915520155211552215523155241552515526155271552815529155301553115532155331553415535155361553715538155391554015541155421554315544155451554615547155481554915550155511555215553155541555515556155571555815559155601556115562155631556415565155661556715568155691557015571155721557315574155751557615577155781557915580155811558215583155841558515586155871558815589155901559115592155931559415595155961559715598155991560015601156021560315604156051560615607156081560915610156111561215613156141561515616156171561815619156201562115622156231562415625156261562715628156291563015631156321563315634156351563615637156381563915640156411564215643156441564515646156471564815649156501565115652156531565415655156561565715658156591566015661156621566315664156651566615667156681566915670156711567215673156741567515676156771567815679156801568115682156831568415685156861568715688156891569015691156921569315694156951569615697156981569915700157011570215703157041570515706157071570815709157101571115712157131571415715157161571715718157191572015721157221572315724157251572615727157281572915730157311573215733157341573515736157371573815739157401574115742157431574415745157461574715748157491575015751157521575315754157551575615757157581575915760157611576215763157641576515766157671576815769157701577115772157731577415775157761577715778157791578015781157821578315784157851578615787157881578915790157911579215793157941579515796157971579815799158001580115802158031580415805158061580715808158091581015811158121581315814158151581615817158181581915820158211582215823158241582515826158271582815829158301583115832158331583415835158361583715838158391584015841158421584315844158451584615847158481584915850158511585215853158541585515856158571585815859158601586115862158631586415865158661586715868158691587015871158721587315874158751587615877158781587915880158811588215883158841588515886158871588815889158901589115892158931589415895158961589715898158991590015901159021590315904159051590615907159081590915910159111591215913159141591515916159171591815919159201592115922159231592415925159261592715928159291593015931159321593315934159351593615937159381593915940159411594215943159441594515946159471594815949159501595115952159531595415955159561595715958159591596015961159621596315964159651596615967159681596915970159711597215973159741597515976159771597815979159801598115982159831598415985159861598715988159891599015991159921599315994159951599615997159981599916000160011600216003160041600516006160071600816009160101601116012160131601416015160161601716018160191602016021160221602316024160251602616027160281602916030160311603216033160341603516036160371603816039160401604116042160431604416045160461604716048160491605016051160521605316054160551605616057160581605916060160611606216063160641606516066160671606816069160701607116072160731607416075160761607716078160791608016081160821608316084160851608616087160881608916090160911609216093160941609516096160971609816099161001610116102161031610416105161061610716108161091611016111161121611316114161151611616117161181611916120161211612216123161241612516126161271612816129161301613116132161331613416135161361613716138161391614016141161421614316144161451614616147161481614916150161511615216153161541615516156161571615816159161601616116162161631616416165161661616716168161691617016171161721617316174161751617616177161781617916180161811618216183161841618516186161871618816189161901619116192161931619416195161961619716198161991620016201162021620316204162051620616207162081620916210162111621216213162141621516216162171621816219162201622116222162231622416225162261622716228162291623016231162321623316234162351623616237162381623916240162411624216243162441624516246162471624816249162501625116252162531625416255162561625716258162591626016261162621626316264162651626616267162681626916270162711627216273162741627516276162771627816279162801628116282162831628416285162861628716288162891629016291162921629316294162951629616297162981629916300163011630216303163041630516306163071630816309163101631116312163131631416315163161631716318163191632016321163221632316324163251632616327163281632916330163311633216333163341633516336163371633816339163401634116342163431634416345163461634716348163491635016351163521635316354163551635616357163581635916360163611636216363163641636516366163671636816369163701637116372163731637416375163761637716378163791638016381163821638316384163851638616387163881638916390163911639216393163941639516396163971639816399164001640116402164031640416405164061640716408164091641016411164121641316414164151641616417164181641916420164211642216423164241642516426164271642816429164301643116432164331643416435164361643716438164391644016441164421644316444164451644616447164481644916450164511645216453164541645516456164571645816459164601646116462164631646416465164661646716468164691647016471164721647316474164751647616477164781647916480164811648216483164841648516486164871648816489164901649116492164931649416495164961649716498164991650016501165021650316504165051650616507165081650916510165111651216513165141651516516165171651816519165201652116522165231652416525165261652716528165291653016531165321653316534165351653616537165381653916540165411654216543165441654516546165471654816549165501655116552165531655416555165561655716558165591656016561165621656316564165651656616567165681656916570165711657216573165741657516576165771657816579165801658116582165831658416585165861658716588165891659016591165921659316594165951659616597165981659916600166011660216603166041660516606166071660816609166101661116612166131661416615166161661716618166191662016621166221662316624166251662616627166281662916630166311663216633166341663516636166371663816639166401664116642166431664416645166461664716648166491665016651166521665316654166551665616657166581665916660166611666216663166641666516666166671666816669166701667116672166731667416675166761667716678166791668016681166821668316684166851668616687166881668916690166911669216693166941669516696166971669816699167001670116702167031670416705167061670716708167091671016711167121671316714167151671616717167181671916720167211672216723167241672516726167271672816729167301673116732167331673416735167361673716738167391674016741167421674316744167451674616747167481674916750167511675216753167541675516756167571675816759167601676116762167631676416765167661676716768167691677016771167721677316774167751677616777167781677916780167811678216783167841678516786167871678816789167901679116792167931679416795167961679716798167991680016801168021680316804168051680616807168081680916810168111681216813168141681516816168171681816819168201682116822168231682416825168261682716828168291683016831168321683316834168351683616837168381683916840168411684216843168441684516846168471684816849168501685116852168531685416855168561685716858168591686016861168621686316864168651686616867168681686916870168711687216873168741687516876168771687816879168801688116882168831688416885168861688716888168891689016891168921689316894168951689616897168981689916900169011690216903169041690516906169071690816909169101691116912169131691416915169161691716918169191692016921169221692316924169251692616927169281692916930169311693216933169341693516936169371693816939169401694116942169431694416945169461694716948169491695016951169521695316954169551695616957169581695916960169611696216963169641696516966169671696816969169701697116972169731697416975169761697716978169791698016981169821698316984169851698616987169881698916990169911699216993169941699516996169971699816999170001700117002170031700417005170061700717008170091701017011170121701317014170151701617017170181701917020170211702217023170241702517026170271702817029170301703117032170331703417035170361703717038170391704017041170421704317044170451704617047170481704917050170511705217053170541705517056170571705817059170601706117062170631706417065170661706717068170691707017071170721707317074170751707617077170781707917080170811708217083170841708517086170871708817089170901709117092170931709417095170961709717098170991710017101171021710317104171051710617107171081710917110171111711217113171141711517116171171711817119171201712117122171231712417125171261712717128171291713017131171321713317134171351713617137171381713917140171411714217143171441714517146171471714817149171501715117152171531715417155171561715717158171591716017161171621716317164171651716617167171681716917170171711717217173171741717517176171771717817179171801718117182171831718417185171861718717188171891719017191171921719317194171951719617197171981719917200172011720217203172041720517206172071720817209172101721117212172131721417215172161721717218172191722017221172221722317224172251722617227172281722917230172311723217233172341723517236172371723817239172401724117242172431724417245172461724717248172491725017251172521725317254172551725617257172581725917260172611726217263172641726517266172671726817269172701727117272172731727417275172761727717278172791728017281172821728317284172851728617287172881728917290172911729217293172941729517296172971729817299173001730117302173031730417305173061730717308173091731017311173121731317314173151731617317173181731917320173211732217323173241732517326173271732817329173301733117332173331733417335173361733717338173391734017341173421734317344173451734617347173481734917350173511735217353173541735517356173571735817359173601736117362173631736417365173661736717368173691737017371173721737317374173751737617377173781737917380173811738217383173841738517386173871738817389173901739117392173931739417395173961739717398173991740017401174021740317404174051740617407174081740917410174111741217413174141741517416174171741817419174201742117422174231742417425174261742717428174291743017431174321743317434174351743617437174381743917440174411744217443174441744517446174471744817449174501745117452174531745417455174561745717458174591746017461174621746317464174651746617467174681746917470174711747217473174741747517476174771747817479174801748117482174831748417485174861748717488174891749017491174921749317494174951749617497174981749917500175011750217503175041750517506175071750817509175101751117512175131751417515175161751717518175191752017521175221752317524175251752617527175281752917530175311753217533175341753517536175371753817539175401754117542175431754417545175461754717548175491755017551175521755317554175551755617557175581755917560175611756217563175641756517566175671756817569175701757117572175731757417575175761757717578175791758017581175821758317584175851758617587175881758917590175911759217593175941759517596175971759817599176001760117602176031760417605176061760717608176091761017611176121761317614176151761617617176181761917620176211762217623176241762517626176271762817629176301763117632176331763417635176361763717638176391764017641176421764317644176451764617647176481764917650176511765217653176541765517656176571765817659176601766117662176631766417665176661766717668176691767017671176721767317674176751767617677176781767917680176811768217683176841768517686176871768817689176901769117692176931769417695176961769717698176991770017701177021770317704177051770617707177081770917710177111771217713177141771517716177171771817719177201772117722177231772417725177261772717728177291773017731177321773317734177351773617737177381773917740177411774217743177441774517746177471774817749177501775117752177531775417755177561775717758177591776017761177621776317764177651776617767177681776917770177711777217773177741777517776177771777817779177801778117782177831778417785177861778717788177891779017791177921779317794177951779617797177981779917800178011780217803178041780517806178071780817809178101781117812178131781417815178161781717818178191782017821178221782317824178251782617827178281782917830178311783217833178341783517836178371783817839178401784117842178431784417845178461784717848178491785017851178521785317854178551785617857178581785917860178611786217863178641786517866178671786817869178701787117872178731787417875178761787717878178791788017881178821788317884178851788617887178881788917890178911789217893178941789517896178971789817899179001790117902179031790417905179061790717908179091791017911179121791317914179151791617917179181791917920179211792217923179241792517926179271792817929179301793117932179331793417935179361793717938179391794017941179421794317944179451794617947179481794917950179511795217953179541795517956179571795817959179601796117962179631796417965179661796717968179691797017971179721797317974179751797617977179781797917980179811798217983179841798517986179871798817989179901799117992179931799417995179961799717998179991800018001180021800318004180051800618007180081800918010180111801218013180141801518016180171801818019180201802118022180231802418025180261802718028180291803018031180321803318034180351803618037180381803918040180411804218043180441804518046180471804818049180501805118052180531805418055180561805718058180591806018061180621806318064180651806618067180681806918070180711807218073180741807518076180771807818079180801808118082180831808418085180861808718088180891809018091180921809318094180951809618097180981809918100181011810218103181041810518106181071810818109181101811118112181131811418115181161811718118181191812018121181221812318124181251812618127181281812918130181311813218133181341813518136181371813818139181401814118142181431814418145181461814718148181491815018151181521815318154181551815618157181581815918160181611816218163181641816518166181671816818169181701817118172181731817418175181761817718178181791818018181181821818318184181851818618187181881818918190181911819218193181941819518196181971819818199182001820118202182031820418205182061820718208182091821018211182121821318214182151821618217182181821918220182211822218223182241822518226182271822818229182301823118232182331823418235182361823718238182391824018241182421824318244182451824618247182481824918250182511825218253182541825518256182571825818259182601826118262182631826418265182661826718268182691827018271182721827318274182751827618277182781827918280182811828218283182841828518286182871828818289182901829118292182931829418295182961829718298182991830018301183021830318304183051830618307183081830918310183111831218313183141831518316183171831818319183201832118322183231832418325183261832718328183291833018331183321833318334183351833618337183381833918340183411834218343183441834518346183471834818349183501835118352183531835418355183561835718358183591836018361183621836318364183651836618367183681836918370183711837218373183741837518376183771837818379183801838118382183831838418385183861838718388183891839018391183921839318394183951839618397183981839918400184011840218403184041840518406184071840818409184101841118412184131841418415184161841718418184191842018421184221842318424184251842618427184281842918430184311843218433
  1. #if defined _DEBUG || true
  2. #define BEMC_ASSERT(_Expression) (void)( (!!(_Expression)) || (AssertFail(#_Expression, __LINE__), 0) )
  3. #else
  4. #define BEMC_ASSERT(_Expression) (void)(0)
  5. #endif
  6. #include <deque>
  7. #include "BeMCContext.h"
  8. #include "BeCOFFObject.h"
  9. #include "BeIRCodeGen.h"
  10. #include "../Compiler/BfIRCodeGen.h"
  11. #include "BeefySysLib/util/BeefPerf.h"
  12. #include "BeefySysLib/util/AllocDebug.h"
  13. #pragma warning(disable:4146)
  14. USING_NS_BF;
  15. //////////////////////////////////////////////////////////////////////////
  16. #define IS_BIT_SET_32(bits, idx) (((bits)[(idx) / 32] & (1 << ((idx) % 32))) != 0)
  17. // Only use "rep stosb"/"rep movsb" if total size is at least this value. The drawback is requiring saving/restore of RAX, RDI, and RCX (and RSI for movsb)
  18. const int BF_REP_MOV_LIMIT = 128;
  19. static const X64CPURegister gVolatileRegs[] = { X64Reg_RAX, X64Reg_RCX, X64Reg_RDX, X64Reg_R8, X64Reg_R9, X64Reg_R10, X64Reg_R11,
  20. X64Reg_XMM0_f64, X64Reg_XMM1_f64, X64Reg_XMM2_f64, X64Reg_XMM3_f64, X64Reg_XMM4_f64, X64Reg_XMM5_f64 };
  21. static const char* gOpName[] =
  22. {
  23. "None",
  24. "@Def",
  25. "@DefLoad",
  26. "@DefPhi",
  27. "@DbgDecl",
  28. "@LifetimeExtend",
  29. "@LifetimeStart",
  30. "@LifetimeEnd",
  31. "@LifetimeSoftEnd",
  32. "@ValueScopeSoftEnd",
  33. "@ValueScopeHardEnd",
  34. "@Label",
  35. //"PhiValue",
  36. "CmpToBool",
  37. "MemSet",
  38. "MemCpy",
  39. "FastCheckStack",
  40. "TLSSetup",
  41. "PreserveVolatiles",
  42. "RestoreVolatiles",
  43. "Unwind_PushReg",
  44. "Unwind_SaveXMM",
  45. "Unwind_Alloc",
  46. "Unwind_SetBP",
  47. "Rem",
  48. "IRem",
  49. "Nop",
  50. "Unreachable",
  51. "EnsureCodeAt",
  52. "DbgBreak",
  53. "MFence",
  54. "Mov",
  55. "MovRaw",
  56. "MovSX",
  57. "XChg",
  58. "XAdd",
  59. "CmpXChg",
  60. "Load",
  61. "Store",
  62. "Push",
  63. "Pop",
  64. "Neg",
  65. "Not",
  66. "Add",
  67. "Sub",
  68. "Mul",
  69. "IMul",
  70. "Div",
  71. "IDiv",
  72. "Cmp",
  73. "And",
  74. "Or",
  75. "Xor",
  76. "Shl",
  77. "Shr",
  78. "Sar",
  79. "Test",
  80. "CondBr",
  81. "Br",
  82. "Ret",
  83. "Call",
  84. };
  85. static_assert(BF_ARRAY_COUNT(gOpName) == (int)BeMCInstKind_COUNT, "gOpName incorrect size");
  86. void PrintBoolVec(Array<bool>& boolVec)
  87. {
  88. String str;
  89. for (int i = 0; i < (int)boolVec.size(); i++)
  90. {
  91. str += StrFormat("%d: %s\n", i, boolVec[i] ? "true" : "false");
  92. }
  93. OutputDebugStr(str);
  94. }
  95. static bool IsPowerOfTwo(int64 val)
  96. {
  97. return (val != 0) && ((val & (val - 1)) == 0);
  98. }
  99. bool BeVTrackingBits::IsSet(int idx)
  100. {
  101. return IS_BIT_SET_32((uint32*)this, idx);
  102. }
  103. void BeVTrackingBits::Set(int idx)
  104. {
  105. uint32* bits = (uint32*)this;
  106. bits[(idx) / 32] |= (1 << (idx % 32));
  107. }
  108. void BeVTrackingBits::Clear(int idx)
  109. {
  110. uint32* bits = (uint32*)this;
  111. bits[(idx) / 32] &= ~(1 << (idx % 32));
  112. }
  113. BeVTrackingContext::BeVTrackingContext(BeMCContext* mcContext)
  114. {
  115. mMCContext = mcContext;
  116. mNumEntries = 0;
  117. mNumItems = 0;
  118. mNumBlocks = -1;
  119. mTrackKindCount = (int)BeTrackKind_COUNT;
  120. }
  121. void BeVTrackingContext::Init(int numItems)
  122. {
  123. mNumItems = numItems;
  124. mNumEntries = mNumItems * mTrackKindCount;
  125. mNumBlocks = (mNumEntries + 31) / 32;
  126. }
  127. void BeVTrackingContext::Clear()
  128. {
  129. mAlloc.Clear();
  130. mNumBlocks = -1;
  131. }
  132. int BeVTrackingContext::GetBitsBytes()
  133. {
  134. return mNumBlocks * 4;
  135. }
  136. int BeVTrackingContext::GetIdx(int baseIdx, BeTrackKind liveKind)
  137. {
  138. return baseIdx + mNumItems * (int)liveKind;
  139. }
  140. void BeVTrackingContext::Print(BeVTrackingList* list)
  141. {
  142. String str;
  143. for (int i : *list)
  144. str += StrFormat("%d ", i);
  145. if (list->mNumChanges > 0)
  146. {
  147. str += " |";
  148. for (int changeIdx = 0; changeIdx < list->mNumChanges; changeIdx++)
  149. {
  150. int change = list->GetChange(changeIdx);
  151. if (change >= 0)
  152. str += StrFormat(" +%d", change);
  153. else
  154. str += StrFormat(" -%d", -change - 1);
  155. }
  156. }
  157. str += "\n";
  158. OutputDebugStr(str);
  159. }
  160. BeVTrackingList* BeVTrackingContext::AllocEmptyList()
  161. {
  162. int allocBytes = sizeof(int) * (2);
  163. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  164. mStats.mListBytes += allocBytes;
  165. newList->mSize = 0;
  166. newList->mNumChanges = 0;
  167. return newList;
  168. }
  169. BeVTrackingList* BeVTrackingContext::AddFiltered(BeVTrackingList* list, SizedArrayImpl<int>& filteredAdds, bool perserveChangeList)
  170. {
  171. int newSize = list->mSize + filteredAdds.size();
  172. int allocBytes = sizeof(int) * (2 + newSize);
  173. if (perserveChangeList)
  174. allocBytes += sizeof(int) * (int)(list->mNumChanges + filteredAdds.size());
  175. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  176. mStats.mListBytes += allocBytes;
  177. {
  178. if (filteredAdds.size() > 1)
  179. std::sort(filteredAdds.begin(), filteredAdds.end());
  180. int addIdx = 0;
  181. int nextAdd;
  182. if (addIdx < (int)filteredAdds.size())
  183. nextAdd = filteredAdds[addIdx++];
  184. else
  185. nextAdd = 0x7FFFFFFF;
  186. int* outPtr = &newList->mEntries[0];
  187. for (auto idx : *list)
  188. {
  189. while (idx > nextAdd)
  190. {
  191. *(outPtr++) = nextAdd;
  192. if (addIdx < (int)filteredAdds.size())
  193. nextAdd = filteredAdds[addIdx++];
  194. else
  195. nextAdd = 0x7FFFFFFF;
  196. }
  197. *(outPtr++) = idx;
  198. }
  199. while (nextAdd != 0x7FFFFFFF)
  200. {
  201. *(outPtr++) = nextAdd;
  202. if (addIdx >= (int)filteredAdds.size())
  203. break;
  204. nextAdd = filteredAdds[addIdx++];
  205. }
  206. }
  207. newList->mSize = newSize;
  208. if (perserveChangeList)
  209. {
  210. for (int changeIdx = 0; changeIdx < list->mNumChanges; changeIdx++)
  211. {
  212. newList->mEntries[newSize + changeIdx] = list->GetChange(changeIdx);
  213. }
  214. for (int changeIdx = 0; changeIdx < (int)filteredAdds.size(); changeIdx++)
  215. {
  216. newList->mEntries[newSize + list->mNumChanges + changeIdx] = filteredAdds[changeIdx];
  217. }
  218. newList->mNumChanges = list->mNumChanges + (int)filteredAdds.size();
  219. }
  220. else
  221. newList->mNumChanges = 0;
  222. return newList;
  223. }
  224. BeVTrackingList* BeVTrackingContext::AddFiltered(BeVTrackingList* list, int idx, bool perserveChangeList)
  225. {
  226. int newSize = list->mSize + 1;
  227. int allocBytes = sizeof(int) * (2 + newSize);
  228. if (perserveChangeList)
  229. allocBytes += sizeof(int) * (int)(list->mNumChanges + 1);
  230. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  231. mStats.mListBytes += allocBytes;
  232. {
  233. int addIdx = 0;
  234. int nextAdd;
  235. nextAdd = idx;
  236. int* outPtr = &newList->mEntries[0];
  237. for (auto idx : *list)
  238. {
  239. while (idx > nextAdd)
  240. {
  241. *(outPtr++) = nextAdd;
  242. nextAdd = 0x7FFFFFFF;
  243. }
  244. *(outPtr++) = idx;
  245. }
  246. while (nextAdd != 0x7FFFFFFF)
  247. {
  248. *(outPtr++) = nextAdd;
  249. break;
  250. }
  251. }
  252. newList->mSize = newSize;
  253. if (perserveChangeList)
  254. {
  255. for (int changeIdx = 0; changeIdx < list->mNumChanges; changeIdx++)
  256. {
  257. newList->mEntries[newSize + changeIdx] = list->GetChange(changeIdx);
  258. }
  259. newList->mEntries[newSize + list->mNumChanges] = idx;
  260. newList->mNumChanges = list->mNumChanges + (int)1;
  261. }
  262. else
  263. newList->mNumChanges = 0;
  264. return newList;
  265. }
  266. BeVTrackingList* BeVTrackingContext::Add(BeVTrackingList* list, const SizedArrayImpl<int>& indices, bool perserveChangeList)
  267. {
  268. SizedArray<int, 16> newIndices;
  269. for (int idx : indices)
  270. {
  271. if (!IsSet(list, idx))
  272. {
  273. newIndices.push_back(idx);
  274. }
  275. }
  276. if (newIndices.empty())
  277. return list;
  278. return AddFiltered(list, newIndices, perserveChangeList);
  279. }
  280. BeVTrackingList* BeVTrackingContext::Add(BeVTrackingList* list, int idx, bool perserveChangeList)
  281. {
  282. if (IsSet(list, idx))
  283. return list;
  284. return AddFiltered(list, idx, perserveChangeList);
  285. }
  286. // Performs an 'add' for items that were in prevDest
  287. BeVTrackingList* BeVTrackingContext::SetChanges(BeVTrackingList* prevDestEntry, BeVTrackingList* mergeFrom)
  288. {
  289. int removeCount = 0;
  290. int addCount = 0;
  291. int newSize = prevDestEntry->mSize;
  292. auto prevItr = prevDestEntry->begin();
  293. auto prevEnd = prevDestEntry->end();
  294. auto mergeFromItr = mergeFrom->begin();
  295. auto mergeFromEnd = mergeFrom->end();
  296. while ((prevItr != prevEnd) && (mergeFromItr != mergeFromEnd))
  297. {
  298. int prevIdx = *prevItr;
  299. int mergeIdx = *mergeFromItr;
  300. bool done = false;
  301. while (mergeIdx < prevIdx)
  302. {
  303. removeCount++;
  304. ++mergeFromItr;
  305. if (mergeFromItr == mergeFromEnd)
  306. {
  307. done = true;
  308. break;
  309. }
  310. mergeIdx = *mergeFromItr;
  311. }
  312. if (done)
  313. break;
  314. while (prevIdx < mergeIdx)
  315. {
  316. addCount++;
  317. ++prevItr;
  318. if (prevItr == prevEnd)
  319. {
  320. done = true;
  321. break;
  322. }
  323. prevIdx = *prevItr;
  324. }
  325. if (done)
  326. break;
  327. if (prevIdx == mergeIdx)
  328. {
  329. ++prevItr;
  330. ++mergeFromItr;
  331. }
  332. }
  333. while (prevItr != prevEnd)
  334. {
  335. addCount++;
  336. ++prevItr;
  337. }
  338. while (mergeFromItr != mergeFromEnd)
  339. {
  340. removeCount++;
  341. ++mergeFromItr;
  342. }
  343. int allocBytes = sizeof(int) * (2 + newSize + addCount + removeCount);
  344. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  345. mStats.mListBytes += allocBytes;
  346. int* outPtr = &newList->mEntries[0];
  347. int* changePtr = &newList->mEntries[newSize];
  348. prevItr = prevDestEntry->begin();
  349. mergeFromItr = mergeFrom->begin();
  350. while ((prevItr != prevEnd) && (mergeFromItr != mergeFromEnd))
  351. {
  352. int prevIdx = *prevItr;
  353. int mergeIdx = *mergeFromItr;
  354. bool done = false;
  355. while (mergeIdx < prevIdx)
  356. {
  357. *(changePtr++) = ~mergeIdx;
  358. ++mergeFromItr;
  359. if (mergeFromItr == mergeFromEnd)
  360. {
  361. done = true;
  362. break;
  363. }
  364. mergeIdx = *mergeFromItr;
  365. }
  366. if (done)
  367. break;
  368. while (prevIdx < mergeIdx)
  369. {
  370. *(outPtr++) = prevIdx;
  371. *(changePtr++) = prevIdx;
  372. ++prevItr;
  373. if (prevItr == prevEnd)
  374. {
  375. done = true;
  376. break;
  377. }
  378. prevIdx = *prevItr;
  379. }
  380. if (done)
  381. break;
  382. if (prevIdx == mergeIdx)
  383. {
  384. *(outPtr++) = *prevItr;
  385. ++prevItr;
  386. ++mergeFromItr;
  387. }
  388. }
  389. while (prevItr != prevEnd)
  390. {
  391. int prevIdx = *prevItr;
  392. *(outPtr++) = *prevItr;
  393. *(changePtr++) = prevIdx;
  394. ++prevItr;
  395. }
  396. while (mergeFromItr != mergeFromEnd)
  397. {
  398. int mergeIdx = *mergeFromItr;
  399. *(changePtr++) = ~mergeIdx;
  400. ++mergeFromItr;
  401. }
  402. BF_ASSERT((outPtr - &newList->mEntries[0]) == newSize);
  403. BF_ASSERT((changePtr - &newList->mEntries[newSize]) == addCount + removeCount);
  404. newList->mSize = newSize;
  405. newList->mNumChanges = addCount + removeCount;
  406. return newList;
  407. }
  408. BeVTrackingList* BeVTrackingContext::ClearFiltered(BeVTrackingList* list, const SizedArrayImpl<int>& indices)
  409. {
  410. /*int newSize = list->mSize - indices.size();
  411. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(sizeof(int) * (1 + newSize));
  412. if (indices.size() == 1)
  413. {
  414. int findIdx0 = indices[0];
  415. int* outPtr = 0;
  416. for (auto idx : indices)
  417. {
  418. if (idx != findIdx0)
  419. *(outPtr++) = idx;
  420. }
  421. BF_ASSERT(outPtr == &newList->mEntries[0] + indices.size());
  422. }
  423. else if (indices.size() == 2)
  424. {
  425. int findIdx0 = indices[0];
  426. int findIdx1 = indices[1];
  427. int* outPtr = 0;
  428. for (auto idx : indices)
  429. {
  430. if ((idx != findIdx0) && (idx != findIdx1))
  431. *(outPtr++) = idx;
  432. }
  433. BF_ASSERT(outPtr == &newList->mEntries[0] + indices.size());
  434. }
  435. else
  436. {
  437. int* outPtr = 0;
  438. for (auto idx : indices)
  439. {
  440. if (std::find(indices.begin(), indices.end(), idx) == indices.end())
  441. *(outPtr++) = idx;
  442. }
  443. BF_ASSERT(outPtr == &newList->mEntries[0] + indices.size());
  444. }
  445. newList->mSize = newSize;
  446. return newList;*/
  447. BF_FATAL("Broken");
  448. return NULL;
  449. }
  450. BeVTrackingList* BeVTrackingContext::Modify(BeVTrackingList* list, const SizedArrayImpl<int>& inAdds, const SizedArrayImpl<int>& inRemoves, SizedArrayImpl<int>& filteredAdds, SizedArrayImpl<int>& filteredRemoves, bool preserveChanges)
  451. {
  452. for (int idx : inAdds)
  453. {
  454. if (!IsSet(list, idx))
  455. {
  456. filteredAdds.push_back(idx);
  457. }
  458. }
  459. for (int idx : inRemoves)
  460. {
  461. if (IsSet(list, idx))
  462. {
  463. filteredRemoves.push_back(idx);
  464. }
  465. }
  466. if ((filteredAdds.empty()) && (filteredRemoves.empty()))
  467. return list;
  468. int newSize = list->mSize - filteredRemoves.size() + filteredAdds.size();
  469. int changeSize;
  470. if (preserveChanges)
  471. changeSize = list->mNumChanges;
  472. else
  473. changeSize = filteredRemoves.size() + filteredAdds.size();
  474. int allocBytes = sizeof(int) * (2 + newSize + changeSize);
  475. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  476. mStats.mListBytes += allocBytes;
  477. if (filteredAdds.size() > 1)
  478. std::sort(filteredAdds.begin(), filteredAdds.end());
  479. if (filteredRemoves.size() > 1)
  480. std::sort(filteredRemoves.begin(), filteredRemoves.end());
  481. int addIdx = 0;
  482. int nextAdd;
  483. if (addIdx < (int)filteredAdds.size())
  484. nextAdd = filteredAdds[addIdx++];
  485. else
  486. nextAdd = 0x7FFFFFFF;
  487. int removeIdx = 0;
  488. int nextRemove;
  489. if (removeIdx < (int)filteredRemoves.size())
  490. nextRemove = filteredRemoves[removeIdx++];
  491. else
  492. nextRemove = 0x7FFFFFFF;
  493. int* outPtr = &newList->mEntries[0];
  494. for (auto idx : *list)
  495. {
  496. if (idx == nextRemove)
  497. {
  498. if (removeIdx < (int)filteredRemoves.size())
  499. nextRemove = filteredRemoves[removeIdx++];
  500. else
  501. nextRemove = 0x7FFFFFFF;
  502. continue;
  503. }
  504. while (idx > nextAdd)
  505. {
  506. *(outPtr++) = nextAdd;
  507. if (addIdx < (int)filteredAdds.size())
  508. nextAdd = filteredAdds[addIdx++];
  509. else
  510. nextAdd = 0x7FFFFFFF;
  511. }
  512. *(outPtr++) = idx;
  513. }
  514. while (nextAdd != 0x7FFFFFFF)
  515. {
  516. *(outPtr++) = nextAdd;
  517. if (addIdx >= (int)filteredAdds.size())
  518. break;
  519. nextAdd = filteredAdds[addIdx++];
  520. }
  521. BF_ASSERT(outPtr == &newList->mEntries[0] + newSize);
  522. BF_ASSERT((nextAdd = 0x7FFFFFFF) && (nextRemove == 0x7FFFFFFF));
  523. if (preserveChanges)
  524. {
  525. for (int i = 0; i < list->mNumChanges; i++)
  526. newList->mEntries[newSize + i] = list->mEntries[list->mSize + i];
  527. }
  528. else
  529. {
  530. for (int i = 0; i < (int)filteredRemoves.size(); i++)
  531. {
  532. newList->mEntries[newSize + i] = -filteredRemoves[i] - 1;
  533. }
  534. for (int i = 0; i < (int)filteredAdds.size(); i++)
  535. {
  536. newList->mEntries[newSize + filteredRemoves.size() + i] = filteredAdds[i];
  537. }
  538. }
  539. newList->mSize = newSize;
  540. newList->mNumChanges = changeSize;
  541. ///
  542. /*for (int i = 0; i < mNumEntries; i++)
  543. BF_ASSERT(IsSet(newList, i) == unit.mBits->IsSet(i));
  544. int prevIdx = -1;
  545. for (int idx : *newList)
  546. {
  547. BF_ASSERT(idx > prevIdx);
  548. prevIdx = idx;
  549. }
  550. OutputDebugStrF("Modify %d %@\n", modifyItrIdx, newList);*/
  551. ///
  552. return newList;
  553. }
  554. int BeVTrackingContext::FindIndex(BeVTrackingList* entry, int val)
  555. {
  556. int lo = 0;
  557. int hi = entry->mSize - 1;
  558. while (lo <= hi)
  559. {
  560. int i = (lo + hi) / 2;
  561. int midVal = entry->mEntries[i];
  562. int c = midVal - val;
  563. if (c == 0) return i;
  564. if (c < 0)
  565. lo = i + 1;
  566. else
  567. hi = i - 1;
  568. }
  569. return ~lo;
  570. }
  571. bool BeVTrackingContext::IsSet(BeVTrackingList* entry, int idx)
  572. {
  573. return FindIndex(entry, idx) >= 0;
  574. }
  575. bool BeVTrackingContext::IsSet(BeVTrackingList* entry, int idx, BeTrackKind trackKind)
  576. {
  577. return IsSet(entry, GetIdx(idx, trackKind));
  578. }
  579. BeVTrackingList* BeVTrackingContext::Clear(BeVTrackingList* list, const SizedArrayImpl<int>& indices)
  580. {
  581. SizedArray<int, 16> newIndices;
  582. for (int idx : indices)
  583. {
  584. if (IsSet(list, idx))
  585. {
  586. newIndices.push_back(idx);
  587. }
  588. }
  589. if (newIndices.empty())
  590. return list;
  591. return ClearFiltered(list, newIndices);
  592. }
  593. bool BeVTrackingContext::IsEmpty(BeVTrackingList* list)
  594. {
  595. return list->mSize == 0;
  596. }
  597. BeVTrackingList* BeVTrackingContext::Merge(BeVTrackingList* prevDestEntry, BeVTrackingList* mergeFrom)
  598. {
  599. if (prevDestEntry == NULL)
  600. return mergeFrom;
  601. if (mergeFrom->mSize == 0)
  602. return prevDestEntry;
  603. if (prevDestEntry->mSize == 0)
  604. return mergeFrom;
  605. int newSize = prevDestEntry->mSize;
  606. auto prevItr = prevDestEntry->begin();
  607. auto prevEnd = prevDestEntry->end();
  608. auto mergeFromItr = mergeFrom->begin();
  609. auto mergeFromEnd = mergeFrom->end();
  610. while ((prevItr != prevEnd) && (mergeFromItr != mergeFromEnd))
  611. {
  612. int prevIdx = *prevItr;
  613. int mergeIdx = *mergeFromItr;
  614. bool done = false;
  615. while (mergeIdx < prevIdx)
  616. {
  617. newSize++;
  618. ++mergeFromItr;
  619. if (mergeFromItr == mergeFromEnd)
  620. {
  621. done = true;
  622. break;
  623. }
  624. mergeIdx = *mergeFromItr;
  625. }
  626. if (done)
  627. break;
  628. while (prevIdx < mergeIdx)
  629. {
  630. ++prevItr;
  631. if (prevItr == prevEnd)
  632. {
  633. done = true;
  634. break;
  635. }
  636. prevIdx = *prevItr;
  637. }
  638. if (done)
  639. break;
  640. if (prevIdx == mergeIdx)
  641. {
  642. ++prevItr;
  643. ++mergeFromItr;
  644. }
  645. }
  646. while (mergeFromItr != mergeFromEnd)
  647. {
  648. newSize++;
  649. ++mergeFromItr;
  650. }
  651. if (newSize == prevDestEntry->mSize)
  652. return prevDestEntry;
  653. int allocBytes = sizeof(int) * (2 + newSize);
  654. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  655. mStats.mListBytes += allocBytes;
  656. int* outPtr = &newList->mEntries[0];
  657. prevItr = prevDestEntry->begin();
  658. mergeFromItr = mergeFrom->begin();
  659. while ((prevItr != prevEnd) && (mergeFromItr != mergeFromEnd))
  660. {
  661. int prevIdx = *prevItr;
  662. int mergeIdx = *mergeFromItr;
  663. bool done = false;
  664. while (mergeIdx < prevIdx)
  665. {
  666. *(outPtr++) = mergeIdx;
  667. ++mergeFromItr;
  668. if (mergeFromItr == mergeFromEnd)
  669. {
  670. done = true;
  671. break;
  672. }
  673. mergeIdx = *mergeFromItr;
  674. }
  675. if (done)
  676. break;
  677. while (prevIdx < mergeIdx)
  678. {
  679. *(outPtr++) = prevIdx;
  680. ++prevItr;
  681. if (prevItr == prevEnd)
  682. {
  683. done = true;
  684. break;
  685. }
  686. prevIdx = *prevItr;
  687. }
  688. if (done)
  689. break;
  690. if (prevIdx == mergeIdx)
  691. {
  692. *(outPtr++) = *prevItr;
  693. ++prevItr;
  694. ++mergeFromItr;
  695. }
  696. }
  697. while (prevItr != prevEnd)
  698. {
  699. *(outPtr++) = *prevItr;
  700. ++prevItr;
  701. }
  702. while (mergeFromItr != mergeFromEnd)
  703. {
  704. *(outPtr++) = *mergeFromItr;
  705. ++mergeFromItr;
  706. }
  707. BF_ASSERT((outPtr - &newList->mEntries[0]) == newSize);
  708. newList->mSize = newSize;
  709. newList->mNumChanges = 0;
  710. return newList;
  711. }
  712. BeVTrackingList* BeVTrackingContext::MergeChanges(BeVTrackingList* prevDestEntry, BeVTrackingList* mergeFrom)
  713. {
  714. if (prevDestEntry == mergeFrom)
  715. return prevDestEntry;
  716. if (mergeFrom->mNumChanges == 0)
  717. return prevDestEntry;
  718. SizedArray<int, 2> changes;
  719. for (int changeIdx = 0; changeIdx < (int)prevDestEntry->mNumChanges; changeIdx++)
  720. changes.push_back(prevDestEntry->GetChange(changeIdx));
  721. for (int changeIdx = 0; changeIdx < (int)mergeFrom->mNumChanges; changeIdx++)
  722. {
  723. // If there isn't already a change (whether and add or a remove) that refers to this same vreg,
  724. // then we add this change as well
  725. int change = mergeFrom->GetChange(changeIdx);
  726. int negChange = -change - 1;
  727. /*if (((std::find(changes.begin(), changes.end(), change) == changes.end())) &&
  728. ((std::find(changes.begin(), changes.end(), negChange) == changes.end())))*/
  729. if ((!changes.Contains(change)) && (!changes.Contains(negChange)))
  730. {
  731. changes.push_back(change);
  732. }
  733. }
  734. int newSize = prevDestEntry->mSize;
  735. int allocBytes = (int)(sizeof(int) * (2 + newSize + changes.size()));
  736. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  737. mStats.mListBytes += allocBytes;
  738. memcpy(&newList->mEntries[0], &prevDestEntry->mEntries[0], newSize * sizeof(int));
  739. for (int i = 0; i < (int)changes.size(); i++)
  740. {
  741. newList->mEntries[newSize + i] = changes[i];
  742. }
  743. newList->mSize = newSize;
  744. newList->mNumChanges = (int)changes.size();
  745. return newList;
  746. }
  747. BeVTrackingList * BeVTrackingContext::RemoveChange(BeVTrackingList* prevDestEntry, int idx)
  748. {
  749. int newSize = prevDestEntry->mSize;
  750. int allocBytes = (int)(sizeof(int) * (2 + newSize + prevDestEntry->mNumChanges - 1));
  751. auto newList = (BeVTrackingList*)mAlloc.AllocBytes(allocBytes);
  752. mStats.mListBytes += allocBytes;
  753. memcpy(&newList->mEntries[0], &prevDestEntry->mEntries[0], newSize * sizeof(int));
  754. bool found = false;
  755. int outIdx = newSize;
  756. for (int i = 0; i < (int)prevDestEntry->mNumChanges; i++)
  757. {
  758. int change = prevDestEntry->GetChange(i);
  759. if (change == idx)
  760. {
  761. found = true;
  762. continue;
  763. }
  764. newList->mEntries[outIdx++] = change;
  765. }
  766. if (!found)
  767. return prevDestEntry;
  768. //BF_ASSERT(found);
  769. newList->mSize = newSize;
  770. newList->mNumChanges = prevDestEntry->mNumChanges - 1;
  771. return newList;
  772. }
  773. //////////////////////////////////////////////////////////////////////////
  774. BeMCColorizer::BeMCColorizer(BeMCContext* mcContext)
  775. {
  776. mContext = mcContext;
  777. mReserveParamRegs = false;
  778. }
  779. void BeMCColorizer::Prepare()
  780. {
  781. mReserveParamRegs = false;
  782. mNodes.Resize(mContext->mVRegInfo.size());
  783. for (int vregIdx = 0; vregIdx < (int)mNodes.size(); vregIdx++)
  784. {
  785. auto node = &mNodes[vregIdx];
  786. node->Prepare();
  787. //node->mActualVRegIdx = vregIdx;
  788. auto vregInfo = mContext->mVRegInfo[vregIdx];
  789. if ((vregInfo->mIsRetVal) && (mContext->mCompositeRetVRegIdx != -1) && (vregIdx != mContext->mCompositeRetVRegIdx))
  790. continue;
  791. if (vregInfo->mRelTo)
  792. {
  793. BF_ASSERT(vregInfo->mIsExpr);
  794. /*if ((vregInfo->IsDirectRelTo()) && (vregInfo->mRelTo.mKind == BeMCOperandKind_VReg))
  795. node->mActualVRegIdx = vregInfo->mRelTo.mVRegIdx; */
  796. //node->mWantsReg = false;
  797. //vregInfo->mReg = X64Reg_None;
  798. //continue;
  799. }
  800. if ((vregInfo->mRefCount > 0) || (vregInfo->mIsRetVal))
  801. {
  802. node->mWantsReg = ((vregInfo->mType->mSize > 0) && (!vregInfo->mRegNumPinned) && (!vregInfo->mSpilled) &&
  803. (!vregInfo->mIsExpr) && (!vregInfo->mForceMem) && (vregInfo->mFrameOffset == INT_MIN));
  804. }
  805. // if (vregInfo->mIsRetVal)
  806. // node->mWantsReg = true;
  807. if (!node->mWantsReg)
  808. vregInfo->mReg = X64Reg_None;
  809. vregInfo->mVRegAffinity = -1;
  810. /*if (vregInfo->mForceReg)
  811. {
  812. // We can't have reg restrictions when we have forceReg set
  813. BF_ASSERT(!vregInfo->mDisableR12);
  814. BF_ASSERT(!vregInfo->mDisableRDX);
  815. }*/
  816. if (vregInfo->mDisableR11)
  817. node->AdjustRegCost(X64Reg_R11, 0x0FFFFFFF);
  818. if (vregInfo->mDisableR12)
  819. node->AdjustRegCost(X64Reg_R12, 0x0FFFFFFF);
  820. if (vregInfo->mDisableR13)
  821. node->AdjustRegCost(X64Reg_R13, 0x0FFFFFFF);
  822. if (vregInfo->mDisableRAX)
  823. node->AdjustRegCost(X64Reg_RAX, 0x0FFFFFFF);
  824. if (vregInfo->mDisableRDX)
  825. node->AdjustRegCost(X64Reg_RDX, 0x0FFFFFFF);
  826. if (vregInfo->mDisableEx)
  827. {
  828. for (int i = X64Reg_RSI; i <= X64Reg_R15; i++)
  829. node->AdjustRegCost((X64CPURegister)i, 0x0FFFFFFF);
  830. }
  831. }
  832. }
  833. void BeMCColorizer::AddEdge(int vreg0, int vreg1)
  834. {
  835. int checkVRegIdx0 = mContext->GetUnderlyingVReg(vreg0);
  836. int checkVRegIdx1 = mContext->GetUnderlyingVReg(vreg1);
  837. if (checkVRegIdx0 == checkVRegIdx1)
  838. return;
  839. auto node0 = &mNodes[checkVRegIdx0];
  840. auto node1 = &mNodes[checkVRegIdx1];
  841. if ((node0->mWantsReg) && (node1->mWantsReg))
  842. {
  843. node0->mEdges.Add(checkVRegIdx1);
  844. node1->mEdges.Add(checkVRegIdx0);
  845. }
  846. }
  847. void BeMCColorizer::PropogateMemCost(const BeMCOperand & operand, int memCost)
  848. {
  849. if (operand.IsVRegAny())
  850. {
  851. auto vregInfo = mContext->mVRegInfo[operand.mVRegIdx];
  852. mNodes[operand.mVRegIdx].mMemCost += memCost;
  853. PropogateMemCost(vregInfo->mRelTo, memCost);
  854. PropogateMemCost(vregInfo->mRelOffset, memCost);
  855. }
  856. }
  857. void BeMCColorizer::GenerateRegCosts()
  858. {
  859. bool doRegCost = true;
  860. // Disallow param reg cross-refs
  861. {
  862. Array<X64CPURegister> paramRegsLeft = mContext->mParamsUsedRegs;
  863. Array<int> prevRegMovs;
  864. auto mcBlock = mContext->mBlocks[0];
  865. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  866. {
  867. auto inst = mcBlock->mInstructions[instIdx];
  868. if ((inst->IsMov()) && (inst->mArg1.IsNativeReg()) && (inst->mArg0.IsVReg()))
  869. {
  870. int vregIdx = mContext->GetUnderlyingVReg(inst->mArg0.mVRegIdx);
  871. auto reg = mContext->GetFullRegister(inst->mArg1.mReg);
  872. // Don't allow any previous "direct reg movs" assign to a vreg that gets bound
  873. // to this reg. This is to avoid "cross-referencing" param vregs. IE:
  874. // Mov %vreg0<RDX>, RCX
  875. // Mov %vreg1<RCX>, RDX
  876. // Because the first MOV would destroy the incoming RDX
  877. for (auto prevVRegIdx : prevRegMovs)
  878. {
  879. auto node = &mNodes[prevVRegIdx];
  880. node->AdjustRegCost(reg, 0x0FFFFFFF);
  881. }
  882. /*auto itr = std::find(paramRegsLeft.begin(), paramRegsLeft.end(), reg);
  883. if (itr != paramRegsLeft.end())
  884. {
  885. paramRegsLeft.erase(itr);
  886. if (paramRegsLeft.size() == 0)
  887. break;
  888. prevRegMovs.push_back(vregIdx);
  889. }*/
  890. if (paramRegsLeft.Remove(reg))
  891. {
  892. if (paramRegsLeft.size() == 0)
  893. break;
  894. prevRegMovs.push_back(vregIdx);
  895. }
  896. }
  897. }
  898. BF_ASSERT(paramRegsLeft.size() == 0);
  899. }
  900. int preserveDepth = 0;
  901. for (auto mcBlock : mContext->mBlocks)
  902. {
  903. int costMult = mcBlock->mIsLooped ? 4 : 1;
  904. for (int instIdx = (int)mcBlock->mInstructions.size() - 1; instIdx >= 0; instIdx--)
  905. {
  906. auto inst = mcBlock->mInstructions[instIdx];
  907. // We could adjust register cost based on a possibility of cross-dependencies that won't allow us
  908. // to reorder the movs for param setup
  909. if (inst->mKind == BeMCInstKind_PreserveVolatiles)
  910. {
  911. preserveDepth++;
  912. int preserveCost = 8 * costMult;
  913. if (inst->mArg0.mKind == BeMCOperandKind_PreserveFlag)
  914. {
  915. // Just use a small cost for NoReturns, those are implied to be "unlikely" paths (ie: errors)
  916. // But at the least, the code is smaller if we don't need to save the regs so use a small cost
  917. if ((inst->mArg0.mPreserveFlag & BeMCPreserveFlag_NoRestore) != 0)
  918. preserveCost = 1;
  919. }
  920. auto restoreIdx = mContext->FindRestoreVolatiles(mcBlock, instIdx);
  921. if (restoreIdx == -1)
  922. restoreIdx = mcBlock->mInstructions.size() - 1; // 'Ret'
  923. auto restoreInst = mcBlock->mInstructions[restoreIdx];
  924. // Any vregs alive during this call will incur a cost of saving/restoring if we allocate onto a volatile register.
  925. // If the vreg is either input-only or output-only (short lived temporary) then we can map it directly to a volatile register
  926. for (int vregIdx : *inst->mLiveness)
  927. {
  928. if (vregIdx >= mContext->mLivenessContext.mNumItems)
  929. continue;
  930. auto checkVRegIdx = mContext->GetUnderlyingVReg(vregIdx);
  931. // Already handled the underlying vreg?
  932. if ((checkVRegIdx != vregIdx) && (mContext->mLivenessContext.IsSet(inst->mLiveness, checkVRegIdx)))
  933. continue;
  934. if (mContext->mLivenessContext.IsSet(restoreInst->mLiveness, vregIdx))
  935. {
  936. auto node = &mNodes[checkVRegIdx];
  937. if (inst->mArg0.IsNativeReg())
  938. {
  939. // Only one specific register used being preserved
  940. node->AdjustRegCost(inst->mArg0.mReg, preserveCost);
  941. }
  942. else
  943. {
  944. // All volatile registers preserved
  945. for (auto reg : gVolatileRegs)
  946. {
  947. node->AdjustRegCost(reg, preserveCost);
  948. }
  949. }
  950. }
  951. }
  952. }
  953. else if (inst->mKind == BeMCInstKind_RestoreVolatiles)
  954. {
  955. preserveDepth--;
  956. }
  957. if (inst->IsPsuedo())
  958. continue;
  959. // If both args are non-registers then we presume we may have to create a scratch register
  960. // so inflate the cost. Note that the first time through, this will presumably occur for all
  961. // operations but we can change our minds as instructions are legalized
  962. if ((inst->mArg0.IsVReg()) && (inst->mArg1.IsVReg()))
  963. {
  964. auto vregInfo0 = mContext->mVRegInfo[inst->mArg0.mVRegIdx];
  965. auto vregInfo1 = mContext->mVRegInfo[inst->mArg1.mVRegIdx];
  966. if ((vregInfo0->mReg == X64Reg_None) && (vregInfo1->mReg == X64Reg_None))
  967. {
  968. Node* node0 = &mNodes[inst->mArg1.mVRegIdx];
  969. Node* node1 = &mNodes[inst->mArg1.mVRegIdx];
  970. node0->mMemCost += 4 * costMult;
  971. node1->mMemCost += 4 * costMult;
  972. }
  973. }
  974. if ((inst->mKind == BeMCInstKind_Div) || (inst->mKind == BeMCInstKind_IDiv))
  975. {
  976. if (inst->mArg0.IsVReg())
  977. {
  978. Node* node = &mNodes[inst->mArg0.mVRegIdx];
  979. int adjustCost = -4;
  980. if (preserveDepth >= 2)
  981. {
  982. // This has to be large enough to counteract the 'PreserveVolatile RAX'
  983. adjustCost -= 8;
  984. }
  985. node->AdjustRegCost(X64Reg_RAX, costMult * adjustCost);
  986. continue;
  987. }
  988. }
  989. if (inst->mKind == BeMCInstKind_MovSX)
  990. {
  991. // We can only perform MovSX on reg targets
  992. if (inst->mArg0.IsVReg())
  993. mNodes[inst->mArg0.mVRegIdx].mMemCost += 4 * costMult;
  994. }
  995. if (inst->mKind == BeMCInstKind_Mov)
  996. {
  997. if ((inst->mArg0.mKind == BeMCOperandKind_VReg) &&
  998. (inst->mArg1.mKind == BeMCOperandKind_VReg))
  999. {
  1000. int arg0Idx = mContext->GetUnderlyingVReg(inst->mArg0.mVRegIdx);
  1001. int arg1Idx = mContext->GetUnderlyingVReg(inst->mArg1.mVRegIdx);
  1002. auto vregInfo0 = mContext->mVRegInfo[arg0Idx];
  1003. auto vregInfo1 = mContext->mVRegInfo[arg1Idx];
  1004. if (vregInfo0->mVRegAffinity == -1)
  1005. vregInfo0->mVRegAffinity = arg1Idx;
  1006. if (vregInfo1->mVRegAffinity == -1)
  1007. vregInfo1->mVRegAffinity = arg0Idx;
  1008. }
  1009. }
  1010. if (inst->mKind == BeMCInstKind_Call)
  1011. {
  1012. if (inst->mArg0.IsVRegAny())
  1013. {
  1014. for (int checkInstIdx = instIdx - 1; checkInstIdx >= 0; checkInstIdx--)
  1015. {
  1016. auto checkInst = mcBlock->mInstructions[checkInstIdx];
  1017. if ((checkInst->mKind == BeMCInstKind_Mov) && (checkInst->mArg0.IsNativeReg()))
  1018. {
  1019. // This can save us from needing to mov the call addr to RAX before the call
  1020. mNodes[inst->mArg0.mVRegIdx].AdjustRegCost(checkInst->mArg0.mReg, 2 * costMult);
  1021. }
  1022. else
  1023. break;
  1024. }
  1025. }
  1026. }
  1027. if (inst->IsMov())
  1028. {
  1029. auto argTo = inst->mArg0;
  1030. auto vregInfo = mContext->GetVRegInfo(argTo);
  1031. if ((vregInfo != NULL) && (vregInfo->IsDirectRelTo()))
  1032. argTo = vregInfo->mRelTo;
  1033. auto argFrom = inst->mArg1;
  1034. vregInfo = mContext->GetVRegInfo(argFrom);
  1035. if ((vregInfo != NULL) && (vregInfo->IsDirectRelTo()))
  1036. argFrom = vregInfo->mRelTo;
  1037. if (argTo == argFrom)
  1038. continue;
  1039. }
  1040. auto operands = { &inst->mResult, &inst->mArg0, &inst->mArg1 };
  1041. for (auto operand : operands)
  1042. {
  1043. if (operand->IsVRegAny())
  1044. {
  1045. Node* node = &mNodes[operand->mVRegIdx];
  1046. auto vregInfo = mContext->mVRegInfo[operand->mVRegIdx];
  1047. if ((vregInfo->mIsRetVal) && (mContext->mCompositeRetVRegIdx != -1))
  1048. vregInfo = mContext->mVRegInfo[mContext->mCompositeRetVRegIdx];
  1049. if (operand->mKind == BeMCOperandKind_VRegLoad)
  1050. {
  1051. // We propagate this mem cost to RelTo/RelOffset layer
  1052. Node* node = &mNodes[operand->mVRegIdx];
  1053. node->mMemCost += 6 * costMult;
  1054. }
  1055. while (vregInfo->IsDirectRelTo())
  1056. {
  1057. node = &mNodes[vregInfo->mRelTo.mVRegIdx];
  1058. vregInfo = mContext->mVRegInfo[vregInfo->mRelTo.mVRegIdx];
  1059. }
  1060. node->mMemCost += 4 * costMult;
  1061. if (vregInfo->mRelTo.IsVRegAny())
  1062. mNodes[vregInfo->mRelTo.mVRegIdx].mMemCost += 4 * costMult;
  1063. if (vregInfo->mRelOffset.IsVRegAny()) // Higher cost, will definitely require a reg
  1064. mNodes[vregInfo->mRelOffset.mVRegIdx].mMemCost += 8 * costMult;
  1065. if (inst->IsMov())
  1066. {
  1067. if (operand != &inst->mResult)
  1068. {
  1069. auto otherReg = X64Reg_None;
  1070. if (operand == &inst->mArg0)
  1071. {
  1072. if (inst->mArg1.mKind == BeMCOperandKind_NativeReg)
  1073. otherReg = inst->mArg1.mReg;
  1074. if (inst->mArg1.IsImmediateFloat()) // Moving a float immediate into memory takes two instructions
  1075. node->mMemCost += 4 * costMult;
  1076. }
  1077. else
  1078. {
  1079. if (inst->mArg0.mKind == BeMCOperandKind_NativeReg)
  1080. otherReg = inst->mArg0.mReg;
  1081. }
  1082. // If we assigned ourselves to this reg then it would be a "mov reg, reg" which could
  1083. // be eliminated so the cost is negative. We could be tempted to more strongly favor this,
  1084. // but that could cause RAX to be used for __return when it would be less costly to use
  1085. // memory so we don't have to preserve/restore it across call boundaries
  1086. if (otherReg != X64Reg_None)
  1087. node->AdjustRegCost(mContext->GetFullRegister(otherReg), -2 * costMult);
  1088. }
  1089. }
  1090. }
  1091. }
  1092. }
  1093. }
  1094. for (int nodeIdx = 0; nodeIdx < (int)mNodes.size(); nodeIdx++)
  1095. {
  1096. auto node = &mNodes[nodeIdx];
  1097. if (node->mMemCost > 0)
  1098. {
  1099. auto vregInfo = mContext->mVRegInfo[nodeIdx];
  1100. if (vregInfo->mIsExpr)
  1101. {
  1102. PropogateMemCost(vregInfo->mRelTo, node->mMemCost);
  1103. PropogateMemCost(vregInfo->mRelOffset, node->mMemCost);
  1104. }
  1105. }
  1106. }
  1107. BF_ASSERT(preserveDepth == 0);
  1108. }
  1109. void BeMCColorizer::AssignRegs(RegKind regKind)
  1110. {
  1111. X64CPURegister highestReg;
  1112. int totalRegs32 = 0;
  1113. int totalRegs16 = 0;
  1114. SizedArray<X64CPURegister, 32> validRegs;
  1115. if (regKind == BeMCColorizer::RegKind_Ints)
  1116. {
  1117. highestReg = X64Reg_R15;
  1118. validRegs = { X64Reg_RAX, X64Reg_RBX, X64Reg_RCX, X64Reg_RDX, X64Reg_RSI, X64Reg_RDI, X64Reg_R8,
  1119. X64Reg_R9, X64Reg_R10, X64Reg_R11, X64Reg_R12, X64Reg_R13, X64Reg_R14, X64Reg_R15 };
  1120. }
  1121. else
  1122. {
  1123. highestReg = X64Reg_M128_XMM14; // Leave X64Reg_M128_XMM15 as a scratch reg
  1124. validRegs = {
  1125. X64Reg_M128_XMM0, X64Reg_M128_XMM1, X64Reg_M128_XMM2, X64Reg_M128_XMM3,
  1126. X64Reg_M128_XMM4, X64Reg_M128_XMM5, X64Reg_M128_XMM6, X64Reg_M128_XMM7,
  1127. X64Reg_M128_XMM8, X64Reg_M128_XMM9, X64Reg_M128_XMM10, X64Reg_M128_XMM11,
  1128. X64Reg_M128_XMM12, X64Reg_M128_XMM13, X64Reg_M128_XMM14 /*, X64Reg_M128_XMM15*/};
  1129. }
  1130. int totalRegs = (int)validRegs.size();
  1131. #define BF_DEQUE
  1132. #ifdef BF_DEQUE
  1133. std::deque<int> vregStack;
  1134. #else
  1135. std::vector<int> vregStack;
  1136. #endif
  1137. //vregStack.reserve(mNodes.size());
  1138. SizedArray<int, 32> vregHiPriStack;
  1139. vregHiPriStack.reserve(mNodes.size());
  1140. SizedArray<int, 32> vregGraph;
  1141. vregGraph.reserve(mNodes.size());
  1142. SizedArray<int, 32> orderedSpillList;
  1143. //
  1144. {
  1145. BP_ZONE("AssignRegs:vregGraph build");
  1146. for (int vregIdx = (int)mNodes.size() - 1; vregIdx >= 0; vregIdx--)
  1147. {
  1148. auto node = &mNodes[vregIdx];
  1149. if (node->mWantsReg)
  1150. {
  1151. auto vregInfo = mContext->mVRegInfo[vregIdx];
  1152. bool canBeReg = false;
  1153. if (regKind == RegKind_Ints)
  1154. {
  1155. if ((vregInfo->mType->IsInt()) || (vregInfo->mType->mTypeCode == BeTypeCode_Boolean) || (vregInfo->mType->mTypeCode == BeTypeCode_Pointer))
  1156. canBeReg = true;
  1157. }
  1158. else if (regKind == RegKind_Floats)
  1159. {
  1160. if ((vregInfo->mType->IsFloat()) || (vregInfo->mType->IsVector()))
  1161. canBeReg = true;
  1162. }
  1163. if (canBeReg)
  1164. {
  1165. node->mInGraph = true;
  1166. node->mGraphEdgeCount = 0;
  1167. vregGraph.push_back(vregIdx);
  1168. }
  1169. else
  1170. {
  1171. node->mInGraph = false;
  1172. }
  1173. }
  1174. }
  1175. }
  1176. for (int vregIdx = (int)mNodes.size() - 1; vregIdx >= 0; vregIdx--)
  1177. {
  1178. auto node = &mNodes[vregIdx];
  1179. if (node->mInGraph)
  1180. {
  1181. for (auto connNodeIdx : node->mEdges)
  1182. {
  1183. Node* connNode = &mNodes[connNodeIdx];
  1184. if (connNode->mInGraph)
  1185. node->mGraphEdgeCount++;
  1186. }
  1187. }
  1188. }
  1189. //
  1190. {
  1191. int graphSize = (int)vregGraph.size();
  1192. BP_ZONE("AssignRegs:buildStack");
  1193. while (graphSize > 0)
  1194. {
  1195. bool hadNewStackItem = false;
  1196. #ifdef _DEBUG
  1197. // for (int graphIdx = 0; graphIdx < (int)vregGraph.size(); graphIdx++)
  1198. // {
  1199. // int validatedCount = 0;
  1200. // int vregIdx = vregGraph[graphIdx];
  1201. // if (vregIdx == -1)
  1202. // continue;
  1203. // Node* node = &mNodes[vregIdx];
  1204. // for (auto connNodeIdx : node->mEdges)
  1205. // {
  1206. // Node* connNode = &mNodes[connNodeIdx];
  1207. // if (connNode->mInGraph)
  1208. // validatedCount++;
  1209. // }
  1210. // BF_ASSERT(validatedCount == node->mGraphEdgeCount);
  1211. // }
  1212. #endif
  1213. for (int graphIdx = 0; graphIdx < (int)vregGraph.size(); graphIdx++)
  1214. {
  1215. int vregIdx = vregGraph[graphIdx];
  1216. if (vregIdx == -1)
  1217. continue;
  1218. Node* node = &mNodes[vregIdx];
  1219. auto vregInfo = mContext->mVRegInfo[vregIdx];
  1220. int validRegCount = (int)validRegs.size();
  1221. BF_ASSERT(node->mGraphEdgeCount <= vregGraph.size() - 1);
  1222. if ((node->mSpilled) || (node->mGraphEdgeCount < validRegCount))
  1223. {
  1224. node->mInGraph = false;
  1225. for (auto connNodeIdx : node->mEdges)
  1226. {
  1227. Node* connNode = &mNodes[connNodeIdx];
  1228. connNode->mGraphEdgeCount--;
  1229. }
  1230. vregGraph[graphIdx] = -1;
  1231. graphSize--;
  1232. if (!node->mSpilled)
  1233. {
  1234. vregStack.push_back(vregIdx);
  1235. hadNewStackItem = true;
  1236. }
  1237. else
  1238. {
  1239. // We insert spills at the front so we can try to "optimistically" unspill them
  1240. // after all the definite coloring is done
  1241. #ifdef BF_DEQUE
  1242. vregStack.push_front(vregIdx);
  1243. #else
  1244. vregStack.insert(vregStack.begin(), vregIdx);
  1245. #endif
  1246. }
  1247. }
  1248. }
  1249. if (!hadNewStackItem)
  1250. {
  1251. BP_ZONE("Spill");
  1252. // We need to spill!
  1253. int bestSpillVReg = -1;
  1254. for (int regPassIdx = 0; regPassIdx < 2; regPassIdx++)
  1255. {
  1256. while (!orderedSpillList.empty())
  1257. {
  1258. int vregIdx = orderedSpillList.back();
  1259. orderedSpillList.pop_back();
  1260. if (mNodes[vregIdx].mInGraph)
  1261. {
  1262. bestSpillVReg = vregIdx;
  1263. break;
  1264. }
  1265. }
  1266. if (bestSpillVReg != -1)
  1267. break;
  1268. // Order by mem cost
  1269. orderedSpillList.reserve(vregGraph.size());
  1270. for (int graphIdx = 0; graphIdx < (int)vregGraph.size(); graphIdx++)
  1271. {
  1272. int vregIdx = vregGraph[graphIdx];
  1273. if (vregIdx == -1)
  1274. continue;
  1275. auto vregInfo = mContext->mVRegInfo[vregIdx];
  1276. if (vregInfo->mForceReg)
  1277. continue;
  1278. orderedSpillList.push_back(vregIdx);
  1279. }
  1280. std::sort(orderedSpillList.begin(), orderedSpillList.end(), [&](int lhs, int rhs)
  1281. {
  1282. return mNodes[lhs].mMemCost > mNodes[rhs].mMemCost;
  1283. });
  1284. }
  1285. /*int bestSpillVReg = -1;
  1286. int bestSpillCost = INT_MAX;
  1287. // We need to spill now! Try to spill something with the fewest references
  1288. for (int graphIdx = 0; graphIdx < (int)vregGraph.size(); graphIdx++)
  1289. {
  1290. int vregIdx = vregGraph[graphIdx];
  1291. auto vregInfo = mContext->mVRegInfo[vregIdx];
  1292. if (vregInfo->mForceReg)
  1293. continue;
  1294. if (vregInfo->mRefCount < bestSpillCost)
  1295. {
  1296. bestSpillCost = vregInfo->mRefCount;
  1297. bestSpillVReg = vregIdx;
  1298. }
  1299. }*/
  1300. if (bestSpillVReg != -1)
  1301. {
  1302. auto node = &mNodes[bestSpillVReg];
  1303. node->mSpilled = true;
  1304. auto spillVRegInfo = mContext->mVRegInfo[bestSpillVReg];
  1305. spillVRegInfo->mSpilled = true;
  1306. }
  1307. else
  1308. {
  1309. mContext->Fail("Unable to spill vreg");
  1310. }
  1311. }
  1312. }
  1313. }
  1314. SizedArray<bool, 32> globalRegUsedVec;
  1315. globalRegUsedVec.resize(highestReg + 1);
  1316. SizedArray<bool, 32> regUsedVec;
  1317. regUsedVec.resize(highestReg + 1);
  1318. /*String dbgStr;
  1319. if (mContext->mDebugging)
  1320. dbgStr += "AssignRegs ";*/
  1321. BP_ZONE("AssignRegs:assign");
  1322. while (vregStack.size() > 0)
  1323. {
  1324. int vregIdx = vregStack.back();
  1325. vregStack.pop_back();
  1326. /*if (mContext->mDebugging)
  1327. {
  1328. dbgStr += StrFormat("VReg %d ", vregIdx);
  1329. }*/
  1330. BeMCVRegInfo* vregInfo = mContext->mVRegInfo[vregIdx];
  1331. Node* node = &mNodes[vregIdx];
  1332. if (vregInfo->mVRegAffinity != -1)
  1333. {
  1334. auto affinityVRegInfo = mContext->mVRegInfo[vregInfo->mVRegAffinity];
  1335. if (affinityVRegInfo->mReg != X64Reg_None)
  1336. node->AdjustRegCost(affinityVRegInfo->mReg, -2);
  1337. }
  1338. for (int i = 0; i <= highestReg; i++)
  1339. regUsedVec[i] = false;
  1340. if (mReserveParamRegs)
  1341. {
  1342. // This is a fallback case for when the "streams get crossed" during initialization-
  1343. // IE: when arg0 gets assigned to RDX and arg1 gets assigned to RCX and we end up with:
  1344. // MOV arg0<RDX>, RCX
  1345. // MOV arg1<RCX>, RDX
  1346. // Which is bad.
  1347. for (auto reg : mContext->mParamsUsedRegs)
  1348. {
  1349. if (((int)reg < regUsedVec.size()) && (reg != vregInfo->mNaturalReg))
  1350. regUsedVec[(int)reg] = true;
  1351. }
  1352. }
  1353. for (auto connNodeIdx : node->mEdges)
  1354. {
  1355. Node* connNode = &mNodes[connNodeIdx];
  1356. if (connNode->mInGraph)
  1357. {
  1358. auto connVRegInfo = mContext->mVRegInfo[connNodeIdx];
  1359. auto usedReg = mContext->GetFullRegister(connVRegInfo->mReg);
  1360. BF_ASSERT(usedReg != X64Reg_None);
  1361. regUsedVec[(int)usedReg] = true;
  1362. }
  1363. }
  1364. auto bestReg = X64Reg_None;
  1365. int bestRegCost = 0x07FFFFFF; // 0x0FFFFFFF is considered illegal for a reg, so set the mem cost to lower than that...;
  1366. // This is the cost of just leaving the vreg as a memory access. In cases where we bind to a volatile
  1367. // register, we need to consider the cost of preserving and restoring that register across calls, so
  1368. // it cases where we have just a few accesses to this vreg but it spans a lot of calls then we just
  1369. // leave it as memory
  1370. if (!vregInfo->mForceReg)
  1371. bestRegCost = node->mMemCost;
  1372. //for (auto validReg : validRegs)
  1373. int validRegCount = (int)validRegs.size();
  1374. for (int regIdx = 0; regIdx < validRegCount; regIdx++)
  1375. {
  1376. auto validReg = validRegs[regIdx];
  1377. if (!regUsedVec[(int)validReg])
  1378. {
  1379. int checkCost = node->mRegCost[(int)validReg];
  1380. // If this register is non-volatile then we'd have to save and restore it, which costs... unless
  1381. // some other vreg has already used this, then there's no additional cost
  1382. if ((!globalRegUsedVec[(int)validReg]) && (!mContext->IsVolatileReg(validReg)))
  1383. {
  1384. int costMult = 1;
  1385. if (regKind == BeMCColorizer::RegKind_Floats)
  1386. costMult = 2;
  1387. checkCost += 7 * costMult;
  1388. }
  1389. if (checkCost < bestRegCost)
  1390. {
  1391. // Try not to use registers that other params may want
  1392. for (auto argReg : mContext->mParamsUsedRegs)
  1393. {
  1394. if (validReg == argReg)
  1395. checkCost += 1;
  1396. }
  1397. }
  1398. /*if (mContext->mDebugging)
  1399. {
  1400. dbgStr += StrFormat("Cost %d:%d ", validReg, checkCost);
  1401. }*/
  1402. if (checkCost < bestRegCost)
  1403. {
  1404. bestReg = validReg;
  1405. bestRegCost = checkCost;
  1406. }
  1407. }
  1408. }
  1409. if (mContext->mDebugging)
  1410. {
  1411. //auto itr = mContext->mDbgPreferredRegs.find(vregIdx);
  1412. //if (itr != mContext->mDbgPreferredRegs.end())
  1413. X64CPURegister* regPtr = NULL;
  1414. if (mContext->mDbgPreferredRegs.TryGetValue(vregIdx, &regPtr))
  1415. {
  1416. auto reg = *regPtr;
  1417. if (reg == X64Reg_None)
  1418. {
  1419. if (!vregInfo->mForceReg)
  1420. bestReg = reg;
  1421. }
  1422. else
  1423. {
  1424. if (!regUsedVec[(int)reg])
  1425. bestReg = reg;
  1426. }
  1427. }
  1428. }
  1429. if (vregInfo->mSpilled)
  1430. {
  1431. if (bestReg != X64Reg_None)
  1432. {
  1433. // We managed to optimistically unspill
  1434. vregInfo->mSpilled = false;
  1435. }
  1436. }
  1437. else
  1438. {
  1439. if (vregInfo->mForceReg)
  1440. {
  1441. // We didn't end up with a usable reg -- steal a reg from one of our edges
  1442. if (bestReg == X64Reg_None)
  1443. {
  1444. int bestSpillVReg = -1;
  1445. int bestSpillCost = INT_MAX;
  1446. for (auto connNodeIdx : node->mEdges)
  1447. {
  1448. Node* connNode = &mNodes[connNodeIdx];
  1449. if (connNode->mInGraph)
  1450. {
  1451. auto connVRegInfo = mContext->mVRegInfo[connNodeIdx];
  1452. auto usedReg = mContext->GetFullRegister(connVRegInfo->mReg);
  1453. if (connVRegInfo->mForceReg)
  1454. continue;
  1455. if (node->mRegCost[usedReg] < 0x07FFFFFFF)
  1456. {
  1457. if (connVRegInfo->mRefCount < bestSpillCost)
  1458. {
  1459. bestSpillCost = connVRegInfo->mRefCount;
  1460. bestSpillVReg = connNodeIdx;
  1461. }
  1462. }
  1463. }
  1464. }
  1465. if (bestSpillVReg != -1)
  1466. {
  1467. /*if (mContext->mDebugging)
  1468. {
  1469. dbgStr += StrFormat("StealingVReg %d ", bestSpillVReg);
  1470. }*/
  1471. auto connNode = &mNodes[bestSpillVReg];
  1472. auto connVRegInfo = mContext->mVRegInfo[bestSpillVReg];
  1473. bestReg = mContext->GetFullRegister(connVRegInfo->mReg);
  1474. connVRegInfo->mReg = X64Reg_None;
  1475. connNode->mSpilled = true;
  1476. connVRegInfo->mSpilled = true;
  1477. connNode->mInGraph = false;
  1478. for (auto connNodeEdgeIdx : connNode->mEdges)
  1479. {
  1480. Node* connNodeEdge = &mNodes[connNodeEdgeIdx];
  1481. connNodeEdge->mGraphEdgeCount--;
  1482. }
  1483. // We insert spills at the front so we can try to "optimistically" unspill them
  1484. // after all the definite coloring is done
  1485. vregStack.insert(vregStack.begin(), bestSpillVReg);
  1486. }
  1487. }
  1488. // The only way we should have failed to allocate a register is if we spilled...
  1489. BF_ASSERT((bestReg != X64Reg_None) /*|| (mReserveParamRegs)*/);
  1490. }
  1491. }
  1492. vregInfo->mReg = mContext->ResizeRegister(bestReg, vregInfo->mType);
  1493. if (bestReg != X64Reg_None)
  1494. {
  1495. node->mInGraph = true;
  1496. globalRegUsedVec[(int)bestReg] = true;
  1497. for (auto connNodeIdx : node->mEdges)
  1498. {
  1499. Node* connNode = &mNodes[connNodeIdx];
  1500. connNode->mGraphEdgeCount++;
  1501. }
  1502. }
  1503. /*if (mContext->mDebugging)
  1504. {
  1505. dbgStr += StrFormat("Reg %d ", vregInfo->mReg);
  1506. }*/
  1507. }
  1508. /*if (!dbgStr.empty())
  1509. {
  1510. dbgStr += "\n\n";
  1511. OutputDebugStr(dbgStr);
  1512. }*/
  1513. /*bool dumpStats = true;
  1514. if (dumpStats)
  1515. {
  1516. String str;
  1517. str += "Register costs:\n";
  1518. str += " Mem ";
  1519. for (int regNum = 0; regNum <= X64Reg_EDI; regNum++)
  1520. {
  1521. str += StrFormat("%4s", X64CPURegisters::GetRegisterName((X64CPURegister)regNum));
  1522. }
  1523. str += "\n";
  1524. for (int liveVRregIdx = 0; liveVRregIdx < mContext->mVRegInfo.size(); liveVRregIdx++)
  1525. {
  1526. auto vregInfo = mContext->mVRegInfo[liveVRregIdx];
  1527. auto node = &mNodes[liveVRregIdx];
  1528. String name = mContext->ToString(BeMCOperand::FromVReg(liveVRregIdx));
  1529. str += StrFormat("%24s", name.c_str());
  1530. str += StrFormat(" %3d", node->mMemCost);
  1531. for (int regNum = 0; regNum <= X64Reg_EDI; regNum++)
  1532. {
  1533. str += StrFormat(" %3d", node->mRegCost[regNum]);
  1534. }
  1535. str += "\n";
  1536. }
  1537. OutputDebugStr(str);
  1538. }*/
  1539. }
  1540. bool BeMCColorizer::Validate()
  1541. {
  1542. #ifdef _DEBUG
  1543. auto paramsLeft = mContext->mParamsUsedRegs;
  1544. for (auto mcBlock : mContext->mBlocks)
  1545. {
  1546. for (auto inst : mcBlock->mInstructions)
  1547. {
  1548. if (paramsLeft.size() == 0)
  1549. break;
  1550. if (inst->IsMov())
  1551. {
  1552. if (inst->mArg1.IsNativeReg())
  1553. {
  1554. BF_ASSERT(inst->mArg0.IsVReg());
  1555. BF_ASSERT(mContext->GetFullRegister(inst->mArg1.mReg) == paramsLeft[0]);
  1556. paramsLeft.erase(paramsLeft.begin());
  1557. auto vregInfo = mContext->mVRegInfo[inst->mArg0.mVRegIdx];
  1558. if (vregInfo->mReg != X64Reg_None)
  1559. {
  1560. auto checkReg = mContext->GetFullRegister(vregInfo->mReg);
  1561. //auto itr = std::find(paramsLeft.begin(), paramsLeft.end(), checkReg);
  1562. //if (itr != paramsLeft.end())
  1563. if (paramsLeft.Contains(checkReg))
  1564. {
  1565. // This will happen if we have assigned a 'wrong' register to a parameter-
  1566. // a register that is not the 'natural' register (ie: rcx for param0),
  1567. // but it's a register that is already bound to another parameter which
  1568. // will be needed later. Given the 'natural' order of RCX, RDX, R8, R9,
  1569. // This is a valid order still: R10, RCX, R8, R9 since RCX will get written
  1570. // to R10 first so it won't get clobbered, but: RDX, R10, R8, R9 is not
  1571. // valid because RDX gets clobbered before it can be written to R10.
  1572. return false;
  1573. }
  1574. }
  1575. }
  1576. }
  1577. }
  1578. }
  1579. #endif
  1580. return true;
  1581. }
  1582. //////////////////////////////////////////////////////////////////////////
  1583. BeMCLoopDetector::BeMCLoopDetector(BeMCContext* context) : mTrackingContext(context)
  1584. {
  1585. mMCContext = context;
  1586. mTrackingContext.mTrackKindCount = 1;
  1587. mTrackingContext.Init((int)mMCContext->mMCBlockAlloc.size());
  1588. mNodes.Resize(mMCContext->mMCBlockAlloc.size());
  1589. }
  1590. void BeMCLoopDetector::DetectLoops(BeMCBlock* mcBlock, BeVTrackingList* predBlocksSeen)
  1591. {
  1592. mMCContext->mDetectLoopIdx++;
  1593. auto node = &mNodes[mcBlock->mBlockIdx];
  1594. auto blocksSeen = mTrackingContext.Merge(node->mPredBlocksSeen, predBlocksSeen);
  1595. if (blocksSeen == node->mPredBlocksSeen)
  1596. return;
  1597. node->mPredBlocksSeen = blocksSeen;
  1598. //SizedArray<int, 2> addVec = { mcBlock->mBlockIdx };
  1599. //auto newBlocksSeen = mTrackingContext.Add(blocksSeen, addVec, false);
  1600. auto newBlocksSeen = mTrackingContext.Add(blocksSeen, mcBlock->mBlockIdx, false);
  1601. if (newBlocksSeen == blocksSeen)
  1602. {
  1603. // Our ID was already set, so this is a re-entry and thus we are looped
  1604. mcBlock->mIsLooped = true;
  1605. }
  1606. blocksSeen = newBlocksSeen;
  1607. for (auto succ : mcBlock->mSuccs)
  1608. {
  1609. DetectLoops(succ, blocksSeen);
  1610. }
  1611. }
  1612. void BeMCLoopDetector::DetectLoops()
  1613. {
  1614. for (auto block : mMCContext->mBlocks)
  1615. {
  1616. for (auto succ : block->mSuccs)
  1617. {
  1618. if (succ->mBlockIdx < block->mBlockIdx)
  1619. {
  1620. auto prevBlock = mMCContext->mBlocks[block->mBlockIdx - 1];
  1621. //if ((!succ->mIsLooped) || (!prevBlock->mIsLooped))
  1622. {
  1623. for (int setBlockIdx = succ->mBlockIdx; setBlockIdx < block->mBlockIdx; setBlockIdx++)
  1624. mMCContext->mBlocks[setBlockIdx]->mIsLooped = true;
  1625. }
  1626. }
  1627. }
  1628. }
  1629. }
  1630. //////////////////////////////////////////////////////////////////////////
  1631. void BeMCBlock::AddPred(BeMCBlock* pred)
  1632. {
  1633. if (!mPreds.Contains(pred))
  1634. {
  1635. pred->mSuccs.push_back(this);
  1636. mPreds.push_back(pred);
  1637. }
  1638. }
  1639. int BeMCBlock::FindLabelInstIdx(int labelIdx)
  1640. {
  1641. for (int instIdx = 0; instIdx < (int)mInstructions.size(); instIdx++)
  1642. {
  1643. auto inst = mInstructions[instIdx];
  1644. if ((inst->mKind == BeMCInstKind_Label) && (inst->mArg0.mLabelIdx == labelIdx))
  1645. return instIdx;
  1646. }
  1647. return -1;
  1648. }
  1649. //////////////////////////////////////////////////////////////////////////
  1650. void BeInstEnumerator::Next()
  1651. {
  1652. if (mRemoveCurrent)
  1653. {
  1654. if ((mWriteIdx > 0) && (mReadIdx < mBlock->mInstructions.size() - 1))
  1655. mContext->MergeInstFlags(mBlock->mInstructions[mWriteIdx - 1], mBlock->mInstructions[mReadIdx], mBlock->mInstructions[mReadIdx + 1]);
  1656. mBlock->mInstructions[mReadIdx] = NULL;
  1657. mRemoveCurrent = false;
  1658. }
  1659. else
  1660. {
  1661. if (mWriteIdx != mReadIdx)
  1662. {
  1663. mBlock->mInstructions[mWriteIdx] = mBlock->mInstructions[mReadIdx];
  1664. mBlock->mInstructions[mReadIdx] = NULL;
  1665. }
  1666. mWriteIdx++;
  1667. }
  1668. mReadIdx++;
  1669. }
  1670. //////////////////////////////////////////////////////////////////////////
  1671. BeMCContext::BeMCContext(BeCOFFObject* coffObject) : mOut(coffObject->mTextSect.mData), mLivenessContext(this), mVRegInitializedContext(this), mColorizer(this)
  1672. {
  1673. mLivenessContext.mTrackKindCount = 1;
  1674. mCOFFObject = coffObject;
  1675. mModule = NULL;
  1676. mBeFunction = NULL;
  1677. mActiveBeBlock = NULL;
  1678. mActiveBlock = NULL;
  1679. mActiveInst = NULL;
  1680. mDbgFunction = NULL;
  1681. mCompositeRetVRegIdx = -1;
  1682. mTLSVRegIdx = -1;
  1683. mStackSize = 0;
  1684. mCurLabelIdx = 0;
  1685. mCurPhiIdx = 0;
  1686. mMaxCallParamCount = -1;
  1687. mCurDbgLoc = NULL;
  1688. mCurVRegsInit = NULL;
  1689. mCurVRegsLive = NULL;
  1690. mUseBP = false;
  1691. mHasVAStart = false;
  1692. mInsertInstIdxRef = NULL;
  1693. mNativeIntType = mCOFFObject->mBeModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  1694. mDebugging = false;
  1695. mFailed = false;
  1696. mDetectLoopIdx = 0;
  1697. mLegalizationIterations = 0;
  1698. }
  1699. void BeMCContext::NotImpl()
  1700. {
  1701. Fail("Not implemented");
  1702. }
  1703. void BeMCContext::Fail(const StringImpl& str)
  1704. {
  1705. String errStr = StrFormat("Failure during codegen of %s in %s: %s", mBeFunction->mName.c_str(), mModule->mModuleName.c_str(), str.c_str());
  1706. if (mActiveBlock != NULL)
  1707. errStr += StrFormat("\n MCBlock: %s", mActiveBlock->mName.c_str());
  1708. if ((mActiveInst != NULL) && (mActiveInst->mDbgLoc != NULL))
  1709. {
  1710. BeDumpContext dumpCtx;
  1711. errStr += "\n DbgLoc : ";
  1712. dumpCtx.ToString(errStr, mActiveInst->mDbgLoc);
  1713. }
  1714. BfpSystem_FatalError(errStr.c_str(), "FATAL ERROR");
  1715. }
  1716. void BeMCContext::AssertFail(const StringImpl& str, int line)
  1717. {
  1718. Fail(StrFormat("Assert '%s' failed on line %d", str.c_str(), line));
  1719. }
  1720. void BeMCContext::SoftFail(const StringImpl& str, BeDbgLoc* dbgLoc)
  1721. {
  1722. if (mFailed)
  1723. return;
  1724. mFailed = true;
  1725. String errStr = StrFormat("Failure during codegen of %s: %s", mBeFunction->mName.c_str(), str.c_str());
  1726. if (dbgLoc != NULL)
  1727. {
  1728. auto dbgFile = dbgLoc->GetDbgFile();
  1729. if (dbgFile != NULL)
  1730. errStr += StrFormat(" at line %d:%d in %s/%s", dbgLoc->mLine + 1, dbgLoc->mColumn + 1, dbgFile->mDirectory.c_str(), dbgFile->mFileName.c_str());
  1731. }
  1732. mModule->mBeIRCodeGen->Fail(errStr);
  1733. }
  1734. String BeMCContext::ToString(const BeMCOperand& operand)
  1735. {
  1736. if (operand.mKind == BeMCOperandKind_NativeReg)
  1737. {
  1738. return String("%") + X64CPURegisters::GetRegisterName((int)operand.mReg);
  1739. }
  1740. if (operand.IsVRegAny())
  1741. {
  1742. auto vregInfo = GetVRegInfo(operand);
  1743. String str;
  1744. mModule->ToString(str, GetType(operand));
  1745. str += " ";
  1746. if (operand.IsVRegAny())
  1747. {
  1748. auto vregInfo = GetVRegInfo(operand);
  1749. if (operand.mKind == BeMCOperandKind_VRegAddr)
  1750. str += "&";
  1751. if (operand.mKind == BeMCOperandKind_VRegLoad)
  1752. str += "*";
  1753. if (vregInfo->mDbgVariable != NULL)
  1754. str += "#" + vregInfo->mDbgVariable->mName + StrFormat("/%d", operand.mVRegIdx);
  1755. else
  1756. str += StrFormat("%%vreg%d", operand.mVRegIdx);
  1757. }
  1758. if (vregInfo->mReg != X64Reg_None)
  1759. {
  1760. str += "<";
  1761. str += X64CPURegisters::GetRegisterName((int)vregInfo->mReg);
  1762. str += ">";
  1763. }
  1764. else if (vregInfo->mForceReg)
  1765. {
  1766. str += "<reg>";
  1767. }
  1768. if (vregInfo->mDisableR11)
  1769. {
  1770. str += "<NoR11>";
  1771. }
  1772. if (vregInfo->mDisableR12)
  1773. {
  1774. str += "<NoR12>";
  1775. }
  1776. if (vregInfo->mDisableR13)
  1777. {
  1778. str += "<NoR13>";
  1779. }
  1780. if (vregInfo->mDisableRAX)
  1781. {
  1782. str += "<NoRAX>";
  1783. }
  1784. if (vregInfo->mDisableRDX)
  1785. {
  1786. str += "<NoRDX>";
  1787. }
  1788. if (vregInfo->mDisableEx)
  1789. {
  1790. str += "<NoEx>";
  1791. }
  1792. if (vregInfo->mForceMem)
  1793. {
  1794. str += "<mem>";
  1795. }
  1796. if (vregInfo->mIsRetVal)
  1797. {
  1798. str += "<retval>";
  1799. }
  1800. if (vregInfo->mForceMerge)
  1801. {
  1802. str += "<canMerge>";
  1803. }
  1804. if (vregInfo->mRelTo)
  1805. {
  1806. str += "(";
  1807. str += ToString(vregInfo->mRelTo);
  1808. if (vregInfo->mRelOffset)
  1809. {
  1810. str += "+";
  1811. str += ToString(vregInfo->mRelOffset);
  1812. }
  1813. if (vregInfo->mRelOffsetScale != 1)
  1814. str += StrFormat("*%d", vregInfo->mRelOffsetScale);
  1815. str += ")";
  1816. }
  1817. return str;
  1818. }
  1819. if (operand.IsImmediate())
  1820. {
  1821. String str;
  1822. switch (operand.mKind)
  1823. {
  1824. case BeMCOperandKind_Immediate_i8: str += "i8 "; break;
  1825. case BeMCOperandKind_Immediate_i16: str += "i16 "; break;
  1826. case BeMCOperandKind_Immediate_i32: str += "i32 "; break;
  1827. case BeMCOperandKind_Immediate_i64: str += "i64 "; break;
  1828. case BeMCOperandKind_Immediate_HomeSize: str += "homesize"; break;
  1829. case BeMCOperandKind_Immediate_Null:
  1830. if (operand.mType != NULL)
  1831. {
  1832. mModule->ToString(str, operand.mType);
  1833. str += " null";
  1834. }
  1835. else
  1836. str += "null";
  1837. return str;
  1838. case BeMCOperandKind_Immediate_f32: return StrFormat("f32 %f", operand.mImmF32);
  1839. case BeMCOperandKind_Immediate_f32_Packed128: return StrFormat("f32_packed %f", operand.mImmF32);
  1840. case BeMCOperandKind_Immediate_f64: return StrFormat("f64 %f", operand.mImmF64);
  1841. case BeMCOperandKind_Immediate_f64_Packed128: return StrFormat("f64_packed %f", operand.mImmF64);
  1842. }
  1843. //if (operand.mImmediate < 10)
  1844. str += StrFormat("%lld", operand.mImmediate);
  1845. /*else
  1846. str += StrFormat("0x%llX", operand.mImmediate);*/
  1847. return str;
  1848. }
  1849. if (operand.mKind == BeMCOperandKind_Block)
  1850. return "%" + operand.mBlock->mName;
  1851. if (operand.mKind == BeMCOperandKind_Label)
  1852. return StrFormat("%%label%d", operand.mLabelIdx);
  1853. if (operand.mKind == BeMCOperandKind_CmpKind)
  1854. return BeDumpContext::ToString(operand.mCmpKind);
  1855. if (operand.mKind == BeMCOperandKind_CmpResult)
  1856. {
  1857. String result = StrFormat("%%CmpResult%d", operand.mCmpResultIdx);
  1858. auto& cmpResult = mCmpResults[operand.mCmpResultIdx];
  1859. if (cmpResult.mResultVRegIdx != -1)
  1860. result += StrFormat("<vreg%d>", cmpResult.mResultVRegIdx);
  1861. result += " ";
  1862. if (cmpResult.mInverted)
  1863. result += "!";
  1864. result += BeDumpContext::ToString(cmpResult.mCmpKind);
  1865. return result;
  1866. }
  1867. if (operand.mKind == BeMCOperandKind_NotResult)
  1868. {
  1869. auto mcResult = GetOperand(operand.mNotResult->mValue, true, true);
  1870. String result = "NOT ";
  1871. result += ToString(mcResult);
  1872. return result;
  1873. }
  1874. if (operand.mKind == BeMCOperandKind_Symbol)
  1875. return mCOFFObject->mSymbols[operand.mSymbolIdx]->mName;
  1876. if (operand.mKind == BeMCOperandKind_SymbolAddr)
  1877. return "&" + mCOFFObject->mSymbols[operand.mSymbolIdx]->mName;
  1878. if (operand.mKind == BeMCOperandKind_MemSetInfo)
  1879. return StrFormat("size=%d val=%d align=%d", operand.mMemSetInfo.mSize, operand.mMemSetInfo.mValue, operand.mMemSetInfo.mAlign);
  1880. if (operand.mKind == BeMCOperandKind_MemCpyInfo)
  1881. return StrFormat("size=%d align=%d", operand.mMemCpyInfo.mSize, operand.mMemCpyInfo.mAlign);
  1882. if (operand.mKind == BeMCOperandKind_VRegPair)
  1883. {
  1884. String str = "(";
  1885. str += ToString(BeMCOperand::FromEncoded(operand.mVRegPair.mVRegIdx0));
  1886. str += ", ";
  1887. str += ToString(BeMCOperand::FromEncoded(operand.mVRegPair.mVRegIdx1));
  1888. str += ")";
  1889. return str;
  1890. }
  1891. if (operand.mKind == BeMCOperandKind_Phi)
  1892. return StrFormat("%%PHI%d", operand.mPhi->mIdx);
  1893. if (operand.mKind == BeMCOperandKind_PreserveFlag)
  1894. {
  1895. if (operand.mPreserveFlag == BeMCPreserveFlag_NoRestore)
  1896. return "PreserveFlag:NoReturn";
  1897. }
  1898. if (operand.mKind == BeMCOperandKind_ConstAgg)
  1899. {
  1900. BeDumpContext dumpContext;
  1901. String str = "const ";
  1902. str += dumpContext.ToString(operand.mConstant);
  1903. return str;
  1904. }
  1905. return "???";
  1906. }
  1907. BeMCOperand BeMCContext::GetOperand(BeValue* value, bool allowMetaResult, bool allowFail, bool skipForceVRegAddr)
  1908. {
  1909. if (value == NULL)
  1910. return BeMCOperand();
  1911. switch (value->GetTypeId())
  1912. {
  1913. case BeGlobalVariable::TypeId:
  1914. {
  1915. auto globalVar = (BeGlobalVariable*)value;
  1916. if ((globalVar->mIsTLS) && (mTLSVRegIdx == -1))
  1917. {
  1918. auto tlsVReg = AllocVirtualReg(mNativeIntType);
  1919. auto vregInfo = GetVRegInfo(tlsVReg);
  1920. vregInfo->mMustExist = true;
  1921. vregInfo->mForceReg = true;
  1922. vregInfo->mDisableR12 = true;
  1923. vregInfo->mDisableR13 = true;
  1924. mTLSVRegIdx = tlsVReg.mVRegIdx;
  1925. }
  1926. auto sym = mCOFFObject->GetSymbol(globalVar);
  1927. if (sym != NULL)
  1928. {
  1929. BeMCOperand mcOperand;
  1930. mcOperand.mKind = BeMCOperandKind_SymbolAddr;
  1931. mcOperand.mSymbolIdx = sym->mIdx;
  1932. return mcOperand;
  1933. }
  1934. }
  1935. break;
  1936. case BeCastConstant::TypeId:
  1937. {
  1938. auto constant = (BeCastConstant*)value;
  1939. BeMCOperand mcOperand;
  1940. auto relTo = GetOperand(constant->mTarget);
  1941. if (relTo.mKind == BeMCOperandKind_Immediate_Null)
  1942. {
  1943. mcOperand.mKind = BeMCOperandKind_Immediate_Null;
  1944. mcOperand.mType = constant->mType;
  1945. return mcOperand;
  1946. }
  1947. mcOperand = AllocVirtualReg(constant->mType);
  1948. auto vregInfo = GetVRegInfo(mcOperand);
  1949. vregInfo->mDefOnFirstUse = true;
  1950. vregInfo->mRelTo = relTo;
  1951. vregInfo->mIsExpr = true;
  1952. return mcOperand;
  1953. }
  1954. break;
  1955. case BeConstant::TypeId:
  1956. {
  1957. auto constant = (BeConstant*)value;
  1958. BeMCOperand mcOperand;
  1959. switch (constant->mType->mTypeCode)
  1960. {
  1961. case BeTypeCode_Boolean:
  1962. case BeTypeCode_Int8: mcOperand.mKind = BeMCOperandKind_Immediate_i8; break;
  1963. case BeTypeCode_Int16: mcOperand.mKind = BeMCOperandKind_Immediate_i16; break;
  1964. case BeTypeCode_Int32: mcOperand.mKind = BeMCOperandKind_Immediate_i32; break;
  1965. case BeTypeCode_Int64: mcOperand.mKind = BeMCOperandKind_Immediate_i64; break;
  1966. case BeTypeCode_Float:
  1967. mcOperand.mImmF32 = constant->mDouble;
  1968. mcOperand.mKind = BeMCOperandKind_Immediate_f32;
  1969. return mcOperand;
  1970. case BeTypeCode_Double:
  1971. mcOperand.mImmF64 = constant->mDouble;
  1972. mcOperand.mKind = BeMCOperandKind_Immediate_f64;
  1973. return mcOperand;
  1974. case BeTypeCode_Pointer:
  1975. {
  1976. if (constant->mTarget == NULL)
  1977. {
  1978. mcOperand.mKind = BeMCOperandKind_Immediate_Null;
  1979. mcOperand.mType = constant->mType;
  1980. return mcOperand;
  1981. }
  1982. else
  1983. {
  1984. auto relTo = GetOperand(constant->mTarget);
  1985. if (relTo.mKind == BeMCOperandKind_Immediate_Null)
  1986. {
  1987. mcOperand.mKind = BeMCOperandKind_Immediate_Null;
  1988. mcOperand.mType = constant->mType;
  1989. return mcOperand;
  1990. }
  1991. mcOperand = AllocVirtualReg(constant->mType);
  1992. auto vregInfo = GetVRegInfo(mcOperand);
  1993. vregInfo->mDefOnFirstUse = true;
  1994. vregInfo->mRelTo = relTo;
  1995. vregInfo->mIsExpr = true;
  1996. return mcOperand;
  1997. }
  1998. }
  1999. break;
  2000. case BeTypeCode_Struct:
  2001. case BeTypeCode_SizedArray:
  2002. case BeTypeCode_Vector:
  2003. mcOperand.mImmediate = constant->mInt64;
  2004. mcOperand.mKind = BeMCOperandKind_Immediate_i64;
  2005. break;
  2006. default:
  2007. Fail("Unhandled constant type");
  2008. }
  2009. mcOperand.mImmediate = constant->mInt64;
  2010. return mcOperand;
  2011. }
  2012. break;
  2013. case BeStructConstant::TypeId:
  2014. {
  2015. auto structConstant = (BeStructConstant*)value;
  2016. BeMCOperand mcOperand;
  2017. mcOperand.mKind = BeMCOperandKind_ConstAgg;
  2018. mcOperand.mConstant = structConstant;
  2019. return mcOperand;
  2020. }
  2021. break;
  2022. case BeGEP1Constant::TypeId:
  2023. {
  2024. auto gepConstant = (BeGEP1Constant*)value;
  2025. auto mcVal = GetOperand(gepConstant->mTarget);
  2026. BePointerType* ptrType = (BePointerType*)GetType(mcVal);
  2027. BEMC_ASSERT(ptrType->mTypeCode == BeTypeCode_Pointer);
  2028. auto result = mcVal;
  2029. // We assume we never do both an idx0 and idx1 at once. Fix if we change that.
  2030. int byteOffset = 0;
  2031. BeType* elementType = ptrType->mElementType;
  2032. byteOffset += gepConstant->mIdx0 * ptrType->mElementType->GetStride();
  2033. result = AllocRelativeVirtualReg(ptrType, result, GetImmediate(byteOffset), 1);
  2034. // The def is primary to create a single 'master location' for the GEP vreg to become legalized before use
  2035. auto vregInfo = GetVRegInfo(result);
  2036. vregInfo->mDefOnFirstUse = true;
  2037. result.mKind = BeMCOperandKind_VReg;
  2038. return result;
  2039. }
  2040. break;
  2041. case BeGEP2Constant::TypeId:
  2042. {
  2043. auto gepConstant = (BeGEP2Constant*)value;
  2044. auto mcVal = GetOperand(gepConstant->mTarget);
  2045. BePointerType* ptrType = (BePointerType*)GetType(mcVal);
  2046. BEMC_ASSERT(ptrType->mTypeCode == BeTypeCode_Pointer);
  2047. auto result = mcVal;
  2048. // We assume we never do both an idx0 and idx1 at once. Fix if we change that.
  2049. int byteOffset = 0;
  2050. BeType* elementType = NULL;
  2051. byteOffset += gepConstant->mIdx0 * ptrType->mElementType->GetStride();
  2052. if (ptrType->mElementType->mTypeCode == BeTypeCode_Struct)
  2053. {
  2054. BeStructType* structType = (BeStructType*)ptrType->mElementType;
  2055. auto& structMember = structType->mMembers[gepConstant->mIdx1];
  2056. elementType = structMember.mType;
  2057. byteOffset = structMember.mByteOffset;
  2058. }
  2059. else if (ptrType->mElementType->mTypeCode == BeTypeCode_SizedArray)
  2060. {
  2061. BEMC_ASSERT(ptrType->mElementType->mTypeCode == BeTypeCode_SizedArray);
  2062. auto arrayType = (BeSizedArrayType*)ptrType->mElementType;
  2063. elementType = arrayType->mElementType;
  2064. byteOffset = gepConstant->mIdx1 * elementType->GetStride();
  2065. }
  2066. else
  2067. {
  2068. BEMC_ASSERT(ptrType->mElementType->mTypeCode == BeTypeCode_Vector);
  2069. auto arrayType = (BeVectorType*)ptrType->mElementType;
  2070. elementType = arrayType->mElementType;
  2071. byteOffset = gepConstant->mIdx1 * elementType->GetStride();
  2072. }
  2073. auto elementPtrType = mModule->mContext->GetPointerTo(elementType);
  2074. result = AllocRelativeVirtualReg(elementPtrType, result, GetImmediate(byteOffset), 1);
  2075. // The def is primary to create a single 'master location' for the GEP vreg to become legalized before use
  2076. auto vregInfo = GetVRegInfo(result);
  2077. vregInfo->mDefOnFirstUse = true;
  2078. result.mKind = BeMCOperandKind_VReg;
  2079. return result;
  2080. }
  2081. break;
  2082. case BeExtractValueConstant::TypeId:
  2083. {
  2084. // Note: this only handles zero-aggregates
  2085. auto extractConstant = (BeExtractValueConstant*)value;
  2086. auto elementType = extractConstant->GetType();
  2087. auto mcVal = GetOperand(extractConstant->mTarget);
  2088. auto valType = GetType(mcVal);
  2089. BeConstant beConstant;
  2090. beConstant.mType = elementType;
  2091. beConstant.mUInt64 = 0;
  2092. return GetOperand(&beConstant);
  2093. }
  2094. break;
  2095. case BeUndefConstant::TypeId:
  2096. {
  2097. auto undefConstant = (BeUndefConstant*)value;
  2098. BeConstant beConstant;
  2099. beConstant.mType = undefConstant->mType;
  2100. beConstant.mUInt64 = 0;
  2101. return GetOperand(&beConstant);
  2102. }
  2103. break;
  2104. case BeFunction::TypeId:
  2105. {
  2106. auto sym = mCOFFObject->GetSymbol(value);
  2107. BEMC_ASSERT(sym != NULL);
  2108. if (sym != NULL)
  2109. {
  2110. BeMCOperand mcOperand;
  2111. mcOperand.mKind = BeMCOperandKind_SymbolAddr;
  2112. mcOperand.mSymbolIdx = sym->mIdx;
  2113. return mcOperand;
  2114. }
  2115. }
  2116. break;
  2117. case BeCallInst::TypeId:
  2118. {
  2119. auto callInst = (BeCallInst*)value;
  2120. if (callInst->mInlineResult != NULL)
  2121. return GetOperand(callInst->mInlineResult);
  2122. }
  2123. break;
  2124. case BeDbgVariable::TypeId:
  2125. {
  2126. }
  2127. break;
  2128. }
  2129. BeMCOperand* operandPtr = NULL;
  2130. mValueToOperand.TryGetValue(value, &operandPtr);
  2131. //auto itr = mValueToOperand.find(value);
  2132. if (!allowFail)
  2133. {
  2134. if (operandPtr == NULL)
  2135. {
  2136. BeDumpContext dumpCtx;
  2137. String str;
  2138. dumpCtx.ToString(str, value);
  2139. Fail(StrFormat("Unable to find bevalue for operand: %s", str.c_str()));
  2140. }
  2141. }
  2142. if (operandPtr == NULL)
  2143. {
  2144. if (allowFail)
  2145. return BeMCOperand();
  2146. BeMCOperand mcOperand;
  2147. mcOperand.mKind = BeMCOperandKind_Immediate_i64;
  2148. mcOperand.mImmediate = 0;
  2149. return mcOperand;
  2150. }
  2151. auto operand = *operandPtr;
  2152. if ((operand.mKind == BeMCOperandKind_Phi) && (!allowMetaResult))
  2153. {
  2154. auto phi = operand.mPhi;
  2155. int phiInstIdx = 0;
  2156. auto mcBlock = phi->mBlock;
  2157. for (auto instIdx = 0; instIdx < mcBlock->mInstructions.size(); instIdx++)
  2158. {
  2159. auto inst = mcBlock->mInstructions[instIdx];
  2160. if (inst->mKind == BeMCInstKind_DefPhi)
  2161. {
  2162. BEMC_ASSERT(inst->mArg0.mPhi == phi);
  2163. phiInstIdx = instIdx;
  2164. RemoveInst(mcBlock, phiInstIdx);
  2165. break;
  2166. }
  2167. }
  2168. SetAndRestoreValue<BeMCBlock*> prevBlock(mActiveBlock, mcBlock);
  2169. SetAndRestoreValue<int*> prevInstIdxRef(mInsertInstIdxRef, &phiInstIdx);
  2170. auto resultType = value->GetType();
  2171. auto result = AllocVirtualReg(resultType);
  2172. auto vregInfo = GetVRegInfo(result);
  2173. vregInfo->mHasDynLife = true; // No specific 'def' location
  2174. mValueToOperand[value] = result;
  2175. if (resultType->mTypeCode == BeTypeCode_Boolean)
  2176. {
  2177. CreateDefineVReg(result);
  2178. BeMCOperand falseLabel = BeMCOperand::FromLabel(mCurLabelIdx++);
  2179. BeMCOperand trueLabel = BeMCOperand::FromLabel(mCurLabelIdx++);
  2180. BeMCOperand endLabel = BeMCOperand::FromLabel(mCurLabelIdx++);
  2181. CreateCondBr(mActiveBlock, operand, trueLabel, falseLabel);
  2182. AllocInst(BeMCInstKind_Label, falseLabel);
  2183. AllocInst(BeMCInstKind_Mov, result, BeMCOperand::FromImmediate(0));
  2184. AllocInst(BeMCInstKind_Br, endLabel);
  2185. AllocInst(BeMCInstKind_Label, trueLabel);
  2186. AllocInst(BeMCInstKind_Mov, result, BeMCOperand::FromImmediate(1));
  2187. AllocInst(BeMCInstKind_Label, endLabel);
  2188. }
  2189. else
  2190. {
  2191. // Attempt to find common ancestor to insert a 'def' at
  2192. SizedArray<BeMCBlock*, 16> blockSearch;
  2193. blockSearch.reserve(phi->mValues.size());
  2194. BeMCBlock* lowestBlock = NULL;
  2195. for (auto& phiValue : phi->mValues)
  2196. {
  2197. if ((lowestBlock == NULL) || (phiValue.mBlockFrom->mBlockIdx < lowestBlock->mBlockIdx))
  2198. lowestBlock = phiValue.mBlockFrom;
  2199. blockSearch.push_back(phiValue.mBlockFrom);
  2200. }
  2201. while (true)
  2202. {
  2203. bool allMatched = true;
  2204. bool didWork = false;
  2205. for (int searchIdx = 0; searchIdx < (int)blockSearch.size(); searchIdx++)
  2206. {
  2207. auto& blockRef = blockSearch[searchIdx];
  2208. if (blockRef != lowestBlock)
  2209. {
  2210. allMatched = false;
  2211. for (auto& pred : blockRef->mPreds)
  2212. {
  2213. // Try find a block closer to start, but not below the current lowestBlock
  2214. if ((pred->mBlockIdx >= lowestBlock->mBlockIdx) && (pred->mBlockIdx < blockRef->mBlockIdx))
  2215. {
  2216. blockRef = pred;
  2217. didWork = true;
  2218. }
  2219. }
  2220. }
  2221. }
  2222. if (allMatched)
  2223. {
  2224. SetAndRestoreValue<BeMCBlock*> prevActiveBlock(mActiveBlock, lowestBlock);
  2225. SetAndRestoreValue<int*> prevInstIdxRef(mInsertInstIdxRef, NULL);
  2226. auto inst = CreateDefineVReg(result);
  2227. inst->mVRegsInitialized = NULL;
  2228. inst->mDbgLoc = NULL;
  2229. break;
  2230. }
  2231. if (!didWork)
  2232. {
  2233. BeMCBlock* nextLowestBlock = NULL;
  2234. // Find the next candidate block
  2235. for (auto& blockRef : blockSearch)
  2236. {
  2237. for (auto& pred : blockRef->mPreds)
  2238. {
  2239. if (pred->mBlockIdx < lowestBlock->mBlockIdx)
  2240. {
  2241. if ((nextLowestBlock == NULL) || (pred->mBlockIdx > nextLowestBlock->mBlockIdx))
  2242. nextLowestBlock = pred;
  2243. }
  2244. }
  2245. }
  2246. if (nextLowestBlock == NULL)
  2247. break;
  2248. lowestBlock = nextLowestBlock;
  2249. }
  2250. }
  2251. BeMCOperand doneLabel = BeMCOperand::FromLabel(mCurLabelIdx++);
  2252. CreatePhiAssign(mActiveBlock, operand, result, doneLabel);
  2253. // Don't use an explicit dbgLoc
  2254. SetAndRestoreValue<BeDbgLoc*> prevDbgLoc(mCurDbgLoc, NULL);
  2255. AllocInst(BeMCInstKind_Label, doneLabel);
  2256. }
  2257. return result;
  2258. }
  2259. if ((operand.mKind == BeMCOperandKind_CmpResult) && (!allowMetaResult))
  2260. {
  2261. auto& cmpResult = mCmpResults[operand.mCmpResultIdx];
  2262. if (cmpResult.mResultVRegIdx == -1)
  2263. {
  2264. // Create the vreg now, and insert the CmpToBool during legalization
  2265. BeType* boolType = mModule->mContext->GetPrimitiveType(BeTypeCode_Boolean);
  2266. operand = AllocVirtualReg(boolType);
  2267. cmpResult.mResultVRegIdx = operand.mVRegIdx;
  2268. auto vregInfo = GetVRegInfo(operand);
  2269. vregInfo->mDefOnFirstUse = true;
  2270. }
  2271. operand = BeMCOperand::FromVReg(cmpResult.mResultVRegIdx);
  2272. }
  2273. if ((operand.mKind == BeMCOperandKind_NotResult) && (!allowMetaResult))
  2274. {
  2275. auto mcValue = GetOperand(operand.mNotResult->mValue, false, allowFail);
  2276. operand = AllocVirtualReg(GetType(mcValue));
  2277. CreateDefineVReg(operand);
  2278. AllocInst(BeMCInstKind_Mov, operand, mcValue);
  2279. BeMCOperand xorVal;
  2280. xorVal.mKind = BeMCOperandKind_Immediate_i8;
  2281. xorVal.mImmediate = 0x1;
  2282. AllocInst(BeMCInstKind_Xor, operand, xorVal);
  2283. }
  2284. if ((operand.mKind == BeMCOperandKind_VRegAddr) && (!skipForceVRegAddr))
  2285. {
  2286. auto vregInfo = GetVRegInfo(operand);
  2287. if (!vregInfo->mForceMem)
  2288. vregInfo->mForceMem = true;
  2289. }
  2290. return operand;
  2291. }
  2292. BeMCOperand BeMCContext::TryToVector(BeValue* value)
  2293. {
  2294. auto operand = GetOperand(value);
  2295. auto type = GetType(operand);
  2296. if (!type->IsPointer())
  2297. return operand;
  2298. auto pointerType = (BePointerType*)type;
  2299. if (!pointerType->mElementType->IsVector())
  2300. return operand;
  2301. return CreateLoad(operand);
  2302. }
  2303. BeType* BeMCContext::GetType(const BeMCOperand& operand)
  2304. {
  2305. if (operand.mKind == BeMCOperandKind_NativeReg)
  2306. {
  2307. if ((operand.mReg >= X64Reg_RAX) && (operand.mReg <= X64Reg_EFL))
  2308. return mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  2309. if ((operand.mReg >= X64Reg_EAX) && (operand.mReg <= X64Reg_R15D))
  2310. return mModule->mContext->GetPrimitiveType(BeTypeCode_Int32);
  2311. if ((operand.mReg >= X64Reg_AX) && (operand.mReg <= X64Reg_R15W))
  2312. return mModule->mContext->GetPrimitiveType(BeTypeCode_Int16);
  2313. if ((operand.mReg >= X64Reg_AL) && (operand.mReg <= X64Reg_R15B))
  2314. return mModule->mContext->GetPrimitiveType(BeTypeCode_Int8);
  2315. if ((operand.mReg >= X64Reg_XMM0_f64) && (operand.mReg <= X64Reg_XMM15_f64))
  2316. return mModule->mContext->GetPrimitiveType(BeTypeCode_Double);
  2317. if ((operand.mReg >= X64Reg_XMM0_f32) && (operand.mReg <= X64Reg_XMM15_f32))
  2318. return mModule->mContext->GetPrimitiveType(BeTypeCode_Float);
  2319. if ((operand.mReg >= X64Reg_M128_XMM0) && (operand.mReg <= X64Reg_M128_XMM15))
  2320. return mModule->mContext->GetPrimitiveType(BeTypeCode_M128);
  2321. }
  2322. if (operand.mKind == BeMCOperandKind_VReg)
  2323. return mVRegInfo[operand.mVRegIdx]->mType;
  2324. if (operand.mKind == BeMCOperandKind_VRegAddr)
  2325. return mModule->mContext->GetPointerTo(mVRegInfo[operand.mVRegIdx]->mType);
  2326. if (operand.mKind == BeMCOperandKind_VRegLoad)
  2327. {
  2328. auto type = mVRegInfo[operand.mVRegIdx]->mType;
  2329. BEMC_ASSERT(type->IsPointer());
  2330. return ((BePointerType*)type)->mElementType;
  2331. }
  2332. switch (operand.mKind)
  2333. {
  2334. case BeMCOperandKind_Immediate_i8: return mModule->mContext->GetPrimitiveType(BeTypeCode_Int8); break;
  2335. case BeMCOperandKind_Immediate_i16: return mModule->mContext->GetPrimitiveType(BeTypeCode_Int16); break;
  2336. case BeMCOperandKind_Immediate_i32: return mModule->mContext->GetPrimitiveType(BeTypeCode_Int32); break;
  2337. case BeMCOperandKind_Immediate_i64:
  2338. case BeMCOperandKind_Immediate_HomeSize: return mModule->mContext->GetPrimitiveType(BeTypeCode_Int64); break;
  2339. case BeMCOperandKind_Immediate_Null: return operand.mType; break;
  2340. case BeMCOperandKind_Immediate_f32:
  2341. case BeMCOperandKind_Immediate_f32_Packed128: return mModule->mContext->GetPrimitiveType(BeTypeCode_Float); break;
  2342. case BeMCOperandKind_Immediate_f64:
  2343. case BeMCOperandKind_Immediate_f64_Packed128: return mModule->mContext->GetPrimitiveType(BeTypeCode_Double); break;
  2344. }
  2345. if (operand.mKind == BeMCOperandKind_SymbolAddr)
  2346. {
  2347. auto symbol = mCOFFObject->mSymbols[operand.mSymbolIdx];
  2348. if (symbol->mType == NULL)
  2349. return NULL;
  2350. return mModule->mContext->GetPointerTo(symbol->mType);
  2351. }
  2352. if (operand.mKind == BeMCOperandKind_Symbol)
  2353. {
  2354. auto symbol = mCOFFObject->mSymbols[operand.mSymbolIdx];
  2355. return symbol->mType;
  2356. }
  2357. if (operand.mKind == BeMCOperandKind_CmpResult)
  2358. {
  2359. return mModule->mContext->GetPrimitiveType(BeTypeCode_Boolean);
  2360. }
  2361. if (operand.mKind == BeMCOperandKind_ConstAgg)
  2362. {
  2363. return operand.mConstant->mType;
  2364. }
  2365. return NULL;
  2366. }
  2367. bool BeMCContext::AreTypesEquivalent(BeType* type0, BeType* type1)
  2368. {
  2369. if ((type0->IsFloat()) != (type1->IsFloat()))
  2370. return false;
  2371. return type0->mSize == type1->mSize;
  2372. }
  2373. void BeMCContext::AddRelRefs(BeMCOperand& operand, int refCount)
  2374. {
  2375. if (!operand.IsVRegAny())
  2376. return;
  2377. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  2378. vregInfo->mRefCount += refCount;
  2379. if (vregInfo->mRelTo)
  2380. AddRelRefs(vregInfo->mRelTo, vregInfo->mRefCount + refCount);
  2381. if (vregInfo->mRelOffset)
  2382. AddRelRefs(vregInfo->mRelOffset, vregInfo->mRefCount + refCount);
  2383. }
  2384. int BeMCContext::FindSafeInstInsertPos(int instIdx, bool forceSafeCheck)
  2385. {
  2386. auto inst = mActiveBlock->mInstructions[instIdx];
  2387. bool doSafeCheck = false;
  2388. if (forceSafeCheck)
  2389. doSafeCheck = true;
  2390. if ((inst->mKind == BeMCInstKind_Call) ||
  2391. (inst->mKind == BeMCInstKind_MemSet) ||
  2392. (inst->mKind == BeMCInstKind_MemCpy))
  2393. doSafeCheck = true;
  2394. if ((inst->IsMov()) && (inst->mArg0.IsNativeReg()))
  2395. doSafeCheck = true;
  2396. if (!doSafeCheck)
  2397. return instIdx;
  2398. // If we're in a CALL then move this Def before the param loads
  2399. int bestIdx = instIdx;
  2400. while (bestIdx > 0)
  2401. {
  2402. inst = mActiveBlock->mInstructions[bestIdx - 1];
  2403. bestIdx--;
  2404. if (inst->mKind == BeMCInstKind_PreserveVolatiles)
  2405. return bestIdx;
  2406. }
  2407. return bestIdx;
  2408. }
  2409. BeMCInst* BeMCContext::AllocInst(int insertIdx)
  2410. {
  2411. auto mcInst = mAlloc.Alloc<BeMCInst>();
  2412. mcInst->mKind = BeMCInstKind_None;
  2413. mcInst->mDbgLoc = mCurDbgLoc;
  2414. mcInst->mVRegsInitialized = mCurVRegsInit;
  2415. mcInst->mLiveness = mCurVRegsLive;
  2416. if ((insertIdx == -1) && (mInsertInstIdxRef != NULL))
  2417. insertIdx = (*mInsertInstIdxRef)++;
  2418. if (insertIdx == -1)
  2419. mActiveBlock->mInstructions.push_back(mcInst);
  2420. else if ((insertIdx < mActiveBlock->mInstructions.mSize) && (mActiveBlock->mInstructions[insertIdx] == NULL))
  2421. mActiveBlock->mInstructions[insertIdx] = mcInst;
  2422. else
  2423. mActiveBlock->mInstructions.Insert(insertIdx, mcInst);
  2424. return mcInst;
  2425. }
  2426. BeMCInst* BeMCContext::AllocInst(BeMCInstKind instKind, int insertIdx)
  2427. {
  2428. auto mcInst = AllocInst(insertIdx);
  2429. mcInst->mKind = instKind;
  2430. return mcInst;
  2431. }
  2432. BeMCInst* BeMCContext::AllocInst(BeMCInstKind instKind, const BeMCOperand& arg0, int insertIdx)
  2433. {
  2434. auto mcInst = AllocInst(insertIdx);
  2435. mcInst->mKind = instKind;
  2436. mcInst->mArg0 = arg0;
  2437. return mcInst;
  2438. }
  2439. BeMCInst* BeMCContext::AllocInst(BeMCInstKind instKind, const BeMCOperand& arg0, const BeMCOperand& arg1, int insertIdx)
  2440. {
  2441. auto mcInst = AllocInst(insertIdx);
  2442. mcInst->mKind = instKind;
  2443. mcInst->mArg0 = arg0;
  2444. mcInst->mArg1 = arg1;
  2445. return mcInst;
  2446. }
  2447. void BeMCContext::MergeInstFlags(BeMCInst* prevInst, BeMCInst* inst, BeMCInst* nextInst)
  2448. {
  2449. if (prevInst->mVRegsInitialized == inst->mVRegsInitialized)
  2450. return;
  2451. if ((inst->mVRegsInitialized != NULL) && (nextInst->mVRegsInitialized != NULL))
  2452. nextInst->mVRegsInitialized = mVRegInitializedContext.MergeChanges(nextInst->mVRegsInitialized, inst->mVRegsInitialized);
  2453. }
  2454. void BeMCContext::RemoveInst(BeMCBlock* block, int instIdx, bool needChangesMerged, bool removeFromList)
  2455. {
  2456. // If neither the instruction before or after this one shares the vregsInitialized flags, then we need to
  2457. // merge down our Changes to the next instruction
  2458. auto inst = block->mInstructions[instIdx];
  2459. if (instIdx > 0)
  2460. {
  2461. auto prevInst = block->mInstructions[instIdx - 1];
  2462. if (prevInst->mVRegsInitialized == inst->mVRegsInitialized)
  2463. needChangesMerged = false;
  2464. }
  2465. if (needChangesMerged)
  2466. {
  2467. if (instIdx < (int)block->mInstructions.size() - 1)
  2468. {
  2469. auto nextInst = block->mInstructions[instIdx + 1];
  2470. if ((inst->mVRegsInitialized != NULL) && (nextInst->mVRegsInitialized != NULL))
  2471. {
  2472. nextInst->mVRegsInitialized = mVRegInitializedContext.MergeChanges(nextInst->mVRegsInitialized, inst->mVRegsInitialized);
  2473. }
  2474. }
  2475. }
  2476. if (removeFromList)
  2477. block->mInstructions.RemoveAt(instIdx);
  2478. }
  2479. BeMCOperand BeMCContext::GetCallArgVReg(int argIdx, BeTypeCode typeCode)
  2480. {
  2481. int pIdx = argIdx;
  2482. BeMCNativeTypeCode vregType = BeMCNativeTypeCode_Int64;
  2483. switch (typeCode)
  2484. {
  2485. case BeTypeCode_Int8:
  2486. vregType = BeMCNativeTypeCode_Int8;
  2487. break;
  2488. case BeTypeCode_Int16:
  2489. vregType = BeMCNativeTypeCode_Int16;
  2490. break;
  2491. case BeTypeCode_Int32:
  2492. vregType = BeMCNativeTypeCode_Int32;
  2493. break;
  2494. case BeTypeCode_Int64:
  2495. vregType = BeMCNativeTypeCode_Int64;
  2496. break;
  2497. case BeTypeCode_Float:
  2498. vregType = BeMCNativeTypeCode_Float;
  2499. break;
  2500. case BeTypeCode_Double:
  2501. vregType = BeMCNativeTypeCode_Double;
  2502. break;
  2503. case BeTypeCode_M128:
  2504. vregType = BeMCNativeTypeCode_M128;
  2505. break;
  2506. case BeTypeCode_M256:
  2507. vregType = BeMCNativeTypeCode_M256;
  2508. break;
  2509. case BeTypeCode_M512:
  2510. vregType = BeMCNativeTypeCode_M512;
  2511. break;
  2512. default:
  2513. typeCode = BeTypeCode_Int64;
  2514. vregType = BeMCNativeTypeCode_Int64;
  2515. break;
  2516. }
  2517. Array<int>& callArgVRegs = mCallArgVRegs[vregType];
  2518. while (pIdx >= (int)callArgVRegs.size())
  2519. callArgVRegs.push_back(-1);
  2520. if (callArgVRegs[pIdx] == -1)
  2521. {
  2522. auto nativeType = mModule->mContext->GetPrimitiveType(typeCode);
  2523. auto nativePtrType = mModule->mContext->GetPointerTo(nativeType);
  2524. auto mcArg = AllocVirtualReg(nativePtrType);
  2525. auto vregInfo = mVRegInfo[mcArg.mVRegIdx];
  2526. vregInfo->mMustExist = true;
  2527. vregInfo->mIsExpr = true;
  2528. vregInfo->mRelTo = BeMCOperand::FromReg(X64Reg_RSP);
  2529. vregInfo->mRelOffset = BeMCOperand::FromImmediate(argIdx * 8);
  2530. callArgVRegs[pIdx] = mcArg.mVRegIdx;
  2531. mcArg.mKind = BeMCOperandKind_VRegLoad;
  2532. return mcArg;
  2533. }
  2534. else
  2535. {
  2536. auto mcArg = BeMCOperand::FromVReg(callArgVRegs[pIdx]);
  2537. mcArg.mKind = BeMCOperandKind_VRegLoad;
  2538. return mcArg;
  2539. }
  2540. }
  2541. BeMCOperand BeMCContext::CreateCall(const BeMCOperand &func, const SizedArrayImpl<BeValue*>& args, BeType* retType, BfIRCallingConv callingConv, bool structRet, bool noReturn, int varArgStart)
  2542. {
  2543. SizedArray<BeMCOperand, 4> opArgs;
  2544. for (auto itr = args.begin(); itr != args.end(); ++itr)
  2545. {
  2546. auto& arg = *itr;
  2547. opArgs.push_back(GetOperand(arg));
  2548. }
  2549. return CreateCall(func, opArgs, retType, callingConv, structRet, noReturn, varArgStart);
  2550. }
  2551. BeMCOperand BeMCContext::CreateLoad(const BeMCOperand& mcTarget)
  2552. {
  2553. if (mcTarget.mKind == BeMCOperandKind_Immediate_Null)
  2554. {
  2555. auto fakeType = GetType(mcTarget);
  2556. auto fakePtr = AllocVirtualReg(fakeType);
  2557. CreateDefineVReg(fakePtr);
  2558. AllocInst(BeMCInstKind_Mov, fakePtr, BeMCOperand::FromImmediate(0));
  2559. return CreateLoad(fakePtr);
  2560. }
  2561. if (HasImmediateTarget(mcTarget))
  2562. {
  2563. BeMCOperand scratchReg = AllocVirtualReg(GetType(mcTarget), 2);
  2564. CreateDefineVReg(scratchReg);
  2565. AllocInst(BeMCInstKind_Mov, scratchReg, mcTarget);
  2566. return CreateLoad(scratchReg);
  2567. }
  2568. BeMCOperand result;
  2569. auto loadedTarget = BeMCOperand::ToLoad(mcTarget);
  2570. auto targetType = GetType(loadedTarget);
  2571. result = AllocVirtualReg(targetType);
  2572. auto vregInfo = GetVRegInfo(result);
  2573. vregInfo->mIsExpr = true;
  2574. vregInfo->mRelTo = loadedTarget;
  2575. result.mKind = BeMCOperandKind_VReg;
  2576. AllocInst(BeMCInstKind_DefLoad, result);
  2577. return result;
  2578. }
  2579. static bool NeedsDecompose(BeConstant* constant)
  2580. {
  2581. if (auto arrayConst = BeValueDynCast<BeStructConstant>(constant))
  2582. {
  2583. for (auto& val : arrayConst->mMemberValues)
  2584. {
  2585. if (NeedsDecompose(val))
  2586. return true;
  2587. }
  2588. return false;
  2589. }
  2590. if (auto globalVar = BeValueDynCast<BeGlobalVariable>(constant))
  2591. {
  2592. return true;
  2593. }
  2594. else if (auto castConst = BeValueDynCast<BeCastConstant>(constant))
  2595. {
  2596. return true;
  2597. }
  2598. else if (auto castConst = BeValueDynCast<BeBitCastInst>(constant))
  2599. {
  2600. if (auto targetConstant = BeValueDynCast<BeConstant>(castConst->mValue))
  2601. return NeedsDecompose(targetConstant);
  2602. }
  2603. else if (auto castConst = BeValueDynCast<BeGEP1Constant>(constant))
  2604. {
  2605. return NeedsDecompose(castConst->mTarget);
  2606. }
  2607. else if (auto castConst = BeValueDynCast<BeGEP2Constant>(constant))
  2608. {
  2609. return NeedsDecompose(castConst->mTarget);
  2610. }
  2611. return false;
  2612. }
  2613. void BeMCContext::CreateStore(BeMCInstKind instKind, const BeMCOperand& val, const BeMCOperand& ptr)
  2614. {
  2615. BeMCOperand mcVal = val;
  2616. BeMCOperand mcPtr = ptr;
  2617. if (mcVal.mKind == BeMCOperandKind_ConstAgg)
  2618. {
  2619. if (auto aggConst = BeValueDynCast<BeStructConstant>(mcVal.mConstant))
  2620. {
  2621. if (NeedsDecompose(mcVal.mConstant))
  2622. {
  2623. int offset = 0;
  2624. auto aggType = aggConst->GetType();
  2625. for (int memberIdx = 0; memberIdx < (int)aggConst->mMemberValues.size(); memberIdx++)
  2626. {
  2627. auto val = aggConst->mMemberValues[memberIdx];
  2628. BeType* elemType = NULL;
  2629. if (aggType->IsSizedArray())
  2630. elemType = ((BeSizedArrayType*)aggType)->mElementType;
  2631. else
  2632. {
  2633. auto& memberInfo = ((BeStructType*)aggType)->mMembers[memberIdx];
  2634. offset = memberInfo.mByteOffset;
  2635. elemType = memberInfo.mType;
  2636. }
  2637. if (elemType->mSize == 0)
  2638. continue;
  2639. auto destOperand = AllocVirtualReg(mModule->mContext->GetPointerTo(elemType));
  2640. auto vregInfo = GetVRegInfo(destOperand);
  2641. vregInfo->mDefOnFirstUse = true;
  2642. vregInfo->mRelTo = mcPtr;
  2643. vregInfo->mIsExpr = true;
  2644. vregInfo->mRelOffset = BeMCOperand::FromImmediate(offset);
  2645. //destOperand.mKind = BeMCOperandKind_VRegLoad;
  2646. auto elementVal = GetOperand(val);
  2647. //AllocInst(instKind, destOperand, elementVal);
  2648. CreateStore(instKind, elementVal, destOperand);
  2649. offset += elemType->GetStride();
  2650. }
  2651. return;
  2652. }
  2653. }
  2654. }
  2655. // Addr mov infers something like a "int* intPtr = &intVal"
  2656. if (mcVal.mKind == BeMCOperandKind_VRegAddr)
  2657. {
  2658. auto vregInfo = GetVRegInfo(mcVal);
  2659. vregInfo->mForceMem = true;
  2660. CheckForce(vregInfo);
  2661. }
  2662. if (mcPtr.mKind == BeMCOperandKind_VRegAddr)
  2663. {
  2664. mcPtr.mKind = BeMCOperandKind_VReg;
  2665. AllocInst(instKind, mcPtr, mcVal);
  2666. }
  2667. else if (mcPtr.mKind == BeMCOperandKind_VReg)
  2668. {
  2669. mcPtr.mKind = BeMCOperandKind_VRegLoad;
  2670. AllocInst(instKind, mcPtr, mcVal);
  2671. }
  2672. else if (mcPtr.mKind == BeMCOperandKind_SymbolAddr)
  2673. {
  2674. mcPtr.mKind = BeMCOperandKind_Symbol;
  2675. AllocInst(instKind, mcPtr, mcVal);
  2676. }
  2677. else
  2678. {
  2679. mcPtr = CreateLoad(mcPtr);
  2680. AllocInst(instKind, mcPtr, mcVal);
  2681. }
  2682. }
  2683. BeMCOperand BeMCContext::CreateCall(const BeMCOperand& func, const SizedArrayImpl<BeMCOperand>& args, BeType* retType, BfIRCallingConv callingConv, bool structRet, bool noReturn, int varArgStart)
  2684. {
  2685. BeMCOperand mcResult;
  2686. //TODO: Allow user to directly specify ret addr with "sret" attribute
  2687. int argOfs = 0;
  2688. X64CPURegister compositeRetReg = X64Reg_None;
  2689. bool flipFirstRegs = (structRet) && (callingConv == BfIRCallingConv_ThisCall);
  2690. if ((retType != NULL) && (retType->IsNonVectorComposite()))
  2691. {
  2692. mcResult = AllocVirtualReg(retType);
  2693. auto vregInfo = GetVRegInfo(mcResult);
  2694. vregInfo->mMustExist = true;
  2695. CreateDefineVReg(mcResult);
  2696. // 'this' always goes in RCX, so push compositeRetReg out by one
  2697. compositeRetReg = (callingConv == BfIRCallingConv_ThisCall) ? X64Reg_RDX : X64Reg_RCX;
  2698. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(compositeRetReg), BeMCOperand::FromVRegAddr(mcResult.mVRegIdx));
  2699. argOfs = 1;
  2700. }
  2701. bool didPreserveRegs = false;
  2702. auto _AddPreserveRegs = [&]()
  2703. {
  2704. if (noReturn)
  2705. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromPreserveFlag(BeMCPreserveFlag_NoRestore));
  2706. else
  2707. AllocInst(BeMCInstKind_PreserveVolatiles);
  2708. didPreserveRegs = true;
  2709. };
  2710. int argCount = (int)args.size() + argOfs;
  2711. int dynStackSize = 0;
  2712. if (dynStackSize > 0)
  2713. AllocInst(BeMCInstKind_Sub, BeMCOperand::FromReg(X64Reg_RSP), BeMCOperand::FromImmediate(dynStackSize));
  2714. struct _ShadowReg
  2715. {
  2716. X64CPURegister mFReg;
  2717. X64CPURegister mIReg;
  2718. };
  2719. SizedArray<_ShadowReg, 8> shadowRegs;
  2720. mMaxCallParamCount = BF_MAX(mMaxCallParamCount, argCount);
  2721. for (int argIdx = args.size() - 1; argIdx >= 0; argIdx--)
  2722. {
  2723. if ((argIdx == 0) && (compositeRetReg == X64Reg_RDX))
  2724. argOfs = 0;
  2725. bool isVarArg = (varArgStart != -1) && (argIdx >= varArgStart);
  2726. auto mcValue = args[argIdx];
  2727. auto argType = GetType(mcValue);
  2728. X64CPURegister useReg = X64Reg_None;
  2729. int useArgIdx = argIdx + argOfs;
  2730. if (argType->IsVector())
  2731. {
  2732. switch (useArgIdx)
  2733. {
  2734. case 0:
  2735. useReg = X64Reg_M128_XMM0;
  2736. break;
  2737. case 1:
  2738. useReg = X64Reg_M128_XMM1;
  2739. break;
  2740. case 2:
  2741. useReg = X64Reg_M128_XMM2;
  2742. break;
  2743. case 3:
  2744. useReg = X64Reg_M128_XMM3;
  2745. break;
  2746. }
  2747. }
  2748. else if (argType->IsFloat())
  2749. {
  2750. switch (useArgIdx)
  2751. {
  2752. case 0:
  2753. useReg = (argType->mTypeCode == BeTypeCode_Float) ? X64Reg_XMM0_f32 : X64Reg_XMM0_f64;
  2754. break;
  2755. case 1:
  2756. useReg = (argType->mTypeCode == BeTypeCode_Float) ? X64Reg_XMM1_f32 : X64Reg_XMM1_f64;
  2757. break;
  2758. case 2:
  2759. useReg = (argType->mTypeCode == BeTypeCode_Float) ? X64Reg_XMM2_f32 : X64Reg_XMM2_f64;
  2760. break;
  2761. case 3:
  2762. useReg = (argType->mTypeCode == BeTypeCode_Float) ? X64Reg_XMM3_f32 : X64Reg_XMM3_f64;
  2763. break;
  2764. }
  2765. if (isVarArg)
  2766. {
  2767. X64CPURegister shadowReg = X64Reg_None;
  2768. switch (useArgIdx)
  2769. {
  2770. case 0:
  2771. shadowReg = !flipFirstRegs ? X64Reg_RCX : X64Reg_RDX;
  2772. break;
  2773. case 1:
  2774. shadowReg = !flipFirstRegs ? X64Reg_RDX : X64Reg_RCX;
  2775. break;
  2776. case 2:
  2777. shadowReg = X64Reg_R8;
  2778. break;
  2779. case 3:
  2780. shadowReg = X64Reg_R9;
  2781. break;
  2782. }
  2783. if ((shadowReg != X64Reg_None) && (useReg != X64Reg_None))
  2784. {
  2785. shadowRegs.push_back(_ShadowReg{ useReg, shadowReg });
  2786. }
  2787. }
  2788. }
  2789. else
  2790. {
  2791. switch (useArgIdx)
  2792. {
  2793. case 0:
  2794. useReg = !flipFirstRegs ? X64Reg_RCX : X64Reg_RDX;
  2795. break;
  2796. case 1:
  2797. useReg = !flipFirstRegs ? X64Reg_RDX : X64Reg_RCX;
  2798. break;
  2799. case 2:
  2800. useReg = X64Reg_R8;
  2801. break;
  2802. case 3:
  2803. useReg = X64Reg_R9;
  2804. break;
  2805. }
  2806. }
  2807. if ((!argType->IsNonVectorComposite()) && (!isVarArg)) // VarArg uses full 64 bits
  2808. useReg = ResizeRegister(useReg, argType);
  2809. if (mcValue.mKind == BeMCOperandKind_VRegAddr)
  2810. {
  2811. auto vregInfo = GetVRegInfo(mcValue);
  2812. vregInfo->mForceMem = true;
  2813. CheckForce(vregInfo);
  2814. }
  2815. if (useReg != X64Reg_None)
  2816. {
  2817. // We want to do the non-register params before the PreserveRegs call,
  2818. // because those may be a memory-to-memory mov, which will result in
  2819. // a temporary variable which will allocate a register which may
  2820. // conflict with our param regs
  2821. if (!didPreserveRegs)
  2822. _AddPreserveRegs();
  2823. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(useReg), mcValue);
  2824. }
  2825. else
  2826. {
  2827. auto useTypeCode = argType->mTypeCode;
  2828. if (isVarArg)
  2829. {
  2830. if (argType->IsFloat())
  2831. useTypeCode = BeTypeCode_Double;
  2832. else
  2833. useTypeCode = BeTypeCode_Int64;
  2834. }
  2835. auto callArgVReg = GetCallArgVReg(useArgIdx, useTypeCode);
  2836. // Do a 'def' for every usage, to clear out the 'init' at all paths
  2837. CreateDefineVReg(BeMCOperand::FromVReg(callArgVReg.mVRegIdx));
  2838. if (argType->IsNonVectorComposite())
  2839. {
  2840. BEMC_ASSERT(mcValue.mKind == BeMCOperandKind_VReg);
  2841. AllocInst(BeMCInstKind_Mov, callArgVReg, BeMCOperand::FromVRegAddr(mcValue.mVRegIdx));
  2842. }
  2843. else
  2844. AllocInst(BeMCInstKind_Mov, callArgVReg, mcValue);
  2845. }
  2846. }
  2847. if (!didPreserveRegs)
  2848. _AddPreserveRegs();
  2849. auto mcFunc = func;
  2850. for (auto& shadowReg : shadowRegs) // Do float shadowing
  2851. {
  2852. AllocInst(BeMCInstKind_MovRaw, BeMCOperand::FromReg(shadowReg.mIReg), BeMCOperand::FromReg(shadowReg.mFReg));
  2853. }
  2854. AllocInst(BeMCInstKind_Call, mcFunc);
  2855. if (dynStackSize > 0)
  2856. AllocInst(BeMCInstKind_Add, BeMCOperand::FromReg(X64Reg_RSP), BeMCOperand::FromImmediate(dynStackSize));
  2857. if ((!mcResult) && (retType != NULL) && (retType->mTypeCode != BeTypeCode_None))
  2858. {
  2859. mcResult = AllocVirtualReg(retType);
  2860. CreateDefineVReg(mcResult);
  2861. X64CPURegister resultReg;
  2862. if (retType->IsFloat())
  2863. {
  2864. resultReg = ResizeRegister(X64Reg_M128_XMM0, retType->mSize);
  2865. }
  2866. else if (retType->IsVector())
  2867. {
  2868. resultReg = X64Reg_M128_XMM0;
  2869. }
  2870. else
  2871. {
  2872. BEMC_ASSERT(retType->IsIntable());
  2873. resultReg = ResizeRegister(X64Reg_RAX, retType->mSize);
  2874. }
  2875. AllocInst(BeMCInstKind_Mov, mcResult, BeMCOperand::FromReg(resultReg));
  2876. }
  2877. if (noReturn)
  2878. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromPreserveFlag(BeMCPreserveFlag_NoRestore));
  2879. else
  2880. AllocInst(BeMCInstKind_RestoreVolatiles);
  2881. return mcResult;
  2882. }
  2883. void BeMCContext::CreateMemSet(const BeMCOperand& addr, uint8 val, int size, int align)
  2884. {
  2885. if ((size == 8) || (size == 4) || (size == 2) || (size == 1))
  2886. {
  2887. BeType* type = NULL;
  2888. BeMCOperand zeroVal;
  2889. zeroVal.mImmediate = 0;
  2890. switch (size)
  2891. {
  2892. case 8:
  2893. type = mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  2894. zeroVal.mKind = BeMCOperandKind_Immediate_i64;
  2895. zeroVal.mImmediate = ((int64)val << 56) | ((int64)val << 48) | ((int64)val << 40) | ((int64)val << 32) |
  2896. (val << 24) | (val << 16) | (val << 8) | val;
  2897. break;
  2898. case 4:
  2899. type = mModule->mContext->GetPrimitiveType(BeTypeCode_Int32);
  2900. zeroVal.mKind = BeMCOperandKind_Immediate_i32;
  2901. zeroVal.mImmediate = (val << 24) | (val << 16) | (val << 8) | val;
  2902. break;
  2903. case 2:
  2904. type = mModule->mContext->GetPrimitiveType(BeTypeCode_Int16);
  2905. zeroVal.mKind = BeMCOperandKind_Immediate_i16;
  2906. zeroVal.mImmediate = (val << 8) | val;
  2907. break;
  2908. case 1:
  2909. type = mModule->mContext->GetPrimitiveType(BeTypeCode_Int8);
  2910. zeroVal.mKind = BeMCOperandKind_Immediate_i8;
  2911. zeroVal.mImmediate = val;
  2912. break;
  2913. }
  2914. auto ptrType = mModule->mContext->GetPointerTo(type);
  2915. auto result = AllocVirtualReg(ptrType);
  2916. CreateDefineVReg(result);
  2917. auto vregInfo = GetVRegInfo(result);
  2918. vregInfo->mRelTo = addr;
  2919. vregInfo->mIsExpr = true;
  2920. BeMCOperand dest;
  2921. dest.mKind = BeMCOperandKind_VRegLoad;
  2922. dest.mVRegIdx = result.mVRegIdx;
  2923. AllocInst(BeMCInstKind_Mov, dest, zeroVal);
  2924. return;
  2925. }
  2926. if (addr.IsVRegAny())
  2927. {
  2928. auto vregInfo = GetVRegInfo(addr);
  2929. if (!vregInfo->mIsRetVal)
  2930. vregInfo->mForceMem = true;
  2931. CheckForce(vregInfo);
  2932. }
  2933. if (size <= 256)
  2934. {
  2935. auto temp = addr;
  2936. BeMCOperand mcMemSetInfo;
  2937. mcMemSetInfo.mKind = BeMCOperandKind_MemSetInfo;
  2938. mcMemSetInfo.mMemSetInfo.mSize = size;
  2939. mcMemSetInfo.mMemSetInfo.mAlign = align;
  2940. mcMemSetInfo.mMemSetInfo.mValue = val;
  2941. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_R11));
  2942. AllocInst(BeMCInstKind_MemSet, mcMemSetInfo, temp);
  2943. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_R11));
  2944. return;
  2945. }
  2946. SizedArray<BeMCOperand, 3> args = { addr, BeMCOperand::FromImmediate(val), BeMCOperand::FromImmediate(size) };
  2947. auto mcFunc = BeMCOperand::FromSymbolAddr(mCOFFObject->GetSymbolRef("memset")->mIdx);
  2948. CreateCall(mcFunc, args);
  2949. }
  2950. void BeMCContext::CreateMemCpy(const BeMCOperand& dest, const BeMCOperand& src, int size, int align)
  2951. {
  2952. auto destVRegInfo = GetVRegInfo(dest);
  2953. auto srcVRegInfo = GetVRegInfo(src);
  2954. if ((destVRegInfo != NULL) && (srcVRegInfo != NULL) && (destVRegInfo->mIsRetVal) && (srcVRegInfo->mIsRetVal))
  2955. return;
  2956. if (size <= 256)
  2957. {
  2958. int destIdx = -1;
  2959. if (!GetEncodedOperand(dest, destIdx))
  2960. {
  2961. BeMCOperand tempDest = AllocVirtualReg(GetType(dest), 1);
  2962. CreateDefineVReg(tempDest);
  2963. AllocInst(BeMCInstKind_Mov, tempDest, dest);
  2964. auto vregInfo = mVRegInfo[tempDest.mVRegIdx];
  2965. vregInfo->mForceReg = true;
  2966. vregInfo->mDisableR11 = true;
  2967. destIdx = tempDest.mVRegIdx;
  2968. }
  2969. int srcIdx = -1;
  2970. if (!GetEncodedOperand(src, srcIdx))
  2971. {
  2972. BeMCOperand tempSrc = AllocVirtualReg(GetType(src), 1);
  2973. CreateDefineVReg(tempSrc);
  2974. AllocInst(BeMCInstKind_Mov, tempSrc, src);
  2975. auto vregInfo = mVRegInfo[tempSrc.mVRegIdx];
  2976. vregInfo->mForceReg = true;
  2977. vregInfo->mDisableR11 = true;
  2978. srcIdx = tempSrc.mVRegIdx;
  2979. }
  2980. BeMCOperand mcMemCpyInfo;
  2981. mcMemCpyInfo.mKind = BeMCOperandKind_MemCpyInfo;
  2982. mcMemCpyInfo.mMemCpyInfo.mSize = size;
  2983. mcMemCpyInfo.mMemCpyInfo.mAlign = align;
  2984. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_R11));
  2985. BeMCOperand mcVRegPair;
  2986. mcVRegPair.mKind = BeMCOperandKind_VRegPair;
  2987. mcVRegPair.mVRegPair.mVRegIdx0 = destIdx;
  2988. mcVRegPair.mVRegPair.mVRegIdx1 = srcIdx;
  2989. AllocInst(BeMCInstKind_MemCpy, mcMemCpyInfo, mcVRegPair);
  2990. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_R11));
  2991. return;
  2992. }
  2993. SizedArray<BeMCOperand, 3> args = { dest, src, BeMCOperand::FromImmediate(size) };
  2994. auto mcFunc = BeMCOperand::FromSymbolAddr(mCOFFObject->GetSymbolRef("memcpy")->mIdx);
  2995. CreateCall(mcFunc, args);
  2996. }
  2997. void BeMCContext::CreateTableSwitchSection(BeSwitchInst* switchInst, int startIdx, int endIdx)
  2998. {
  2999. auto& sect = mCOFFObject->mRDataSect;
  3000. auto defaultBlock = GetOperand(switchInst->mDefaultBlock);
  3001. auto int32Type = mModule->mContext->GetPrimitiveType(BeTypeCode_Int32);
  3002. auto int32PtrType = mModule->mContext->GetPointerTo(int32Type);
  3003. auto nativeType = mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  3004. auto nativePtrType = mModule->mContext->GetPointerTo(nativeType);
  3005. int64 loVal = switchInst->mCases.front().mValue->mInt64;
  3006. int64 hiVal = switchInst->mCases.back().mValue->mInt64;
  3007. int numVals = switchInst->mCases.size();
  3008. BeMCSymbol* sym = mCOFFObject->mSymbols.Alloc();
  3009. sym->mType = int32Type;
  3010. sym->mName = StrFormat("@jumpTab%d", mCOFFObject->mCurJumpTableIdx++);
  3011. sym->mIsStatic = true;
  3012. sym->mSymKind = BeMCSymbolKind_External;
  3013. sym->mIdx = (int)mCOFFObject->mSymbols.size() - 1;
  3014. mCOFFObject->MarkSectionUsed(sect);
  3015. sym->mSectionNum = sect.mSectionIdx + 1;
  3016. sect.mData.Align(4);
  3017. sect.mAlign = BF_MAX(sect.mAlign, 4);
  3018. sym->mValue = sect.mData.GetSize();
  3019. auto mcValue = GetOperand(switchInst->mValue);
  3020. auto beType = GetType(mcValue);
  3021. auto mcOfsValue = AllocVirtualReg(nativeType);
  3022. CreateDefineVReg(mcOfsValue);
  3023. if (beType->mSize < 8)
  3024. AllocInst(BeMCInstKind_MovSX, mcOfsValue, mcValue);
  3025. else
  3026. AllocInst(BeMCInstKind_Mov, mcOfsValue, mcValue);
  3027. AllocInst(BeMCInstKind_Sub, mcOfsValue, BeMCOperand::FromImmediate(loVal));
  3028. AllocInst(BeMCInstKind_CondBr, defaultBlock, BeMCOperand::FromCmpKind(BeCmpKind_SLT));
  3029. AllocInst(BeMCInstKind_Cmp, mcOfsValue, BeMCOperand::FromImmediate(hiVal - loVal));
  3030. AllocInst(BeMCInstKind_CondBr, defaultBlock, BeMCOperand::FromCmpKind(BeCmpKind_SGT));
  3031. auto jumpVReg = AllocVirtualReg(nativeType);
  3032. CreateDefineVReg(jumpVReg);
  3033. AllocInst(BeMCInstKind_Mov, jumpVReg, BeMCOperand::FromSymbolAddr(sym->mIdx));
  3034. auto jumpRelVReg = AllocVirtualReg(int32PtrType);
  3035. CreateDefineVReg(jumpRelVReg);
  3036. auto vregInfo = mVRegInfo[jumpRelVReg.mVRegIdx];
  3037. vregInfo->mIsExpr = true;
  3038. vregInfo->mRelTo = jumpVReg;
  3039. vregInfo->mRelOffset = mcOfsValue;
  3040. vregInfo->mRelOffsetScale = 4;
  3041. jumpRelVReg.mKind = BeMCOperandKind_VRegLoad;
  3042. AllocInst(BeMCInstKind_Mov, jumpVReg, jumpRelVReg);
  3043. auto imageBaseSym = mCOFFObject->GetSymbolRef("__ImageBase");
  3044. imageBaseSym->mType = nativeType;
  3045. auto baseVReg = AllocVirtualReg(nativeType);
  3046. CreateDefineVReg(baseVReg);
  3047. AllocInst(BeMCInstKind_Mov, baseVReg, BeMCOperand::FromSymbolAddr(imageBaseSym->mIdx));
  3048. AllocInst(BeMCInstKind_Add, jumpVReg, baseVReg);
  3049. AllocInst(BeMCInstKind_Br, jumpVReg);
  3050. int64 lastVal = loVal - 1;
  3051. defaultBlock.mBlock->AddPred(mActiveBlock);
  3052. for (int caseIdx = 0; caseIdx < (int)switchInst->mCases.size(); caseIdx++)
  3053. {
  3054. auto& switchCase = switchInst->mCases[caseIdx];
  3055. int64 newVal = switchCase.mValue->mInt64;
  3056. for (int64 val = lastVal + 1; val <= newVal; val++)
  3057. {
  3058. BeMCSwitchEntry switchEntry;
  3059. switchEntry.mOfs = sect.mData.GetSize();
  3060. if (val == newVal)
  3061. {
  3062. auto switchBlock = GetOperand(switchCase.mBlock);
  3063. switchBlock.mBlock->AddPred(mActiveBlock);
  3064. switchEntry.mBlock = switchBlock.mBlock;
  3065. }
  3066. else
  3067. {
  3068. switchEntry.mBlock = defaultBlock.mBlock;
  3069. }
  3070. mSwitchEntries.push_back(switchEntry);
  3071. // Placeholder
  3072. sect.mData.Write((int32)0);
  3073. }
  3074. lastVal = newVal;
  3075. }
  3076. }
  3077. void BeMCContext::CreateBinarySwitchSection(BeSwitchInst* switchInst, int startIdx, int endIdx)
  3078. {
  3079. // This is an empirically determined binary switching limit
  3080. if (endIdx - startIdx >= 18)
  3081. {
  3082. int gteLabel = mCurLabelIdx++;
  3083. auto mcDefaultBlock = GetOperand(switchInst->mDefaultBlock);
  3084. int midIdx = startIdx + (endIdx - startIdx) / 2;
  3085. auto& switchCase = switchInst->mCases[midIdx];
  3086. auto switchBlock = GetOperand(switchCase.mBlock);
  3087. auto mcValue = GetOperand(switchInst->mValue);
  3088. auto valueType = GetType(mcValue);
  3089. AllocInst(BeMCInstKind_Cmp, mcValue, GetOperand(switchCase.mValue));
  3090. AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromLabel(gteLabel), BeMCOperand::FromCmpKind(BeCmpKind_SGE));
  3091. switchBlock.mBlock->AddPred(mActiveBlock);
  3092. CreateBinarySwitchSection(switchInst, startIdx, midIdx);
  3093. AllocInst(BeMCInstKind_Br, mcDefaultBlock);
  3094. CreateLabel(-1, gteLabel);
  3095. CreateBinarySwitchSection(switchInst, midIdx, endIdx);
  3096. return;
  3097. }
  3098. for (int caseIdx = startIdx; caseIdx < endIdx; caseIdx++)
  3099. {
  3100. auto& switchCase = switchInst->mCases[caseIdx];
  3101. auto switchBlock = GetOperand(switchCase.mBlock);
  3102. auto mcValue = GetOperand(switchInst->mValue);
  3103. AllocInst(BeMCInstKind_Cmp, mcValue, GetOperand(switchCase.mValue));
  3104. AllocInst(BeMCInstKind_CondBr, switchBlock, BeMCOperand::FromCmpKind(BeCmpKind_EQ));
  3105. switchBlock.mBlock->AddPred(mActiveBlock);
  3106. }
  3107. }
  3108. void BeMCContext::CreateCondBr(BeMCBlock* mcBlock, BeMCOperand& testVal, const BeMCOperand& trueBlock, const BeMCOperand& falseBlock)
  3109. {
  3110. if (testVal.IsImmediate())
  3111. {
  3112. if (testVal.mImmediate != 0)
  3113. AllocInst(BeMCInstKind_Br, trueBlock);
  3114. else
  3115. AllocInst(BeMCInstKind_Br, falseBlock);
  3116. }
  3117. else if (testVal.mKind == BeMCOperandKind_CmpResult)
  3118. {
  3119. // Beef-specific: assuming CMP results aren't stomped
  3120. auto& cmpResult = mCmpResults[testVal.mCmpResultIdx];
  3121. if (cmpResult.mInverted)
  3122. {
  3123. AllocInst(BeMCInstKind_CondBr, falseBlock, BeMCOperand::FromCmpKind(cmpResult.mCmpKind));
  3124. AllocInst(BeMCInstKind_Br, trueBlock);
  3125. }
  3126. else
  3127. {
  3128. AllocInst(BeMCInstKind_CondBr, trueBlock, BeMCOperand::FromCmpKind(cmpResult.mCmpKind));
  3129. AllocInst(BeMCInstKind_Br, falseBlock);
  3130. }
  3131. }
  3132. else if (testVal.mKind == BeMCOperandKind_Phi)
  3133. {
  3134. auto phi = testVal.mPhi;
  3135. auto phiBlock = phi->mBlock;
  3136. if ((phi->mBrTrue != NULL) && (phi->mBrFalse != NULL))
  3137. {
  3138. // Redirect branches
  3139. for (auto instIdx = 0; instIdx < phiBlock->mInstructions.size(); instIdx++)
  3140. {
  3141. auto inst = phiBlock->mInstructions[instIdx];
  3142. if (inst->mKind == BeMCInstKind_Br)
  3143. {
  3144. if (inst->mArg0 == phi->mBrTrue)
  3145. inst->mArg0 = trueBlock;
  3146. else if (inst->mArg0 == phi->mBrFalse)
  3147. inst->mArg0 = falseBlock;
  3148. }
  3149. }
  3150. phi->mBrTrue = trueBlock;
  3151. phi->mBrFalse = falseBlock;
  3152. return;
  3153. }
  3154. phi->mBrTrue = trueBlock;
  3155. phi->mBrFalse = falseBlock;
  3156. // Using a Phi for a CondBr in a different block is not supported
  3157. if (phiBlock != mcBlock)
  3158. {
  3159. // Special case if our using block directly leads into us
  3160. BEMC_ASSERT(mcBlock->mPreds.size() == 1);
  3161. BEMC_ASSERT(mcBlock->mPreds[0] == phiBlock);
  3162. }
  3163. for (auto instIdx = 0; instIdx < phiBlock->mInstructions.size(); instIdx++)
  3164. {
  3165. auto inst = phiBlock->mInstructions[instIdx];
  3166. if (inst->mKind == BeMCInstKind_DefPhi)
  3167. {
  3168. BEMC_ASSERT(inst->mArg0.mPhi == phi);
  3169. RemoveInst(phiBlock, instIdx);
  3170. break;
  3171. }
  3172. }
  3173. for (auto& phiVal : phi->mValues)
  3174. {
  3175. BeMCOperand landinglabel;
  3176. if (phiVal.mValue.mKind != BeMCOperandKind_Phi)
  3177. {
  3178. landinglabel = BeMCOperand::FromLabel(mCurLabelIdx++);
  3179. AllocInst(BeMCInstKind_Label, landinglabel);
  3180. }
  3181. int brInstIdx = -1;
  3182. bool isFalseCmpResult = false;
  3183. if (landinglabel)
  3184. {
  3185. bool found = false;
  3186. auto _CheckBlock = [&](BeMCBlock* block)
  3187. {
  3188. for (int checkIdx = (int)block->mInstructions.size() - 1; checkIdx >= 0; checkIdx--)
  3189. {
  3190. auto checkInst = block->mInstructions[checkIdx];
  3191. if ((checkInst->mArg0.mKind == BeMCOperandKind_Block) &&
  3192. (checkInst->mArg0.mBlock == phi->mBlock))
  3193. {
  3194. brInstIdx = checkIdx;
  3195. checkInst->mArg0 = landinglabel;
  3196. found = true;
  3197. // Don't break, if we're are chained to another PHI then we need to modify all the labels
  3198. isFalseCmpResult = false;
  3199. if ((checkIdx >= 2) && (checkInst->mKind == BeMCInstKind_Br))
  3200. {
  3201. auto prevInst = block->mInstructions[checkIdx - 1];
  3202. auto prevPrevInst = block->mInstructions[checkIdx - 2];
  3203. if ((prevPrevInst->mKind == BeMCInstKind_Cmp) && (prevPrevInst->mResult == phiVal.mValue) &&
  3204. (prevInst->mKind == BeMCInstKind_CondBr) && (prevInst->mArg1.mCmpKind == BeCmpKind_EQ))
  3205. {
  3206. isFalseCmpResult = true;
  3207. }
  3208. }
  3209. }
  3210. }
  3211. };
  3212. _CheckBlock(phiVal.mBlockFrom);
  3213. BEMC_ASSERT(found);
  3214. }
  3215. if (isFalseCmpResult)
  3216. {
  3217. BEMC_ASSERT(phiVal.mValue.mKind == BeMCOperandKind_CmpResult);
  3218. AllocInst(BeMCInstKind_Br, falseBlock);
  3219. }
  3220. else if ((phiVal.mValue.IsImmediate()) ||
  3221. (phiVal.mValue.mKind == BeMCOperandKind_CmpResult) ||
  3222. (phiVal.mValue.mKind == BeMCOperandKind_Phi) ||
  3223. (phiVal.mValue.mKind == BeMCOperandKind_NotResult))
  3224. {
  3225. CreateCondBr(phiVal.mBlockFrom, phiVal.mValue, trueBlock, falseBlock);
  3226. }
  3227. else
  3228. {
  3229. // Do the 'test' in the preceding block. This is needed for liveness checking. Using a vreg in this
  3230. // block would require the def to be hoisted
  3231. SetAndRestoreValue<BeMCBlock*> prevActiveBlock(mActiveBlock, phiVal.mBlockFrom);
  3232. BeMCOperand testImm;
  3233. testImm.mKind = BeMCOperandKind_Immediate_i8;
  3234. testImm.mImmediate = 1;
  3235. auto mcInst = AllocInst(BeMCInstKind_Test, phiVal.mValue, testImm, brInstIdx);
  3236. prevActiveBlock.Restore();
  3237. mcInst = AllocInst(BeMCInstKind_CondBr, trueBlock, BeMCOperand::FromCmpKind(BeCmpKind_NE));
  3238. mcInst = AllocInst(BeMCInstKind_Br, falseBlock);
  3239. }
  3240. }
  3241. }
  3242. else if (testVal.mKind == BeMCOperandKind_NotResult)
  3243. {
  3244. // Just get the original block and swap the true/false blocks
  3245. auto invResult = GetOperand(testVal.mNotResult->mValue, true);
  3246. if (invResult)
  3247. {
  3248. CreateCondBr(mcBlock, invResult, falseBlock, trueBlock);
  3249. }
  3250. }
  3251. else
  3252. {
  3253. BeMCOperand testImm;
  3254. testImm.mKind = BeMCOperandKind_Immediate_i8;
  3255. testImm.mImmediate = 1;
  3256. auto mcInst = AllocInst(BeMCInstKind_Test, testVal, testImm);
  3257. mcInst = AllocInst(BeMCInstKind_CondBr, trueBlock, BeMCOperand::FromCmpKind(BeCmpKind_NE));
  3258. mcInst = AllocInst(BeMCInstKind_Br, falseBlock);
  3259. }
  3260. }
  3261. void BeMCContext::CreatePhiAssign(BeMCBlock* mcBlock, const BeMCOperand& testVal, const BeMCOperand& result, const BeMCOperand& doneLabel)
  3262. {
  3263. if (testVal.mKind == BeMCOperandKind_Phi)
  3264. {
  3265. auto phi = testVal.mPhi;
  3266. for (auto& phiVal : phi->mValues)
  3267. {
  3268. BeMCOperand landinglabel;
  3269. if (phiVal.mValue.mKind == BeMCOperandKind_Phi)
  3270. {
  3271. // Remove PhiDef
  3272. auto phi = phiVal.mValue.mPhi;
  3273. auto phiBlock = phi->mBlock;
  3274. for (auto instIdx = 0; instIdx < phiBlock->mInstructions.size(); instIdx++)
  3275. {
  3276. auto inst = phiBlock->mInstructions[instIdx];
  3277. if (inst->mKind == BeMCInstKind_DefPhi)
  3278. {
  3279. BEMC_ASSERT(inst->mArg0.mPhi == phi);
  3280. RemoveInst(phiBlock, instIdx);
  3281. break;
  3282. }
  3283. }
  3284. CreatePhiAssign(phiVal.mBlockFrom, phiVal.mValue, result, doneLabel);
  3285. }
  3286. else
  3287. {
  3288. bool found = false;
  3289. auto _CheckBlock = [&](BeMCBlock* block)
  3290. {
  3291. SetAndRestoreValue<BeMCBlock*> prevActiveBlock(mActiveBlock, block);
  3292. for (int checkIdx = (int)block->mInstructions.size() - 1; checkIdx >= 0; checkIdx--)
  3293. {
  3294. auto checkInst = block->mInstructions[checkIdx];
  3295. if ((checkInst->mArg0.mKind == BeMCOperandKind_Block) &&
  3296. (checkInst->mArg0.mBlock == phi->mBlock))
  3297. {
  3298. // Don't use an explicit dbgLoc
  3299. SetAndRestoreValue<BeDbgLoc*> prevDbgLoc(mCurDbgLoc, NULL);
  3300. if (checkInst->mKind == BeMCInstKind_CondBr)
  3301. {
  3302. auto falseLabel = BeMCOperand::FromLabel(mCurLabelIdx++);
  3303. auto prevDest = checkInst->mArg0;
  3304. checkInst->mArg0 = falseLabel;
  3305. checkInst->mArg1.mCmpKind = BeModule::InvertCmp(checkInst->mArg1.mCmpKind);
  3306. int insertIdx = checkIdx + 1;
  3307. SetAndRestoreValue<int*> prevInsertIdxRef(mInsertInstIdxRef, &insertIdx);
  3308. CreateStore(BeMCInstKind_Mov, phiVal.mValue, OperandToAddr(result));
  3309. AllocInst(BeMCInstKind_Br, prevDest);
  3310. AllocInst(BeMCInstKind_Label, falseLabel);
  3311. }
  3312. else
  3313. {
  3314. int insertIdx = checkIdx;
  3315. SetAndRestoreValue<int*> prevInsertIdxRef(mInsertInstIdxRef, &insertIdx);
  3316. CreateStore(BeMCInstKind_Mov, phiVal.mValue, OperandToAddr(result));
  3317. }
  3318. found = true;
  3319. }
  3320. }
  3321. };
  3322. _CheckBlock(phiVal.mBlockFrom);
  3323. BEMC_ASSERT(found);
  3324. }
  3325. if (landinglabel)
  3326. {
  3327. AllocInst(BeMCInstKind_Br, doneLabel);
  3328. }
  3329. }
  3330. }
  3331. else
  3332. {
  3333. SoftFail("Unhandled CreatePhiAssign value");
  3334. }
  3335. }
  3336. BeMCOperand BeMCContext::GetImmediate(int64 val)
  3337. {
  3338. BeMCOperand operand;
  3339. operand.mKind = BeMCOperandKind_Immediate_i64;
  3340. operand.mImmediate = val;
  3341. return operand;
  3342. }
  3343. BeMCOperand BeMCContext::OperandToAddr(const BeMCOperand& operand)
  3344. {
  3345. BeMCOperand loadedOperand = operand;
  3346. if (loadedOperand.mKind == BeMCOperandKind_VRegLoad)
  3347. loadedOperand.mKind = BeMCOperandKind_VReg;
  3348. else if (loadedOperand.mKind == BeMCOperandKind_VReg)
  3349. loadedOperand.mKind = BeMCOperandKind_VRegAddr;
  3350. else if (loadedOperand.mKind == BeMCOperandKind_Symbol)
  3351. loadedOperand.mKind = BeMCOperandKind_SymbolAddr;
  3352. else
  3353. Fail("Invalid operand in OperandToAddr");
  3354. return loadedOperand;
  3355. }
  3356. BeMCOperand BeMCContext::GetVReg(int regNum)
  3357. {
  3358. auto vregInfo = mVRegInfo[regNum];
  3359. if (vregInfo->mReg != X64Reg_None)
  3360. return BeMCOperand::FromReg(vregInfo->mReg);
  3361. BeMCOperand operand;
  3362. operand.mKind = BeMCOperandKind_VReg;
  3363. operand.mVRegIdx = regNum;
  3364. return operand;
  3365. }
  3366. BeMCOperand BeMCContext::AllocVirtualReg(BeType* type, int refCount, bool mustBeReg)
  3367. {
  3368. BEMC_ASSERT(type->mTypeCode != BeTypeCode_Function); // We can only have pointers to these
  3369. if (mustBeReg)
  3370. {
  3371. BEMC_ASSERT(!type->IsNonVectorComposite());
  3372. BEMC_ASSERT(type->mSize != 0);
  3373. }
  3374. int vregIdx = (int)mVRegInfo.size();
  3375. BeMCVRegInfo* vregInfo = mAlloc.Alloc<BeMCVRegInfo>();
  3376. vregInfo->mType = type;
  3377. vregInfo->mAlign = type->mAlign;
  3378. vregInfo->mRefCount = refCount;
  3379. vregInfo->mForceReg = mustBeReg;
  3380. mVRegInfo.push_back(vregInfo);
  3381. BeMCOperand mcOperand;
  3382. mcOperand.mKind = BeMCOperandKind_VReg;
  3383. mcOperand.mVRegIdx = vregIdx++;
  3384. if (mDebugging)
  3385. {
  3386. if (mcOperand.mVRegIdx == 8)
  3387. {
  3388. NOP;
  3389. }
  3390. }
  3391. return mcOperand;
  3392. }
  3393. int BeMCContext::GetUnderlyingVReg(int vregIdx)
  3394. {
  3395. while (true)
  3396. {
  3397. auto vregInfo = mVRegInfo[vregIdx];
  3398. if (!vregInfo->mRelTo.IsVRegAny())
  3399. return vregIdx;
  3400. if (vregInfo->mRelOffset)
  3401. return vregIdx;
  3402. vregIdx = vregInfo->mRelTo.mVRegIdx;
  3403. }
  3404. }
  3405. bool BeMCContext::HasForceRegs(const BeMCOperand &operand)
  3406. {
  3407. if (!operand.IsVRegAny())
  3408. return false;
  3409. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  3410. if (vregInfo->mForceReg)
  3411. return true;
  3412. return HasForceRegs(vregInfo->mRelTo) || HasForceRegs(vregInfo->mRelOffset);
  3413. }
  3414. bool BeMCContext::OperandsEqual(const BeMCOperand& op0, const BeMCOperand& op1, bool exact)
  3415. {
  3416. bool isVReg0 = op0.mKind == BeMCOperandKind_VReg;
  3417. bool isVReg1 = op1.mKind == BeMCOperandKind_VReg;
  3418. if (!isVReg0 || !isVReg1)
  3419. {
  3420. return op0 == op1;
  3421. }
  3422. if (exact)
  3423. {
  3424. int vregIdx0 = op0.mVRegIdx;
  3425. int vregIdx1 = op1.mVRegIdx;
  3426. while (true)
  3427. {
  3428. auto vregInfo = mVRegInfo[vregIdx0];
  3429. if (!vregInfo->IsDirectRelTo())
  3430. break;
  3431. if (vregInfo->mRelTo.mKind != BeMCOperandKind_VReg)
  3432. return false;
  3433. vregIdx0 = vregInfo->mRelTo.mVRegIdx;
  3434. }
  3435. while (true)
  3436. {
  3437. auto vregInfo = mVRegInfo[vregIdx1];
  3438. if (!vregInfo->IsDirectRelTo())
  3439. break;
  3440. if (vregInfo->mRelTo.mKind != BeMCOperandKind_VReg)
  3441. return false;
  3442. vregIdx1 = vregInfo->mRelTo.mVRegIdx;
  3443. }
  3444. return vregIdx0 == vregIdx1;
  3445. }
  3446. int vregIdx0 = GetUnderlyingVReg(op0.mVRegIdx);
  3447. int vregIdx1 = GetUnderlyingVReg(op1.mVRegIdx);
  3448. return vregIdx0 == vregIdx1;
  3449. }
  3450. bool BeMCContext::ContainsNonOffsetRef(const BeMCOperand& checkOperand, const BeMCOperand& findOperand)
  3451. {
  3452. if (!checkOperand.IsVRegAny())
  3453. return false;
  3454. if (checkOperand == findOperand)
  3455. return true;
  3456. auto vregInfo = GetVRegInfo(checkOperand);
  3457. if (ContainsNonOffsetRef(vregInfo->mRelTo, findOperand))
  3458. return true;
  3459. return false;
  3460. }
  3461. // For all values that we are certain we will immediately use, we directly do a Def preceding its first use.
  3462. // For Allocas in the head, however, we may not use that memory for a long time so we imply the Def location
  3463. // in DoDefPass. That allows us to limit how long that vreg will hold onto a register, reducing register
  3464. // contention.
  3465. BeMCInst* BeMCContext::CreateDefineVReg(const BeMCOperand& vreg, int insertIdx)
  3466. {
  3467. auto mcInst = AllocInst(insertIdx);
  3468. mcInst->mKind = BeMCInstKind_Def;
  3469. mcInst->mArg0 = vreg;
  3470. return mcInst;
  3471. }
  3472. int BeMCContext::CreateLabel(int insertIdx, int labelIdx)
  3473. {
  3474. auto inst = AllocInst(insertIdx);
  3475. inst->mKind = BeMCInstKind_Label;
  3476. inst->mArg0.mKind = BeMCOperandKind_Label;
  3477. if (labelIdx != -1)
  3478. inst->mArg0.mLabelIdx = labelIdx;
  3479. else
  3480. inst->mArg0.mLabelIdx = mCurLabelIdx++;
  3481. return inst->mArg0.mLabelIdx;
  3482. }
  3483. bool BeMCContext::FindTarget(const BeMCOperand& loc, BeMCBlock*& outBlock, int& outInstIdx)
  3484. {
  3485. if (loc.mKind == BeMCOperandKind_Block)
  3486. {
  3487. outBlock = loc.mBlock;
  3488. outInstIdx = -1;
  3489. return true;
  3490. }
  3491. if (loc.mKind == BeMCOperandKind_Label)
  3492. {
  3493. // Linear search :-(
  3494. for (auto mcBlock : mBlocks)
  3495. {
  3496. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  3497. {
  3498. auto inst = mcBlock->mInstructions[instIdx];
  3499. if ((inst->mKind == BeMCInstKind_Label) && (inst->mArg0.mLabelIdx == loc.mLabelIdx))
  3500. {
  3501. outBlock = mcBlock;
  3502. outInstIdx = instIdx;
  3503. return true;
  3504. }
  3505. }
  3506. }
  3507. }
  3508. return false;
  3509. }
  3510. BeMCOperand BeMCContext::AllocRelativeVirtualReg(BeType* type, const BeMCOperand& relTo, const BeMCOperand& relOffset, int relScale)
  3511. {
  3512. if (!relTo.IsVRegAny())
  3513. {
  3514. auto relToType = GetType(relTo);
  3515. auto tempRelTo = AllocVirtualReg(relToType, 1);
  3516. CreateDefineVReg(tempRelTo);
  3517. AllocInst(BeMCInstKind_Mov, tempRelTo, relTo);
  3518. tempRelTo.mKind = BeMCOperandKind_VReg;
  3519. return AllocRelativeVirtualReg(type, tempRelTo, relOffset, relScale);
  3520. }
  3521. BEMC_ASSERT(relTo.IsVRegAny());
  3522. auto relToVRegInfo = GetVRegInfo(relTo);
  3523. if ((relToVRegInfo->mRelTo) && (relToVRegInfo->mRelOffsetScale == 1) && (relToVRegInfo->mRelOffset.IsImmediate()) &&
  3524. (relOffset.IsImmediate()) && (relScale == 1))
  3525. {
  3526. if (relTo.mKind == BeMCOperandKind_VReg)
  3527. {
  3528. int offset = (int)relOffset.mImmediate;
  3529. if (relToVRegInfo->mRelOffset)
  3530. offset += relToVRegInfo->mRelOffset.mImmediate;
  3531. return AllocRelativeVirtualReg(type, relToVRegInfo->mRelTo, GetImmediate(offset), 1);
  3532. }
  3533. }
  3534. if ((relTo.mKind == BeMCOperandKind_VRegAddr) && (!relToVRegInfo->mRelOffset) && (relOffset.IsZero()) && (relScale == 1))
  3535. {
  3536. auto vregInfo = GetVRegInfo(relTo);
  3537. if (vregInfo->IsDirectRelToAny())
  3538. {
  3539. if (vregInfo->mRelTo.mKind == BeMCOperandKind_Symbol)
  3540. {
  3541. return AllocRelativeVirtualReg(type, BeMCOperand::FromSymbolAddr(vregInfo->mRelTo.mSymbolIdx), BeMCOperand(), 1);
  3542. }
  3543. }
  3544. }
  3545. auto mcVReg = AllocVirtualReg(type);
  3546. auto vregInfo = GetVRegInfo(mcVReg);
  3547. vregInfo->mIsExpr = true;
  3548. vregInfo->mRelTo = relTo;
  3549. if (!relOffset.IsZero())
  3550. vregInfo->mRelOffset = relOffset;
  3551. vregInfo->mRelOffsetScale = relScale;
  3552. mcVReg.mKind = BeMCOperandKind_VReg;
  3553. return mcVReg;
  3554. }
  3555. BeMCVRegInfo* BeMCContext::GetVRegInfo(const BeMCOperand& operand)
  3556. {
  3557. if (!operand.IsVRegAny())
  3558. return NULL;
  3559. return mVRegInfo[operand.mVRegIdx];
  3560. }
  3561. bool BeMCContext::HasSymbolAddr(const BeMCOperand& operand)
  3562. {
  3563. if (operand.mKind == BeMCOperandKind_SymbolAddr)
  3564. return true;
  3565. auto vregInfo = GetVRegInfo(operand);
  3566. if (vregInfo == NULL)
  3567. return false;
  3568. if ((vregInfo->mRelTo) && (HasSymbolAddr(vregInfo->mRelTo)))
  3569. return true;
  3570. if ((vregInfo->mRelOffset) && (HasSymbolAddr(vregInfo->mRelOffset)))
  3571. return true;
  3572. return false;
  3573. }
  3574. BeMCOperand BeMCContext::ReplaceWithNewVReg(BeMCOperand& operand, int& instIdx, bool isInput, bool mustBeReg, bool preserveDeref)
  3575. {
  3576. if ((isInput) && (operand.mKind == BeMCOperandKind_VRegLoad) && (preserveDeref))
  3577. {
  3578. BeMCOperand addrOperand = OperandToAddr(operand);
  3579. BeMCOperand scratchReg = AllocVirtualReg(GetType(addrOperand), 2, mustBeReg);
  3580. CreateDefineVReg(scratchReg, instIdx++);
  3581. AllocInst(BeMCInstKind_Mov, scratchReg, addrOperand, instIdx++);
  3582. operand = BeMCOperand::ToLoad(scratchReg);
  3583. return scratchReg;
  3584. }
  3585. BeMCOperand scratchReg = AllocVirtualReg(GetType(operand), 2, mustBeReg);
  3586. CreateDefineVReg(scratchReg, instIdx++);
  3587. if (isInput)
  3588. AllocInst(BeMCInstKind_Mov, scratchReg, operand, instIdx++);
  3589. else
  3590. AllocInst(BeMCInstKind_Mov, operand, scratchReg, instIdx++ + 1);
  3591. operand = scratchReg;
  3592. return scratchReg;
  3593. }
  3594. BeMCOperand BeMCContext::RemapOperand(BeMCOperand& operand, BeMCRemapper& regRemaps)
  3595. {
  3596. if (!operand.IsVRegAny())
  3597. return operand;
  3598. int regIdx = regRemaps.GetHead(operand.mVRegIdx);
  3599. if (regIdx < 0)
  3600. {
  3601. BeMCOperand newOperand;
  3602. newOperand.mKind = operand.mKind;
  3603. if (newOperand.mKind == BeMCOperandKind_VRegLoad)
  3604. newOperand.mKind = BeMCOperandKind_VReg;
  3605. else
  3606. NotImpl();
  3607. newOperand.mVRegIdx = -regIdx;
  3608. return newOperand;
  3609. }
  3610. else
  3611. {
  3612. BeMCOperand newOperand;
  3613. newOperand.mKind = operand.mKind;
  3614. newOperand.mVRegIdx = regIdx;
  3615. return newOperand;
  3616. }
  3617. }
  3618. bool BeMCContext::IsLive(BeVTrackingList* liveRegs, int origVRegIdx, BeMCRemapper& regRemaps)
  3619. {
  3620. // Check the whole remap chain to determine liveness
  3621. int vregIdx = regRemaps.GetHead(origVRegIdx);
  3622. while (vregIdx != -1)
  3623. {
  3624. if (mLivenessContext.IsSet(liveRegs, vregIdx))
  3625. return true;
  3626. vregIdx = regRemaps.GetNext(vregIdx);
  3627. }
  3628. return false;
  3629. }
  3630. void BeMCContext::AddRegRemap(int from, int to, BeMCRemapper& regRemaps, bool allowRemapToDbgVar)
  3631. {
  3632. auto vregInfoFrom = mVRegInfo[from];
  3633. auto vregInfoTo = mVRegInfo[to];
  3634. BEMC_ASSERT(vregInfoFrom->mDbgVariable == NULL);
  3635. if (vregInfoTo->mDbgVariable != NULL)
  3636. {
  3637. // We can't do a direct remap due to lifetime issues, so do an indirect one
  3638. vregInfoFrom->mIsExpr = true;
  3639. if (!vregInfoFrom->mIsRetVal)
  3640. vregInfoFrom->mForceReg = false;
  3641. vregInfoFrom->mRelTo = BeMCOperand::FromVReg(to);
  3642. vregInfoFrom->mRelOffset = BeMCOperand();
  3643. vregInfoFrom->mRelOffsetScale = 1;
  3644. return;
  3645. }
  3646. regRemaps.Add(from, to);
  3647. }
  3648. bool BeMCContext::GetEncodedOperand(const BeMCOperand& operand, int& encodedVal)
  3649. {
  3650. if (operand.mKind == BeMCOperandKind_VReg)
  3651. {
  3652. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  3653. encodedVal = operand.mVRegIdx;
  3654. return true;
  3655. }
  3656. if (operand.mKind != BeMCOperandKind_VRegAddr)
  3657. return false;
  3658. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  3659. if (vregInfo->IsDirectRelTo())
  3660. {
  3661. if (!GetEncodedOperand(vregInfo->mRelTo, encodedVal))
  3662. return false;
  3663. encodedVal |= BeMCOperand::EncodeFlag_Addr;
  3664. return true;
  3665. }
  3666. if (vregInfo->mIsExpr)
  3667. return false;
  3668. encodedVal = operand.mVRegIdx | BeMCOperand::EncodeFlag_Addr;
  3669. return true;
  3670. }
  3671. bool BeMCContext::HasPointerDeref(const BeMCOperand& operand)
  3672. {
  3673. if (operand.mKind == BeMCOperandKind_VRegLoad)
  3674. return true;
  3675. auto vregInfo = GetVRegInfo(operand);
  3676. if (vregInfo == NULL)
  3677. return false;
  3678. if (!vregInfo->mRelTo)
  3679. return false;
  3680. if ((vregInfo->mRelTo.mKind == BeMCOperandKind_VRegLoad) ||
  3681. (vregInfo->mRelTo.mKind == BeMCOperandKind_VReg))
  3682. return true;
  3683. return false;
  3684. }
  3685. bool BeMCContext::AreOperandsEquivalent(const BeMCOperand& origOp0, const BeMCOperand& origOp1, BeMCRemapper& regRemaps)
  3686. {
  3687. auto op0 = origOp0;
  3688. RemapOperand(op0, regRemaps);
  3689. auto op1 = origOp1;
  3690. RemapOperand(op1, regRemaps);
  3691. if (op0.mKind != op1.mKind)
  3692. return false;
  3693. if (op0.mKind == BeMCOperandKind_None)
  3694. return true;
  3695. if (op0.IsSymbol())
  3696. return op0.mSymbolIdx == op1.mSymbolIdx;
  3697. if (op0.IsImmediate())
  3698. return op0.mImmediate == op1.mImmediate;
  3699. if (op0.IsNativeReg())
  3700. return op0.mReg == op1.mReg;
  3701. if (op0.IsVRegAny())
  3702. {
  3703. if (op0.mVRegIdx == op1.mVRegIdx)
  3704. return true;
  3705. auto vregInfo0 = mVRegInfo[op0.mVRegIdx];
  3706. auto vregInfo1 = mVRegInfo[op1.mVRegIdx];
  3707. if (vregInfo0->mType != vregInfo1->mType)
  3708. return false;
  3709. if ((!vregInfo0->mIsExpr) || (!vregInfo1->mIsExpr))
  3710. return false;
  3711. return
  3712. (vregInfo0->mRelOffsetScale == vregInfo1->mRelOffsetScale) &&
  3713. (AreOperandsEquivalent(vregInfo0->mRelTo, vregInfo1->mRelTo, regRemaps)) &&
  3714. (AreOperandsEquivalent(vregInfo0->mRelOffset, vregInfo1->mRelOffset, regRemaps));
  3715. }
  3716. return false;
  3717. }
  3718. bool BeMCContext::CouldBeReg(const BeMCOperand& operand)
  3719. {
  3720. if (operand.mKind != BeMCOperandKind_VReg)
  3721. return false;
  3722. auto vregInfo = GetVRegInfo(operand);
  3723. if (vregInfo->mForceMem)
  3724. return false;
  3725. if (vregInfo->mIsExpr)
  3726. {
  3727. if (vregInfo->mRelOffset)
  3728. return false;
  3729. }
  3730. return true;
  3731. }
  3732. // bool BeMCContext::CouldBeReg(const BeMCOperand& operand)
  3733. // {
  3734. // if (operand.mKind != BeMCOperandKind_VReg)
  3735. // return false;
  3736. //
  3737. // auto vregInfo = GetVRegInfo(operand);
  3738. // if ((vregInfo->mIsRetVal) && (mCompositeRetVRegIdx != -1) && (mCompositeRetVRegIdx != operand.mVRegIdx))
  3739. // {
  3740. // return CouldBeReg(BeMCOperand::FromVReg(mCompositeRetVRegIdx));
  3741. // }
  3742. //
  3743. // if (vregInfo->mReg != X64Reg_None)
  3744. // return true;
  3745. //
  3746. // if (vregInfo->mForceMem)
  3747. // return false;
  3748. //
  3749. // if (vregInfo->mIsExpr)
  3750. // {
  3751. // if (vregInfo->mRelOffset)
  3752. // return false;
  3753. // return CouldBeReg(vregInfo->mRelTo);
  3754. // }
  3755. //
  3756. // return !vregInfo->mType->IsNonVectorComposite();
  3757. // }
  3758. bool BeMCContext::CheckForce(BeMCVRegInfo* vregInfo)
  3759. {
  3760. if (!vregInfo->mIsRetVal)
  3761. {
  3762. if (vregInfo->mForceMem && vregInfo->mForceReg)
  3763. SoftFail("vreg forceMem/forceReg collision");
  3764. }
  3765. return false;
  3766. }
  3767. void BeMCContext::MarkLive(BeVTrackingList* liveRegs, SizedArrayImpl<int>& newRegs, BeVTrackingList*& vregsInitialized, const BeMCOperand& operand)
  3768. {
  3769. int vregIdx = operand.mVRegIdx;
  3770. auto vregInfo = mVRegInfo[vregIdx];
  3771. if (mLivenessContext.IsSet(liveRegs, operand.mVRegIdx))
  3772. return;
  3773. for (auto newReg : newRegs)
  3774. if (newReg == operand.mVRegIdx)
  3775. return;
  3776. if (!mColorizer.mNodes.empty())
  3777. {
  3778. // Is new
  3779. for (int i = 0; i < liveRegs->mSize; i++)
  3780. {
  3781. int checkReg = liveRegs->mEntries[i];
  3782. if (checkReg >= mLivenessContext.mNumItems)
  3783. continue;
  3784. mColorizer.AddEdge(checkReg, operand.mVRegIdx);
  3785. }
  3786. for (auto checkReg : newRegs)
  3787. mColorizer.AddEdge(checkReg, operand.mVRegIdx);
  3788. }
  3789. if (vregInfo->mHasDynLife)
  3790. {
  3791. if (!mVRegInitializedContext.IsSet(vregsInitialized, vregIdx))
  3792. {
  3793. // This indicates that this is a 'def' usage, meaning the value wasn't actually set yet
  3794. // so don't propagate this index upward
  3795. return;
  3796. }
  3797. }
  3798. if ((vregInfo->mRelTo) && (vregInfo->mRelTo.IsVRegAny()))
  3799. MarkLive(liveRegs, newRegs, vregsInitialized, vregInfo->mRelTo);
  3800. if ((vregInfo->mRelOffset) && (vregInfo->mRelOffset.IsVRegAny()))
  3801. MarkLive(liveRegs, newRegs, vregsInitialized, vregInfo->mRelOffset);
  3802. newRegs.push_back(operand.mVRegIdx);
  3803. }
  3804. BeVTrackingList* BeMCContext::MergeLiveRegs(BeVTrackingList* prevDestEntry, BeVTrackingList* mergeFrom)
  3805. {
  3806. int vregIdx = -1;
  3807. SizedArray<int, 16> newNodes;
  3808. SizedArray<int, 16> prevExclusiveNodes;
  3809. // Take nodes that were exclusive to the new set and add edges to nodes that were exclusive the old set
  3810. /*while (true)
  3811. {
  3812. vregIdx = mLivenessContext.GetNextDiffSetIdx(prevDestEntry.mBits, mergeFrom, vregIdx);
  3813. if (vregIdx == -1)
  3814. break;
  3815. newNodes.push_back(vregIdx);
  3816. if (!mColorizer.mNodes.empty())
  3817. {
  3818. int checkReg = -1;
  3819. while (true)
  3820. {
  3821. checkReg = mLivenessContext.GetNextDiffSetIdx(mergeFrom, prevDestEntry.mBits, checkReg);
  3822. if (checkReg == -1)
  3823. break;
  3824. mColorizer.AddEdge(checkReg, vregIdx);
  3825. }
  3826. }
  3827. }*/
  3828. auto prevItr = prevDestEntry->begin();
  3829. auto prevEnd = prevDestEntry->end();
  3830. auto mergeFromItr = mergeFrom->begin();
  3831. auto mergeFromEnd = mergeFrom->end();
  3832. while ((prevItr != prevEnd) && (mergeFromItr != mergeFromEnd))
  3833. {
  3834. int prevIdx = *prevItr;
  3835. int mergeIdx = *mergeFromItr;
  3836. bool done = false;
  3837. while (mergeIdx < prevIdx)
  3838. {
  3839. newNodes.push_back(mergeIdx);
  3840. ++mergeFromItr;
  3841. if (mergeFromItr == mergeFromEnd)
  3842. {
  3843. done = true;
  3844. break;
  3845. }
  3846. mergeIdx = *mergeFromItr;
  3847. }
  3848. if (done)
  3849. break;
  3850. while (prevIdx < mergeIdx)
  3851. {
  3852. prevExclusiveNodes.push_back(prevIdx);
  3853. ++prevItr;
  3854. if (prevItr == prevEnd)
  3855. {
  3856. done = true;
  3857. break;
  3858. }
  3859. prevIdx = *prevItr;
  3860. }
  3861. if (done)
  3862. break;
  3863. if (prevIdx == mergeIdx)
  3864. {
  3865. ++prevItr;
  3866. ++mergeFromItr;
  3867. }
  3868. }
  3869. while (prevItr != prevEnd)
  3870. {
  3871. prevExclusiveNodes.push_back(*prevItr);
  3872. ++prevItr;
  3873. }
  3874. while (mergeFromItr != mergeFromEnd)
  3875. {
  3876. newNodes.push_back(*mergeFromItr);
  3877. ++mergeFromItr;
  3878. }
  3879. if (!mColorizer.mNodes.empty())
  3880. {
  3881. for (int newIdx : newNodes)
  3882. {
  3883. for (int prevExclusiveIdx : prevExclusiveNodes)
  3884. {
  3885. mColorizer.AddEdge(newIdx, prevExclusiveIdx);
  3886. }
  3887. }
  3888. }
  3889. return mLivenessContext.Add(prevDestEntry, newNodes, false);
  3890. }
  3891. static void DedupPushBack(SizedArrayImpl<int>& vec, int val)
  3892. {
  3893. if (vec.Contains(val))
  3894. return;
  3895. vec.push_back(val);
  3896. }
  3897. static int genLivenessIdx = 0;
  3898. void BeMCContext::GenerateLiveness(BeMCBlock* block, BeVTrackingGenContext* genCtx, bool& modifiedBlockBefore, bool& modifiedBlockAfter)
  3899. {
  3900. genCtx->mBlocks[block->mBlockIdx].mGenerateCount++;
  3901. modifiedBlockBefore = false;
  3902. modifiedBlockAfter = false;
  3903. genCtx->mCalls++;
  3904. bool debugging = false;
  3905. //if (mDebugging) debugging = true;
  3906. //if (mBeFunction->mName == "?Draw@DarkEditWidgetContent@dark@theme@Beefy@@UEAAXPEAVGraphics@gfx@4@@Z")
  3907. //debugging = true;
  3908. //"?DrawEntry@DrawContext@PerfView@BeefPerf@@QEAAXPEAVGraphics@gfx@Beefy@@PEAVTrackNodeEntry@23@MM@Z")
  3909. //debugging &= mcColorizer != NULL;
  3910. if (debugging)
  3911. {
  3912. if (block->mBlockIdx == 220)
  3913. {
  3914. //BF_ASSERT(mLivenessContext.IsSet(succLiveRegs, 36));
  3915. }
  3916. }
  3917. if (debugging)
  3918. {
  3919. OutputDebugStrF("GenerateLiveness %s(%d)\n", ToString(BeMCOperand::FromBlock(block)).c_str(), block->mBlockIdx);
  3920. }
  3921. genCtx->mHandledCalls++;
  3922. BeMDNode* curDbgScope = NULL;
  3923. // Don't use 'ref' here, it's important that mSuccLiveRegs actually means 'successor', otherwise
  3924. // we could miss a later-occurring liveness overlap on a different successor
  3925. BeVTrackingList* liveRegs = block->mSuccLiveness;
  3926. BeVTrackingList* vregsInitialized = block->mSuccVRegsInitialized;
  3927. // When we process our first vregsInitialized (at the bottom of the block), there won't be Change entries
  3928. // for us to use to determine the delta from mSuccVRegsInitialized to inst->mVRegsInitialized
  3929. bool needsManualVRegInitDiff = true;
  3930. for (int instIdx = (int)block->mInstructions.size() - 1; instIdx >= 0; instIdx--)
  3931. {
  3932. genLivenessIdx++;
  3933. auto inst = block->mInstructions[instIdx];
  3934. auto prevLiveness = inst->mLiveness;
  3935. genCtx->mInstructions++;
  3936. SizedArray<int, 16> removeVec;
  3937. SizedArray<int, 16> addVec;
  3938. SizedArray<int, 16> filteredRemoveVec;
  3939. SizedArray<int, 16> filteredAddVec;
  3940. inst->mLiveness = liveRegs;
  3941. if ((inst->mVRegsInitialized != NULL) && (inst->mVRegsInitialized != vregsInitialized))
  3942. {
  3943. auto _VRegUninit = [&](int vregIdxEx)
  3944. {
  3945. int vregIdx = vregIdxEx % mVRegInitializedContext.mNumItems;
  3946. auto vregInfo = mVRegInfo[vregIdx];
  3947. if (!vregInfo->mHasDynLife)
  3948. return;
  3949. bool doClear = false;
  3950. if (vregInfo->mDoConservativeLife)
  3951. {
  3952. // Only do liveness clear when both 'init' and 'non-init' is set
  3953. int otherVRegIdxEx = vregIdx;
  3954. if (vregIdxEx < mVRegInitializedContext.mNumItems)
  3955. otherVRegIdxEx += mVRegInitializedContext.mNumItems;
  3956. if (!mVRegInitializedContext.IsSet(inst->mVRegsInitialized, otherVRegIdxEx))
  3957. doClear = true;
  3958. }
  3959. else
  3960. {
  3961. // Only do liveness clear on 'init' clearing out
  3962. if (vregIdx < mVRegInitializedContext.mNumItems)
  3963. doClear = true;
  3964. }
  3965. if (doClear)
  3966. {
  3967. DedupPushBack(removeVec, vregIdx);
  3968. }
  3969. };
  3970. if (needsManualVRegInitDiff)
  3971. {
  3972. BEMC_ASSERT(vregsInitialized == block->mSuccVRegsInitialized);
  3973. // Manually compare
  3974. auto vregsInit0 = vregsInitialized;
  3975. auto vregsInit1 = inst->mVRegsInitialized;
  3976. if (vregsInit0 != vregsInit1)
  3977. {
  3978. int idx0 = 0;
  3979. int idx1 = 0;
  3980. while ((idx0 != vregsInit0->mSize) && (idx1 != vregsInit1->mSize))
  3981. {
  3982. int val0 = vregsInit0->mEntries[idx0];
  3983. int val1 = vregsInit1->mEntries[idx1];
  3984. if (val0 == val1)
  3985. {
  3986. idx0++;
  3987. idx1++;
  3988. continue;
  3989. }
  3990. bool done = false;
  3991. while (val0 < val1)
  3992. {
  3993. _VRegUninit(val0);
  3994. idx0++;
  3995. if (idx0 >= vregsInit0->mSize)
  3996. {
  3997. done = true;
  3998. break;
  3999. }
  4000. val0 = vregsInit0->mEntries[idx0];
  4001. }
  4002. if (done)
  4003. break;
  4004. while (val1 < val0)
  4005. {
  4006. idx1++;
  4007. if (idx1 >= vregsInit1->mSize)
  4008. {
  4009. done = true;
  4010. break;
  4011. }
  4012. val1 = vregsInit1->mEntries[idx1];
  4013. }
  4014. if (done)
  4015. break;
  4016. }
  4017. while (idx0 < vregsInit0->mSize)
  4018. {
  4019. _VRegUninit(vregsInit0->mEntries[idx0++]);
  4020. }
  4021. }
  4022. }
  4023. else
  4024. {
  4025. for (int changeIdx = 0; changeIdx < vregsInitialized->mNumChanges; changeIdx++)
  4026. {
  4027. int vregIdxEx = vregsInitialized->GetChange(changeIdx);
  4028. if (vregIdxEx < 0)
  4029. continue;
  4030. // It's possible this vregsInitialized is equivalent to the one in 'inst', but also merged with another 'inst'
  4031. // during legalization. We need to avoid uninitializing vregs if that's the case.
  4032. // the entry above this
  4033. if (inst->mVRegsInitialized->ContainsChange(vregIdxEx))
  4034. continue;
  4035. _VRegUninit(vregIdxEx);
  4036. }
  4037. }
  4038. vregsInitialized = inst->mVRegsInitialized;
  4039. }
  4040. if (inst->mVRegsInitialized != NULL)
  4041. needsManualVRegInitDiff = false;
  4042. if (inst->mKind == BeMCInstKind_DbgDecl)
  4043. {
  4044. if (!mVRegInitializedContext.IsSet(inst->mVRegsInitialized, inst->mArg0.mVRegIdx))
  4045. {
  4046. // There are some rare cases with conditional branches where one branch will have a vreg marked as
  4047. // initialized, which causes the variable to be marked as live, which propagates upward into the block
  4048. // containing a variable declaration, before the actual def point
  4049. DedupPushBack(removeVec, inst->mArg0.mVRegIdx);
  4050. }
  4051. liveRegs = mLivenessContext.Modify(liveRegs, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  4052. continue;
  4053. }
  4054. // This is used for clearing out things like usage of inline return values, which will be accessed after their
  4055. // lifetime end (the lifetime ends inside the inlined method but the value is used afterward, in the inlining
  4056. // function. This will emit as a load of a dbgVar, so we need to drill down into the relTo values
  4057. if (inst->mKind == BeMCInstKind_LifetimeStart)
  4058. {
  4059. BEMC_ASSERT(inst->mArg0.IsVRegAny());
  4060. int vregIdx = inst->mArg0.mVRegIdx;
  4061. while (true)
  4062. {
  4063. DedupPushBack(removeVec, vregIdx);
  4064. auto vregInfo = mVRegInfo[vregIdx];
  4065. if (!vregInfo->IsDirectRelTo())
  4066. break;
  4067. BEMC_ASSERT(vregInfo->mRelTo.IsVReg());
  4068. vregIdx = vregInfo->mRelTo.mVRegIdx;
  4069. }
  4070. liveRegs = mLivenessContext.Modify(liveRegs, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  4071. continue;
  4072. }
  4073. if (inst->mKind == BeMCInstKind_LifetimeEnd)
  4074. {
  4075. auto vregInfo = GetVRegInfo(inst->mArg0);
  4076. if (vregInfo->mDbgVariable == NULL)
  4077. {
  4078. // Only extend lifetime down through LifetimeEnd when we have a debug variable we want to keep alive,
  4079. // otherwise constrict lifetime to actual usage
  4080. liveRegs = mLivenessContext.Modify(liveRegs, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  4081. continue;
  4082. }
  4083. }
  4084. if (inst->IsDef())
  4085. {
  4086. DedupPushBack(removeVec, inst->mArg0.mVRegIdx);
  4087. liveRegs = mLivenessContext.Modify(liveRegs, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  4088. continue;
  4089. }
  4090. auto operands = { &inst->mResult, &inst->mArg0, &inst->mArg1 };
  4091. for (auto operand : operands)
  4092. {
  4093. if (operand->IsSymbol())
  4094. {
  4095. auto sym = mCOFFObject->mSymbols[operand->mSymbolIdx];
  4096. if (sym->mIsTLS)
  4097. {
  4098. MarkLive(liveRegs, addVec, vregsInitialized, BeMCOperand::FromVReg(mTLSVRegIdx));
  4099. }
  4100. }
  4101. if (operand->mKind == BeMCOperandKind_VRegPair)
  4102. {
  4103. auto mcOperand = BeMCOperand::FromEncoded(operand->mVRegPair.mVRegIdx0);
  4104. if (mcOperand.IsVRegAny())
  4105. MarkLive(liveRegs, addVec, vregsInitialized, mcOperand);
  4106. mcOperand = BeMCOperand::FromEncoded(operand->mVRegPair.mVRegIdx1);
  4107. if (mcOperand.IsVRegAny())
  4108. MarkLive(liveRegs, addVec, vregsInitialized, mcOperand);
  4109. }
  4110. if (operand->IsVRegAny())
  4111. {
  4112. MarkLive(liveRegs, addVec, vregsInitialized, *operand);
  4113. }
  4114. }
  4115. liveRegs = mLivenessContext.Modify(liveRegs, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  4116. /*if ((!isFirstEntry) && (liveRegs == prevLiveness))
  4117. {
  4118. // We've already been here before and nothing changed
  4119. return;
  4120. }*/
  4121. inst->mLiveness = liveRegs;
  4122. }
  4123. if (block == mBlocks[0])
  4124. {
  4125. BEMC_ASSERT(block->mBlockIdx == 0);
  4126. if (!mLivenessContext.IsEmpty(liveRegs))
  4127. {
  4128. for (int vregIdx = 0; vregIdx < mLivenessContext.mNumEntries; vregIdx++)
  4129. {
  4130. if (mLivenessContext.IsSet(liveRegs, vregIdx))
  4131. {
  4132. auto vregInfo = mVRegInfo[vregIdx];
  4133. // If we are still alive then the only valid reason is because we have mHasDynLife and our 'init' flag is still set
  4134. if (vregInfo->mHasDynLife)
  4135. {
  4136. if (!mVRegInitializedContext.IsSet(vregsInitialized, vregIdx))
  4137. {
  4138. if (vregInfo->mDoConservativeLife)
  4139. BEMC_ASSERT(mVRegInitializedContext.IsSet(vregsInitialized, vregIdx, BeTrackKind_Uninitialized));
  4140. else
  4141. SoftFail("VReg lifetime error");
  4142. }
  4143. }
  4144. else
  4145. {
  4146. SoftFail("VReg lifetime error");
  4147. }
  4148. }
  4149. }
  4150. }
  4151. }
  4152. for (auto pred : block->mPreds)
  4153. {
  4154. auto& entry = genCtx->mBlocks[pred->mBlockIdx];
  4155. BEMC_ASSERT(pred == mBlocks[pred->mBlockIdx]);
  4156. auto newSuccLiveness = MergeLiveRegs(pred->mSuccLiveness, liveRegs);
  4157. if (newSuccLiveness == pred->mSuccLiveness)
  4158. continue;
  4159. pred->mSuccLiveness = newSuccLiveness;
  4160. pred->mSuccVRegsInitialized = mVRegInitializedContext.Merge(pred->mSuccVRegsInitialized, vregsInitialized);
  4161. if (pred->mBlockIdx > block->mBlockIdx)
  4162. modifiedBlockAfter = true;
  4163. else
  4164. modifiedBlockBefore = true;
  4165. entry.mGenerateQueued = true;
  4166. }
  4167. }
  4168. void BeMCContext::GenerateLiveness()
  4169. {
  4170. //GenerateLiveness_OLD(mcColorizer);
  4171. //return;
  4172. mLivenessContext.mStats = BeVTrackingContext::Stats();
  4173. #ifdef _DEBUG
  4174. // So we can Print() while generating liveness (the old values would have been unallocated)
  4175. for (auto mcBlock : mBlocks)
  4176. {
  4177. for (auto mcInst : mcBlock->mInstructions)
  4178. {
  4179. mcInst->mLiveness = NULL;
  4180. }
  4181. }
  4182. #endif
  4183. BP_ZONE("BeMCContext::GenerateLiveness");
  4184. mLivenessContext.Clear();
  4185. mLivenessContext.Init((int)mVRegInfo.size());
  4186. auto emptyList = mLivenessContext.AllocEmptyList();
  4187. BeVTrackingGenContext genCtx;
  4188. genCtx.mEmptyList = emptyList;
  4189. genCtx.mBlocks.Resize(mBlocks.size());
  4190. for (auto mcBlock : mBlocks)
  4191. {
  4192. mcBlock->mSuccLiveness = emptyList;
  4193. mcBlock->mSuccVRegsInitialized = emptyList;
  4194. if (mTLSVRegIdx != -1)
  4195. {
  4196. // Keep TLS alive
  4197. SizedArray<int, 1> vec = { mTLSVRegIdx };
  4198. mcBlock->mSuccLiveness = mLivenessContext.Add(mcBlock->mSuccLiveness, vec, false);
  4199. }
  4200. }
  4201. while (true)
  4202. {
  4203. bool didWork = false;
  4204. // Handle any outstanding pred entries
  4205. for (int blockIdx = (int)mBlocks.size() - 1; blockIdx >= 0; blockIdx--)
  4206. {
  4207. auto& entry = genCtx.mBlocks[blockIdx];
  4208. if (entry.mGenerateQueued)
  4209. {
  4210. entry.mGenerateQueued = false;
  4211. didWork = true;
  4212. auto block = mBlocks[blockIdx];
  4213. bool modifiedBlockBefore;
  4214. bool modifiedBlockAfter;
  4215. GenerateLiveness(block, &genCtx, modifiedBlockBefore, modifiedBlockAfter);
  4216. if (modifiedBlockAfter)
  4217. break;
  4218. }
  4219. }
  4220. // If no pred entries, find blocks that haven't been processed yet
  4221. if (!didWork)
  4222. {
  4223. for (int blockIdx = (int)mBlocks.size() - 1; blockIdx >= 0; blockIdx--)
  4224. {
  4225. auto& entry = genCtx.mBlocks[blockIdx];
  4226. if (entry.mGenerateCount == 0)
  4227. {
  4228. didWork = true;
  4229. auto block = mBlocks[blockIdx];
  4230. bool modifiedBlockBefore;
  4231. bool modifiedBlockAfter;
  4232. GenerateLiveness(block, &genCtx, modifiedBlockBefore, modifiedBlockAfter);
  4233. if (modifiedBlockBefore || modifiedBlockAfter)
  4234. break;
  4235. }
  4236. }
  4237. }
  4238. if (!didWork)
  4239. break;
  4240. }
  4241. int instCount = 0;
  4242. for (auto block : mBlocks)
  4243. instCount += (int)block->mInstructions.size();
  4244. BpEvent("GenerateLiveness Results",
  4245. StrFormat("Blocks: %d\nInstructions: %d\nVRegs: %d\nCalls: %d\nHandled Calls: %d\nProcessed Instructions: %d\nLiveness Bytes: %d\n"
  4246. "Temp Bytes: %d\nBits Bytes: %d\nList Bytes: %d\nSucc Bytes: %d",
  4247. mBlocks.size(), instCount, mVRegInfo.size(), genCtx.mCalls, genCtx.mHandledCalls, genCtx.mInstructions, mLivenessContext.mAlloc.GetAllocSize(),
  4248. genCtx.mAlloc.GetAllocSize(), mLivenessContext.mStats.mBitsBytes, mLivenessContext.mStats.mListBytes, mLivenessContext.mStats.mSuccBytes).c_str());
  4249. }
  4250. void BeMCContext::IntroduceVRegs(const BeMCOperand& newVReg, BeMCBlock* block, int startInstIdx, int lastInstIdx)
  4251. {
  4252. return;
  4253. /*BF_ASSERT((block->mInstructions[startInstIdx]->mKind == BeMCInstKind_Def) && (block->mInstructions[startInstIdx]->mArg0 == newVReg));
  4254. BF_ASSERT(mColorizer.mNodes.size() == newVReg.mVRegIdx);
  4255. mColorizer.mNodes.resize(mColorizer.mNodes.size() + 1);
  4256. BeVTrackingBits* lastLiveness = NULL;
  4257. for (int instIdx = startInstIdx + 1; instIdx <= lastInstIdx; instIdx++)
  4258. {
  4259. auto inst = block->mInstructions[instIdx];
  4260. if (lastLiveness == NULL)
  4261. {
  4262. for (int vregIdx : *inst->mLiveness)
  4263. {
  4264. if (vregIdx >= mLivenessContext.mNumItems)
  4265. continue;
  4266. mColorizer.AddEdge(newVReg.mVRegIdx, vregIdx);
  4267. }
  4268. }
  4269. else
  4270. {
  4271. int vregIdx = -1;
  4272. while (true)
  4273. {
  4274. vregIdx = mLivenessContext.GetNextDiffSetIdx(lastLiveness, inst->mLiveness, vregIdx);
  4275. if (vregIdx == -1)
  4276. break;
  4277. mColorizer.AddEdge(newVReg.mVRegIdx, vregIdx);
  4278. }
  4279. }
  4280. if (inst->mVRegsInitializedEx != NULL)
  4281. {
  4282. auto vregArray = (BeMCVRegArray*)mAlloc.AllocBytes(sizeof(int) * (inst->mVRegsInitializedEx->mSize + 1 + 1), sizeof(int));
  4283. vregArray->mSize = inst->mVRegsInitializedEx->mSize + 1;
  4284. for (int listIdx = 0; listIdx < inst->mVRegsInitializedEx->mSize; listIdx++)
  4285. {
  4286. mColorizer.AddEdge(newVReg.mVRegIdx, inst->mVRegsInitializedEx->mVRegIndices[listIdx]);
  4287. vregArray->mVRegIndices[listIdx] = inst->mVRegsInitializedEx->mVRegIndices[listIdx];
  4288. }
  4289. vregArray->mVRegIndices[vregArray->mSize - 1] = newVReg.mVRegIdx;
  4290. inst->mVRegsInitializedEx = vregArray;
  4291. }
  4292. else
  4293. {
  4294. auto vregArray = (BeMCVRegArray*)mAlloc.AllocBytes(sizeof(int) * 2, sizeof(int));
  4295. vregArray->mSize = 1;
  4296. vregArray->mVRegIndices[0] = newVReg.mVRegIdx;
  4297. inst->mVRegsInitializedEx = vregArray;
  4298. }
  4299. lastLiveness = inst->mLiveness;
  4300. }*/
  4301. }
  4302. bool BeMCContext::IsVolatileReg(X64CPURegister reg)
  4303. {
  4304. switch (ResizeRegister(reg, 8))
  4305. {
  4306. case X64Reg_RAX:
  4307. case X64Reg_RCX:
  4308. case X64Reg_RDX:
  4309. case X64Reg_R8:
  4310. case X64Reg_R9:
  4311. case X64Reg_R10:
  4312. case X64Reg_R11:
  4313. case X64Reg_XMM0_f64:
  4314. case X64Reg_XMM1_f64:
  4315. case X64Reg_XMM2_f64:
  4316. case X64Reg_XMM3_f64:
  4317. case X64Reg_XMM4_f64:
  4318. case X64Reg_XMM5_f64:
  4319. return true;
  4320. default:
  4321. return false;
  4322. }
  4323. }
  4324. bool BeMCContext::IsXMMReg(X64CPURegister reg)
  4325. {
  4326. return (reg >= X64Reg_XMM0_f64) && (reg <= X64Reg_M128_XMM15);
  4327. }
  4328. X64CPURegister BeMCContext::ResizeRegister(X64CPURegister reg, int numBytes)
  4329. {
  4330. if (numBytes == 16)
  4331. {
  4332. switch (reg)
  4333. {
  4334. case X64Reg_XMM0_f32:
  4335. case X64Reg_XMM0_f64:
  4336. case X64Reg_M128_XMM0: return X64Reg_M128_XMM0;
  4337. case X64Reg_XMM1_f32:
  4338. case X64Reg_XMM1_f64:
  4339. case X64Reg_M128_XMM1: return X64Reg_M128_XMM1;
  4340. case X64Reg_XMM2_f32:
  4341. case X64Reg_XMM2_f64:
  4342. case X64Reg_M128_XMM2: return X64Reg_M128_XMM2;
  4343. case X64Reg_XMM3_f32:
  4344. case X64Reg_XMM3_f64:
  4345. case X64Reg_M128_XMM3: return X64Reg_M128_XMM3;
  4346. case X64Reg_XMM4_f32:
  4347. case X64Reg_XMM4_f64:
  4348. case X64Reg_M128_XMM4: return X64Reg_M128_XMM4;
  4349. case X64Reg_XMM5_f32:
  4350. case X64Reg_XMM5_f64:
  4351. case X64Reg_M128_XMM5: return X64Reg_M128_XMM5;
  4352. case X64Reg_XMM6_f32:
  4353. case X64Reg_XMM6_f64:
  4354. case X64Reg_M128_XMM6: return X64Reg_M128_XMM6;
  4355. case X64Reg_XMM7_f32:
  4356. case X64Reg_XMM7_f64:
  4357. case X64Reg_M128_XMM7: return X64Reg_M128_XMM7;
  4358. case X64Reg_XMM8_f32:
  4359. case X64Reg_XMM8_f64:
  4360. case X64Reg_M128_XMM8: return X64Reg_M128_XMM8;
  4361. case X64Reg_XMM9_f32:
  4362. case X64Reg_XMM9_f64:
  4363. case X64Reg_M128_XMM9: return X64Reg_M128_XMM9;
  4364. case X64Reg_XMM10_f32:
  4365. case X64Reg_XMM10_f64:
  4366. case X64Reg_M128_XMM10: return X64Reg_M128_XMM10;
  4367. case X64Reg_XMM11_f32:
  4368. case X64Reg_XMM11_f64:
  4369. case X64Reg_M128_XMM11: return X64Reg_M128_XMM11;
  4370. case X64Reg_XMM12_f32:
  4371. case X64Reg_XMM12_f64:
  4372. case X64Reg_M128_XMM12: return X64Reg_M128_XMM12;
  4373. case X64Reg_XMM13_f32:
  4374. case X64Reg_XMM13_f64:
  4375. case X64Reg_M128_XMM13: return X64Reg_M128_XMM13;
  4376. case X64Reg_XMM14_f32:
  4377. case X64Reg_XMM14_f64:
  4378. case X64Reg_M128_XMM14: return X64Reg_M128_XMM14;
  4379. case X64Reg_XMM15_f32:
  4380. case X64Reg_XMM15_f64:
  4381. case X64Reg_M128_XMM15: return X64Reg_M128_XMM15;
  4382. }
  4383. }
  4384. if (numBytes == 8)
  4385. {
  4386. switch (reg)
  4387. {
  4388. case X64Reg_DIL:
  4389. case X64Reg_DI:
  4390. case X64Reg_EDI:
  4391. case X64Reg_RDI: return X64Reg_RDI;
  4392. case X64Reg_SIL:
  4393. case X64Reg_SI:
  4394. case X64Reg_ESI:
  4395. case X64Reg_RSI: return X64Reg_RSI;
  4396. case X64Reg_AL:
  4397. case X64Reg_AH:
  4398. case X64Reg_AX:
  4399. case X64Reg_EAX:
  4400. case X64Reg_RAX: return X64Reg_RAX;
  4401. case X64Reg_DL:
  4402. case X64Reg_DH:
  4403. case X64Reg_DX:
  4404. case X64Reg_EDX:
  4405. case X64Reg_RDX: return X64Reg_RDX;
  4406. case X64Reg_CL:
  4407. case X64Reg_CH:
  4408. case X64Reg_CX:
  4409. case X64Reg_ECX:
  4410. case X64Reg_RCX: return X64Reg_RCX;
  4411. case X64Reg_BL:
  4412. case X64Reg_BH:
  4413. case X64Reg_BX:
  4414. case X64Reg_EBX:
  4415. case X64Reg_RBX: return X64Reg_RBX;
  4416. case X64Reg_R8B:
  4417. case X64Reg_R8W:
  4418. case X64Reg_R8D:
  4419. case X64Reg_R8: return X64Reg_R8;
  4420. case X64Reg_R9B:
  4421. case X64Reg_R9W:
  4422. case X64Reg_R9D:
  4423. case X64Reg_R9: return X64Reg_R9;
  4424. case X64Reg_R10B:
  4425. case X64Reg_R10W:
  4426. case X64Reg_R10D:
  4427. case X64Reg_R10: return X64Reg_R10;
  4428. case X64Reg_R11B:
  4429. case X64Reg_R11W:
  4430. case X64Reg_R11D:
  4431. case X64Reg_R11: return X64Reg_R11;
  4432. case X64Reg_R12B:
  4433. case X64Reg_R12W:
  4434. case X64Reg_R12D:
  4435. case X64Reg_R12: return X64Reg_R12;
  4436. case X64Reg_R13B:
  4437. case X64Reg_R13W:
  4438. case X64Reg_R13D:
  4439. case X64Reg_R13: return X64Reg_R13;
  4440. case X64Reg_R14B:
  4441. case X64Reg_R14W:
  4442. case X64Reg_R14D:
  4443. case X64Reg_R14: return X64Reg_R14;
  4444. case X64Reg_R15B:
  4445. case X64Reg_R15W:
  4446. case X64Reg_R15D:
  4447. case X64Reg_R15: return X64Reg_R15;
  4448. case X64Reg_XMM0_f32:
  4449. case X64Reg_XMM0_f64:
  4450. case X64Reg_M128_XMM0: return X64Reg_XMM0_f64;
  4451. case X64Reg_XMM1_f32:
  4452. case X64Reg_XMM1_f64:
  4453. case X64Reg_M128_XMM1: return X64Reg_XMM1_f64;
  4454. case X64Reg_XMM2_f32:
  4455. case X64Reg_XMM2_f64:
  4456. case X64Reg_M128_XMM2: return X64Reg_XMM2_f64;
  4457. case X64Reg_XMM3_f32:
  4458. case X64Reg_XMM3_f64:
  4459. case X64Reg_M128_XMM3: return X64Reg_XMM3_f64;
  4460. case X64Reg_XMM4_f32:
  4461. case X64Reg_XMM4_f64:
  4462. case X64Reg_M128_XMM4: return X64Reg_XMM4_f64;
  4463. case X64Reg_XMM5_f32:
  4464. case X64Reg_XMM5_f64:
  4465. case X64Reg_M128_XMM5: return X64Reg_XMM5_f64;
  4466. case X64Reg_XMM6_f32:
  4467. case X64Reg_XMM6_f64:
  4468. case X64Reg_M128_XMM6: return X64Reg_XMM6_f64;
  4469. case X64Reg_XMM7_f32:
  4470. case X64Reg_XMM7_f64:
  4471. case X64Reg_M128_XMM7: return X64Reg_XMM7_f64;
  4472. case X64Reg_XMM8_f32:
  4473. case X64Reg_XMM8_f64:
  4474. case X64Reg_M128_XMM8: return X64Reg_XMM8_f64;
  4475. case X64Reg_XMM9_f32:
  4476. case X64Reg_XMM9_f64:
  4477. case X64Reg_M128_XMM9: return X64Reg_XMM9_f64;
  4478. case X64Reg_XMM10_f32:
  4479. case X64Reg_XMM10_f64:
  4480. case X64Reg_M128_XMM10: return X64Reg_XMM10_f64;
  4481. case X64Reg_XMM11_f32:
  4482. case X64Reg_XMM11_f64:
  4483. case X64Reg_M128_XMM11: return X64Reg_XMM11_f64;
  4484. case X64Reg_XMM12_f32:
  4485. case X64Reg_XMM12_f64:
  4486. case X64Reg_M128_XMM12: return X64Reg_XMM12_f64;
  4487. case X64Reg_XMM13_f32:
  4488. case X64Reg_XMM13_f64:
  4489. case X64Reg_M128_XMM13: return X64Reg_XMM13_f64;
  4490. case X64Reg_XMM14_f32:
  4491. case X64Reg_XMM14_f64:
  4492. case X64Reg_M128_XMM14: return X64Reg_XMM14_f64;
  4493. case X64Reg_XMM15_f32:
  4494. case X64Reg_XMM15_f64:
  4495. case X64Reg_M128_XMM15: return X64Reg_XMM15_f64;
  4496. }
  4497. return reg;
  4498. }
  4499. if (numBytes == 4)
  4500. {
  4501. switch (reg)
  4502. {
  4503. case X64Reg_DIL:
  4504. case X64Reg_DI:
  4505. case X64Reg_EDI:
  4506. case X64Reg_RDI: return X64Reg_EDI;
  4507. case X64Reg_SIL:
  4508. case X64Reg_SI:
  4509. case X64Reg_ESI:
  4510. case X64Reg_RSI: return X64Reg_ESI;
  4511. case X64Reg_AL:
  4512. case X64Reg_AH:
  4513. case X64Reg_AX:
  4514. case X64Reg_EAX:
  4515. case X64Reg_RAX: return X64Reg_EAX;
  4516. case X64Reg_DL:
  4517. case X64Reg_DH:
  4518. case X64Reg_DX:
  4519. case X64Reg_EDX:
  4520. case X64Reg_RDX: return X64Reg_EDX;
  4521. case X64Reg_CL:
  4522. case X64Reg_CH:
  4523. case X64Reg_CX:
  4524. case X64Reg_ECX:
  4525. case X64Reg_RCX: return X64Reg_ECX;
  4526. case X64Reg_BL:
  4527. case X64Reg_BH:
  4528. case X64Reg_BX:
  4529. case X64Reg_EBX:
  4530. case X64Reg_RBX: return X64Reg_EBX;
  4531. case X64Reg_R8B:
  4532. case X64Reg_R8W:
  4533. case X64Reg_R8D:
  4534. case X64Reg_R8: return X64Reg_R8D;
  4535. case X64Reg_R9B:
  4536. case X64Reg_R9W:
  4537. case X64Reg_R9D:
  4538. case X64Reg_R9: return X64Reg_R9D;
  4539. case X64Reg_R10B:
  4540. case X64Reg_R10W:
  4541. case X64Reg_R10D:
  4542. case X64Reg_R10: return X64Reg_R10D;
  4543. case X64Reg_R11B:
  4544. case X64Reg_R11W:
  4545. case X64Reg_R11D:
  4546. case X64Reg_R11: return X64Reg_R11D;
  4547. case X64Reg_R12B:
  4548. case X64Reg_R12W:
  4549. case X64Reg_R12D:
  4550. case X64Reg_R12: return X64Reg_R12D;
  4551. case X64Reg_R13B:
  4552. case X64Reg_R13W:
  4553. case X64Reg_R13D:
  4554. case X64Reg_R13: return X64Reg_R13D;
  4555. case X64Reg_R14B:
  4556. case X64Reg_R14W:
  4557. case X64Reg_R14D:
  4558. case X64Reg_R14: return X64Reg_R14D;
  4559. case X64Reg_R15B:
  4560. case X64Reg_R15W:
  4561. case X64Reg_R15D:
  4562. case X64Reg_R15: return X64Reg_R15D;
  4563. case X64Reg_XMM0_f32:
  4564. case X64Reg_XMM0_f64:
  4565. case X64Reg_M128_XMM0: return X64Reg_XMM0_f32;
  4566. case X64Reg_XMM1_f32:
  4567. case X64Reg_XMM1_f64:
  4568. case X64Reg_M128_XMM1: return X64Reg_XMM1_f32;
  4569. case X64Reg_XMM2_f32:
  4570. case X64Reg_XMM2_f64:
  4571. case X64Reg_M128_XMM2: return X64Reg_XMM2_f32;
  4572. case X64Reg_XMM3_f32:
  4573. case X64Reg_XMM3_f64:
  4574. case X64Reg_M128_XMM3: return X64Reg_XMM3_f32;
  4575. case X64Reg_XMM4_f32:
  4576. case X64Reg_XMM4_f64:
  4577. case X64Reg_M128_XMM4: return X64Reg_XMM4_f32;
  4578. case X64Reg_XMM5_f32:
  4579. case X64Reg_XMM5_f64:
  4580. case X64Reg_M128_XMM5: return X64Reg_XMM5_f32;
  4581. case X64Reg_XMM6_f32:
  4582. case X64Reg_XMM6_f64:
  4583. case X64Reg_M128_XMM6: return X64Reg_XMM6_f32;
  4584. case X64Reg_XMM7_f32:
  4585. case X64Reg_XMM7_f64:
  4586. case X64Reg_M128_XMM7: return X64Reg_XMM7_f32;
  4587. case X64Reg_XMM8_f32:
  4588. case X64Reg_XMM8_f64:
  4589. case X64Reg_M128_XMM8: return X64Reg_XMM8_f32;
  4590. case X64Reg_XMM9_f32:
  4591. case X64Reg_XMM9_f64:
  4592. case X64Reg_M128_XMM9: return X64Reg_XMM9_f32;
  4593. case X64Reg_XMM10_f32:
  4594. case X64Reg_XMM10_f64:
  4595. case X64Reg_M128_XMM10: return X64Reg_XMM10_f32;
  4596. case X64Reg_XMM11_f32:
  4597. case X64Reg_XMM11_f64:
  4598. case X64Reg_M128_XMM11: return X64Reg_XMM11_f32;
  4599. case X64Reg_XMM12_f32:
  4600. case X64Reg_XMM12_f64:
  4601. case X64Reg_M128_XMM12: return X64Reg_XMM12_f32;
  4602. case X64Reg_XMM13_f32:
  4603. case X64Reg_XMM13_f64:
  4604. case X64Reg_M128_XMM13: return X64Reg_XMM13_f32;
  4605. case X64Reg_XMM14_f32:
  4606. case X64Reg_XMM14_f64:
  4607. case X64Reg_M128_XMM14: return X64Reg_XMM14_f32;
  4608. case X64Reg_XMM15_f32:
  4609. case X64Reg_XMM15_f64:
  4610. case X64Reg_M128_XMM15: return X64Reg_XMM15_f32;
  4611. }
  4612. }
  4613. if (numBytes == 2)
  4614. {
  4615. switch (reg)
  4616. {
  4617. case X64Reg_DIL:
  4618. case X64Reg_DI:
  4619. case X64Reg_EDI:
  4620. case X64Reg_RDI: return X64Reg_DI;
  4621. case X64Reg_SIL:
  4622. case X64Reg_SI:
  4623. case X64Reg_ESI:
  4624. case X64Reg_RSI: return X64Reg_SI;
  4625. case X64Reg_AL:
  4626. case X64Reg_AH:
  4627. case X64Reg_AX:
  4628. case X64Reg_EAX:
  4629. case X64Reg_RAX: return X64Reg_AX;
  4630. case X64Reg_DL:
  4631. case X64Reg_DH:
  4632. case X64Reg_DX:
  4633. case X64Reg_EDX:
  4634. case X64Reg_RDX: return X64Reg_DX;
  4635. case X64Reg_CL:
  4636. case X64Reg_CH:
  4637. case X64Reg_CX:
  4638. case X64Reg_ECX:
  4639. case X64Reg_RCX: return X64Reg_CX;
  4640. case X64Reg_BL:
  4641. case X64Reg_BH:
  4642. case X64Reg_BX:
  4643. case X64Reg_EBX:
  4644. case X64Reg_RBX: return X64Reg_BX;
  4645. case X64Reg_R8B:
  4646. case X64Reg_R8W:
  4647. case X64Reg_R8D:
  4648. case X64Reg_R8: return X64Reg_R8W;
  4649. case X64Reg_R9B:
  4650. case X64Reg_R9W:
  4651. case X64Reg_R9D:
  4652. case X64Reg_R9: return X64Reg_R9W;
  4653. case X64Reg_R10B:
  4654. case X64Reg_R10W:
  4655. case X64Reg_R10D:
  4656. case X64Reg_R10: return X64Reg_R10W;
  4657. case X64Reg_R11B:
  4658. case X64Reg_R11W:
  4659. case X64Reg_R11D:
  4660. case X64Reg_R11: return X64Reg_R11W;
  4661. case X64Reg_R12B:
  4662. case X64Reg_R12W:
  4663. case X64Reg_R12D:
  4664. case X64Reg_R12: return X64Reg_R12W;
  4665. case X64Reg_R13B:
  4666. case X64Reg_R13W:
  4667. case X64Reg_R13D:
  4668. case X64Reg_R13: return X64Reg_R13W;
  4669. case X64Reg_R14B:
  4670. case X64Reg_R14W:
  4671. case X64Reg_R14D:
  4672. case X64Reg_R14: return X64Reg_R14W;
  4673. case X64Reg_R15B:
  4674. case X64Reg_R15W:
  4675. case X64Reg_R15D:
  4676. case X64Reg_R15: return X64Reg_R15W;
  4677. }
  4678. }
  4679. if (numBytes == 1)
  4680. {
  4681. switch (reg)
  4682. {
  4683. case X64Reg_DIL:
  4684. case X64Reg_DI:
  4685. case X64Reg_EDI:
  4686. case X64Reg_RDI: return X64Reg_DIL;
  4687. case X64Reg_SIL:
  4688. case X64Reg_SI:
  4689. case X64Reg_ESI:
  4690. case X64Reg_RSI: return X64Reg_SIL;
  4691. case X64Reg_AH: return X64Reg_AH;
  4692. case X64Reg_AL:
  4693. case X64Reg_AX:
  4694. case X64Reg_EAX:
  4695. case X64Reg_RAX: return X64Reg_AL;
  4696. case X64Reg_DH: return X64Reg_DH;
  4697. case X64Reg_DL:
  4698. case X64Reg_DX:
  4699. case X64Reg_EDX:
  4700. case X64Reg_RDX: return X64Reg_DL;
  4701. case X64Reg_CH: return X64Reg_CH;
  4702. case X64Reg_CL:
  4703. case X64Reg_CX:
  4704. case X64Reg_ECX:
  4705. case X64Reg_RCX: return X64Reg_CL;
  4706. case X64Reg_BH: return X64Reg_BH;
  4707. case X64Reg_BL:
  4708. case X64Reg_BX:
  4709. case X64Reg_EBX:
  4710. case X64Reg_RBX: return X64Reg_BL;
  4711. case X64Reg_R8B:
  4712. case X64Reg_R8W:
  4713. case X64Reg_R8D:
  4714. case X64Reg_R8: return X64Reg_R8B;
  4715. case X64Reg_R9B:
  4716. case X64Reg_R9W:
  4717. case X64Reg_R9D:
  4718. case X64Reg_R9: return X64Reg_R9B;
  4719. case X64Reg_R10B:
  4720. case X64Reg_R10W:
  4721. case X64Reg_R10D:
  4722. case X64Reg_R10: return X64Reg_R10B;
  4723. case X64Reg_R11B:
  4724. case X64Reg_R11W:
  4725. case X64Reg_R11D:
  4726. case X64Reg_R11: return X64Reg_R11B;
  4727. case X64Reg_R12B:
  4728. case X64Reg_R12W:
  4729. case X64Reg_R12D:
  4730. case X64Reg_R12: return X64Reg_R12B;
  4731. case X64Reg_R13B:
  4732. case X64Reg_R13W:
  4733. case X64Reg_R13D:
  4734. case X64Reg_R13: return X64Reg_R13B;
  4735. case X64Reg_R14B:
  4736. case X64Reg_R14W:
  4737. case X64Reg_R14D:
  4738. case X64Reg_R14: return X64Reg_R14B;
  4739. case X64Reg_R15B:
  4740. case X64Reg_R15W:
  4741. case X64Reg_R15D:
  4742. case X64Reg_R15: return X64Reg_R15B;
  4743. }
  4744. }
  4745. return X64Reg_None;
  4746. }
  4747. X64CPURegister BeMCContext::ResizeRegister(X64CPURegister reg, BeType* type)
  4748. {
  4749. if (type->IsVector())
  4750. return ResizeRegister(reg, 16);
  4751. return ResizeRegister(reg, type->mSize);
  4752. }
  4753. X64CPURegister BeMCContext::GetFullRegister(X64CPURegister reg)
  4754. {
  4755. switch (reg)
  4756. {
  4757. case X64Reg_XMM0_f32:
  4758. case X64Reg_XMM0_f64:
  4759. case X64Reg_M128_XMM0: return X64Reg_M128_XMM0;
  4760. case X64Reg_XMM1_f32:
  4761. case X64Reg_XMM1_f64:
  4762. case X64Reg_M128_XMM1: return X64Reg_M128_XMM1;
  4763. case X64Reg_XMM2_f32:
  4764. case X64Reg_XMM2_f64:
  4765. case X64Reg_M128_XMM2: return X64Reg_M128_XMM2;
  4766. case X64Reg_XMM3_f32:
  4767. case X64Reg_XMM3_f64:
  4768. case X64Reg_M128_XMM3: return X64Reg_M128_XMM3;
  4769. case X64Reg_XMM4_f32:
  4770. case X64Reg_XMM4_f64:
  4771. case X64Reg_M128_XMM4: return X64Reg_M128_XMM4;
  4772. case X64Reg_XMM5_f32:
  4773. case X64Reg_XMM5_f64:
  4774. case X64Reg_M128_XMM5: return X64Reg_M128_XMM5;
  4775. case X64Reg_XMM6_f32:
  4776. case X64Reg_XMM6_f64:
  4777. case X64Reg_M128_XMM6: return X64Reg_M128_XMM6;
  4778. case X64Reg_XMM7_f32:
  4779. case X64Reg_XMM7_f64:
  4780. case X64Reg_M128_XMM7: return X64Reg_M128_XMM7;
  4781. case X64Reg_XMM8_f32:
  4782. case X64Reg_XMM8_f64:
  4783. case X64Reg_M128_XMM8: return X64Reg_M128_XMM8;
  4784. case X64Reg_XMM9_f32:
  4785. case X64Reg_XMM9_f64:
  4786. case X64Reg_M128_XMM9: return X64Reg_M128_XMM9;
  4787. case X64Reg_XMM10_f32:
  4788. case X64Reg_XMM10_f64:
  4789. case X64Reg_M128_XMM10: return X64Reg_M128_XMM10;
  4790. case X64Reg_XMM11_f32:
  4791. case X64Reg_XMM11_f64:
  4792. case X64Reg_M128_XMM11: return X64Reg_M128_XMM11;
  4793. case X64Reg_XMM12_f32:
  4794. case X64Reg_XMM12_f64:
  4795. case X64Reg_M128_XMM12: return X64Reg_M128_XMM12;
  4796. case X64Reg_XMM13_f32:
  4797. case X64Reg_XMM13_f64:
  4798. case X64Reg_M128_XMM13: return X64Reg_M128_XMM13;
  4799. case X64Reg_XMM14_f32:
  4800. case X64Reg_XMM14_f64:
  4801. case X64Reg_M128_XMM14: return X64Reg_M128_XMM14;
  4802. case X64Reg_XMM15_f32:
  4803. case X64Reg_XMM15_f64:
  4804. case X64Reg_M128_XMM15: return X64Reg_M128_XMM15;
  4805. }
  4806. return ResizeRegister(reg, 8);
  4807. }
  4808. bool BeMCContext::HasLoad(const BeMCOperand& operand)
  4809. {
  4810. if (operand.mKind == BeMCOperandKind_VRegLoad)
  4811. return true;
  4812. if (operand.mKind != BeMCOperandKind_VReg)
  4813. return false;
  4814. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  4815. if (HasLoad(vregInfo->mRelTo))
  4816. return true;
  4817. if (HasLoad(vregInfo->mRelOffset))
  4818. return true;
  4819. return false;
  4820. }
  4821. bool BeMCContext::IsAddress(BeMCOperand& operand)
  4822. {
  4823. if (operand.mKind == BeMCOperandKind_VRegAddr)
  4824. return true;
  4825. if (operand.mKind != BeMCOperandKind_VReg)
  4826. return false;
  4827. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  4828. return IsAddress(vregInfo->mRelTo);
  4829. }
  4830. bool BeMCContext::IsAddressable(BeMCOperand& operand)
  4831. {
  4832. if (operand.mKind == BeMCOperandKind_Symbol)
  4833. return true;
  4834. if (operand.mKind == BeMCOperandKind_NativeReg)
  4835. return true;
  4836. if (operand.mKind == BeMCOperandKind_VRegAddr)
  4837. return true;
  4838. if (operand.mKind != BeMCOperandKind_VReg)
  4839. return false;
  4840. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  4841. return IsAddressable(vregInfo->mRelTo);
  4842. }
  4843. bool BeMCContext::IsVRegExpr(BeMCOperand& operand)
  4844. {
  4845. if (!operand.IsVRegAny())
  4846. return false;
  4847. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  4848. return vregInfo->mIsExpr;
  4849. }
  4850. // Depth -1 is reserved explicitly for reg finalization, so we leave the vector type for emission
  4851. void BeMCContext::FixOperand(BeMCOperand& operand, int depth)
  4852. {
  4853. // We don't want to check for VRegLoad, that would erase the dereference
  4854. if ((operand.mKind != BeMCOperandKind_VReg) && (operand.mKind != BeMCOperandKind_VRegAddr))
  4855. return;
  4856. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  4857. if (vregInfo->mReg != X64Reg_None)
  4858. {
  4859. // For vectors we need the explicit type info
  4860. if ((depth != -1) || (!vregInfo->mType->IsVector()))
  4861. {
  4862. operand.mKind = BeMCOperandKind_NativeReg;
  4863. operand.mReg = vregInfo->mReg;
  4864. return;
  4865. }
  4866. }
  4867. if ((vregInfo->mIsRetVal) && (mCompositeRetVRegIdx != -1) && (mCompositeRetVRegIdx != operand.mVRegIdx))
  4868. {
  4869. BF_ASSERT(mCompositeRetVRegIdx != -1);
  4870. BeMCOperand origOperand = operand;
  4871. operand = BeMCOperand::FromVReg(mCompositeRetVRegIdx);
  4872. if ((origOperand.mKind == BeMCOperandKind_VReg) && (vregInfo->mType->IsNonVectorComposite()))
  4873. operand.mKind = BeMCOperandKind_VRegLoad;
  4874. FixOperand(operand, depth + 1);
  4875. //auto retVRegInfo = mVRegInfo[mCompositeRetVRegIdx];
  4876. //operand = BeMCOperand::FromReg(retVRegInfo->mReg);
  4877. }
  4878. if (vregInfo->IsDirectRelToAny())
  4879. {
  4880. auto checkOperand = vregInfo->mRelTo;
  4881. if (checkOperand.IsVReg())
  4882. FixOperand(checkOperand, depth + 1);
  4883. if (checkOperand.IsNativeReg())
  4884. {
  4885. auto resizedReg = ResizeRegister(checkOperand.mReg, vregInfo->mType);
  4886. if (resizedReg != X64Reg_None)
  4887. operand = BeMCOperand::FromReg(resizedReg);
  4888. }
  4889. else if (checkOperand.mKind == BeMCOperandKind_Symbol)
  4890. {
  4891. auto symbol = mCOFFObject->mSymbols[checkOperand.mSymbolIdx];
  4892. if (AreTypesEquivalent(vregInfo->mType, symbol->mType))
  4893. {
  4894. if (checkOperand.mKind == BeMCOperandKind_VRegAddr)
  4895. {
  4896. operand = OperandToAddr(checkOperand);
  4897. }
  4898. else
  4899. operand = checkOperand;
  4900. }
  4901. }
  4902. else if (checkOperand.mKind == BeMCOperandKind_SymbolAddr)
  4903. {
  4904. operand = checkOperand;
  4905. }
  4906. else if (checkOperand.IsImmediate())
  4907. {
  4908. operand = checkOperand;
  4909. }
  4910. }
  4911. }
  4912. BeMCOperand BeMCContext::GetFixedOperand(const BeMCOperand& operand)
  4913. {
  4914. BeMCOperand copyOp = operand;
  4915. FixOperand(copyOp);
  4916. return copyOp;
  4917. }
  4918. uint8 BeMCContext::GetREX(const BeMCOperand& r, const BeMCOperand& rm, bool is64Bit)
  4919. {
  4920. //bool is64Bit = false;
  4921. bool is64BitExR = false;
  4922. bool is64BitExRM = false;
  4923. bool forceRex = false;
  4924. if (r.mKind == BeMCOperandKind_NativeReg)
  4925. {
  4926. /*if ((r.mReg >= X64Reg_RAX) && (r.mReg <= X64Reg_EFL))
  4927. BF_ASSERT(is64Bit); */
  4928. switch (r.mReg)
  4929. {
  4930. case X64Reg_SIL:
  4931. case X64Reg_DIL:
  4932. forceRex = true;
  4933. break;
  4934. case X64Reg_R8: case X64Reg_R9: case X64Reg_R10: case X64Reg_R11: case X64Reg_R12: case X64Reg_R13: case X64Reg_R14: case X64Reg_R15:
  4935. case X64Reg_R8D: case X64Reg_R9D: case X64Reg_R10D: case X64Reg_R11D: case X64Reg_R12D: case X64Reg_R13D: case X64Reg_R14D: case X64Reg_R15D:
  4936. case X64Reg_R8W: case X64Reg_R9W: case X64Reg_R10W: case X64Reg_R11W: case X64Reg_R12W: case X64Reg_R13W: case X64Reg_R14W: case X64Reg_R15W:
  4937. case X64Reg_R8B: case X64Reg_R9B: case X64Reg_R10B: case X64Reg_R11B: case X64Reg_R12B: case X64Reg_R13B: case X64Reg_R14B: case X64Reg_R15B:
  4938. case X64Reg_XMM8_f64: case X64Reg_XMM9_f64: case X64Reg_XMM10_f64: case X64Reg_XMM11_f64:
  4939. case X64Reg_XMM12_f64: case X64Reg_XMM13_f64: case X64Reg_XMM14_f64: case X64Reg_XMM15_f64:
  4940. case X64Reg_XMM8_f32: case X64Reg_XMM9_f32: case X64Reg_XMM10_f32: case X64Reg_XMM11_f32:
  4941. case X64Reg_XMM12_f32: case X64Reg_XMM13_f32: case X64Reg_XMM14_f32: case X64Reg_XMM15_f32:
  4942. case X64Reg_M128_XMM8: case X64Reg_M128_XMM9: case X64Reg_M128_XMM10: case X64Reg_M128_XMM11:
  4943. case X64Reg_M128_XMM12: case X64Reg_M128_XMM13: case X64Reg_M128_XMM14: case X64Reg_M128_XMM15:
  4944. is64BitExR = true;
  4945. }
  4946. }
  4947. bool hasSibExRM = false;
  4948. if (rm.mKind == BeMCOperandKind_NativeReg)
  4949. {
  4950. switch (rm.mReg)
  4951. {
  4952. case X64Reg_SIL:
  4953. case X64Reg_DIL:
  4954. forceRex = true;
  4955. break;
  4956. case X64Reg_R8: case X64Reg_R9: case X64Reg_R10: case X64Reg_R11: case X64Reg_R12: case X64Reg_R13: case X64Reg_R14: case X64Reg_R15:
  4957. case X64Reg_R8D: case X64Reg_R9D: case X64Reg_R10D: case X64Reg_R11D: case X64Reg_R12D: case X64Reg_R13D: case X64Reg_R14D: case X64Reg_R15D:
  4958. case X64Reg_R8W: case X64Reg_R9W: case X64Reg_R10W: case X64Reg_R11W: case X64Reg_R12W: case X64Reg_R13W: case X64Reg_R14W: case X64Reg_R15W:
  4959. case X64Reg_R8B: case X64Reg_R9B: case X64Reg_R10B: case X64Reg_R11B: case X64Reg_R12B: case X64Reg_R13B: case X64Reg_R14B: case X64Reg_R15B:
  4960. case X64Reg_XMM8_f64: case X64Reg_XMM9_f64: case X64Reg_XMM10_f64: case X64Reg_XMM11_f64:
  4961. case X64Reg_XMM12_f64: case X64Reg_XMM13_f64: case X64Reg_XMM14_f64: case X64Reg_XMM15_f64:
  4962. case X64Reg_XMM8_f32: case X64Reg_XMM9_f32: case X64Reg_XMM10_f32: case X64Reg_XMM11_f32:
  4963. case X64Reg_XMM12_f32: case X64Reg_XMM13_f32: case X64Reg_XMM14_f32: case X64Reg_XMM15_f32:
  4964. case X64Reg_M128_XMM8: case X64Reg_M128_XMM9: case X64Reg_M128_XMM10: case X64Reg_M128_XMM11:
  4965. case X64Reg_M128_XMM12: case X64Reg_M128_XMM13: case X64Reg_M128_XMM14: case X64Reg_M128_XMM15:
  4966. is64BitExRM = true;
  4967. }
  4968. }
  4969. else if (rm.IsVRegAny())
  4970. {
  4971. auto vregInfo = mVRegInfo[rm.mVRegIdx];
  4972. if (vregInfo->IsDirectRelTo())
  4973. return GetREX(r, vregInfo->mRelTo, is64Bit);
  4974. BeRMParamsInfo rmInfo;
  4975. GetRMParams(rm, rmInfo);
  4976. is64BitExRM |= ((rmInfo.mRegA >= X64Reg_R8) && (rmInfo.mRegA <= X64Reg_R15)) || ((rmInfo.mRegA >= X64Reg_R8D) && (rmInfo.mRegA <= X64Reg_R15D));
  4977. hasSibExRM |= ((rmInfo.mRegB >= X64Reg_R8) && (rmInfo.mRegB <= X64Reg_R15)) || ((rmInfo.mRegB >= X64Reg_R8D) && (rmInfo.mRegB <= X64Reg_R15D));
  4978. }
  4979. else if (rm.IsSymbol())
  4980. {
  4981. auto sym = mCOFFObject->mSymbols[rm.mSymbolIdx];
  4982. if (sym->mIsTLS)
  4983. {
  4984. auto tlsReg = mVRegInfo[mTLSVRegIdx]->mReg;
  4985. is64BitExRM |= (tlsReg >= X64Reg_R8) && (tlsReg <= X64Reg_R15);
  4986. }
  4987. }
  4988. uint8 flags = 0;
  4989. if (is64Bit)
  4990. flags |= 8;
  4991. if (is64BitExR)
  4992. flags |= 4;
  4993. if (hasSibExRM)
  4994. flags |= 2;
  4995. if (is64BitExRM)
  4996. flags |= 1;
  4997. if ((flags != 0) || (forceRex))
  4998. return (uint8)(0x40 | flags);
  4999. return 0;
  5000. }
  5001. void BeMCContext::EmitREX(const BeMCOperand& r, const BeMCOperand& rm, bool is64Bit)
  5002. {
  5003. uint8 rex = GetREX(r, rm, is64Bit);
  5004. if (rex != 0)
  5005. mOut.Write(rex);
  5006. }
  5007. uint8 BeMCContext::EncodeRegNum(X64CPURegister regNum)
  5008. {
  5009. switch (regNum)
  5010. {
  5011. case X64Reg_AL:
  5012. case X64Reg_AX:
  5013. case X64Reg_EAX:
  5014. case X64Reg_RAX:
  5015. case X64Reg_R8:
  5016. case X64Reg_R8D:
  5017. case X64Reg_R8W:
  5018. case X64Reg_R8B:
  5019. case X64Reg_MM0:
  5020. case X64Reg_XMM0_f32:
  5021. case X64Reg_XMM0_f64:
  5022. case X64Reg_M128_XMM0:
  5023. case X64Reg_XMM8_f32:
  5024. case X64Reg_XMM8_f64:
  5025. case X64Reg_M128_XMM8:
  5026. return 0;
  5027. case X64Reg_CL:
  5028. case X64Reg_CX:
  5029. case X64Reg_ECX:
  5030. case X64Reg_RCX:
  5031. case X64Reg_R9:
  5032. case X64Reg_R9D:
  5033. case X64Reg_R9W:
  5034. case X64Reg_R9B:
  5035. case X64Reg_MM1:
  5036. case X64Reg_XMM1_f32:
  5037. case X64Reg_XMM1_f64:
  5038. case X64Reg_M128_XMM1:
  5039. case X64Reg_XMM9_f32:
  5040. case X64Reg_XMM9_f64:
  5041. case X64Reg_M128_XMM9:
  5042. return 1;
  5043. case X64Reg_DL:
  5044. case X64Reg_DX:
  5045. case X64Reg_EDX:
  5046. case X64Reg_RDX:
  5047. case X64Reg_R10:
  5048. case X64Reg_R10D:
  5049. case X64Reg_R10W:
  5050. case X64Reg_R10B:
  5051. case X64Reg_MM2:
  5052. case X64Reg_XMM2_f32:
  5053. case X64Reg_XMM2_f64:
  5054. case X64Reg_M128_XMM2:
  5055. case X64Reg_XMM10_f32:
  5056. case X64Reg_XMM10_f64:
  5057. case X64Reg_M128_XMM10:
  5058. return 2;
  5059. case X64Reg_BL:
  5060. case X64Reg_BX:
  5061. case X64Reg_EBX:
  5062. case X64Reg_RBX:
  5063. case X64Reg_R11:
  5064. case X64Reg_R11D:
  5065. case X64Reg_R11W:
  5066. case X64Reg_R11B:
  5067. case X64Reg_MM3:
  5068. case X64Reg_XMM3_f32:
  5069. case X64Reg_XMM3_f64:
  5070. case X64Reg_M128_XMM3:
  5071. case X64Reg_XMM11_f32:
  5072. case X64Reg_XMM11_f64:
  5073. case X64Reg_M128_XMM11:
  5074. return 3;
  5075. case X64Reg_None: // Useful for SIB/RM addr encodings
  5076. case X64Reg_AH:
  5077. //case X64Reg_SP:
  5078. //case X64Reg_ESP:
  5079. case X64Reg_RSP:
  5080. case X64Reg_R12:
  5081. case X64Reg_R12D:
  5082. case X64Reg_R12W:
  5083. case X64Reg_R12B:
  5084. case X64Reg_MM4:
  5085. case X64Reg_XMM4_f32:
  5086. case X64Reg_XMM4_f64:
  5087. case X64Reg_M128_XMM4:
  5088. case X64Reg_XMM12_f32:
  5089. case X64Reg_XMM12_f64:
  5090. case X64Reg_M128_XMM12:
  5091. return 4;
  5092. case X64Reg_CH:
  5093. //case X64Reg_BP:
  5094. //case X64Reg_EBP:
  5095. case X64Reg_RBP:
  5096. case X64Reg_R13:
  5097. case X64Reg_R13D:
  5098. case X64Reg_R13W:
  5099. case X64Reg_R13B:
  5100. case X64Reg_MM5:
  5101. case X64Reg_XMM5_f32:
  5102. case X64Reg_XMM5_f64:
  5103. case X64Reg_M128_XMM5:
  5104. case X64Reg_XMM13_f32:
  5105. case X64Reg_XMM13_f64:
  5106. case X64Reg_M128_XMM13:
  5107. return 5;
  5108. case X64Reg_DH:
  5109. case X64Reg_SIL:
  5110. case X64Reg_SI:
  5111. case X64Reg_ESI:
  5112. case X64Reg_RSI:
  5113. case X64Reg_R14:
  5114. case X64Reg_R14D:
  5115. case X64Reg_R14W:
  5116. case X64Reg_R14B:
  5117. case X64Reg_MM6:
  5118. case X64Reg_XMM6_f32:
  5119. case X64Reg_XMM6_f64:
  5120. case X64Reg_M128_XMM6:
  5121. case X64Reg_XMM14_f32:
  5122. case X64Reg_XMM14_f64:
  5123. case X64Reg_M128_XMM14:
  5124. return 6;
  5125. case X64Reg_BH:
  5126. case X64Reg_DIL:
  5127. case X64Reg_DI:
  5128. case X64Reg_EDI:
  5129. case X64Reg_RDI:
  5130. case X64Reg_R15:
  5131. case X64Reg_R15D:
  5132. case X64Reg_R15W:
  5133. case X64Reg_R15B:
  5134. case X64Reg_MM7:
  5135. case X64Reg_XMM7_f32:
  5136. case X64Reg_XMM7_f64:
  5137. case X64Reg_M128_XMM7:
  5138. case X64Reg_XMM15_f32:
  5139. case X64Reg_XMM15_f64:
  5140. case X64Reg_M128_XMM15:
  5141. return 7;
  5142. }
  5143. SoftFail("Invalid reg");
  5144. return -1;
  5145. }
  5146. int BeMCContext::GetRegSize(int regNum)
  5147. {
  5148. if ((regNum >= X64Reg_EAX) && (regNum <= X64Reg_EDI))
  5149. return 4;
  5150. if ((regNum >= X64Reg_AX) && (regNum <= X64Reg_BX))
  5151. return 2;
  5152. if ((regNum >= X64Reg_AL) && (regNum <= X64Reg_BH))
  5153. return 1;
  5154. return 8;
  5155. }
  5156. void BeMCContext::ValidateRMResult(const BeMCOperand& operand, BeRMParamsInfo& rmInfo, bool doValidate)
  5157. {
  5158. if (!doValidate)
  5159. return;
  5160. if (rmInfo.mMode == BeMCRMMode_Invalid)
  5161. return;
  5162. //TODO: WTF- this previous version just seems to be wrong! Why did think this was true? the REX.X and REX.B flags fix these
  5163. // in a SIB, the base can't be R13 (which is RBP+REX), and the scaled index can't be R12 (which is RSP+REX)
  5164. //if ((regB != X64Reg_None) &&
  5165. // ((regA == X64Reg_R13) || (regB == X64Reg_R12)))
  5166. //{
  5167. // // We can't just swap the regs if we have a scale applied
  5168. // if (bScale != 1)
  5169. // {
  5170. // if (errorVReg != NULL)
  5171. // *errorVReg = -2; // Scale error
  5172. // return BeMCRMMode_Invalid;
  5173. // }
  5174. // BF_SWAP(regA, regB);
  5175. //}
  5176. // In a SIB, the base can't be RBP, and the scaled index can't be RSP
  5177. if ((rmInfo.mRegB != X64Reg_None) &&
  5178. ((rmInfo.mRegA == X64Reg_RBP) || (rmInfo.mRegB == X64Reg_RSP)))
  5179. {
  5180. // We can't just swap the regs if we have a scale applied
  5181. if (rmInfo.mBScale != 1)
  5182. {
  5183. rmInfo.mErrorVReg = -2; // Scale error
  5184. rmInfo.mMode = BeMCRMMode_Invalid;
  5185. return;
  5186. }
  5187. BF_SWAP(rmInfo.mRegA, rmInfo.mRegB);
  5188. }
  5189. return;
  5190. }
  5191. void BeMCContext::GetRMParams(const BeMCOperand& operand, BeRMParamsInfo& rmInfo, bool doValidate)
  5192. {
  5193. BeMCRMMode rmMode = BeMCRMMode_Invalid;
  5194. if (operand.mKind == BeMCOperandKind_NativeReg)
  5195. {
  5196. if (rmInfo.mRegA == X64Reg_None)
  5197. rmInfo.mRegA = operand.mReg;
  5198. else if (rmInfo.mRegB == X64Reg_None)
  5199. rmInfo.mRegB = operand.mReg;
  5200. else
  5201. {
  5202. rmInfo.mMode = BeMCRMMode_Invalid;
  5203. return;
  5204. }
  5205. rmInfo.mMode = BeMCRMMode_Direct;
  5206. return ValidateRMResult(operand, rmInfo, doValidate);
  5207. }
  5208. else if (operand.IsImmediateInt())
  5209. {
  5210. rmInfo.mDisp += (int)operand.mImmediate;
  5211. rmInfo.mMode = BeMCRMMode_Direct;
  5212. return ValidateRMResult(operand, rmInfo, doValidate);
  5213. }
  5214. if (operand.mKind == BeMCOperandKind_VReg)
  5215. {
  5216. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5217. if (!vregInfo->mIsExpr)
  5218. {
  5219. auto reg = vregInfo->mReg;
  5220. if (reg != X64Reg_None)
  5221. {
  5222. if (rmInfo.mRegA == X64Reg_None)
  5223. rmInfo.mRegA = reg;
  5224. else if (rmInfo.mRegB == X64Reg_None)
  5225. rmInfo.mRegB = reg;
  5226. else
  5227. {
  5228. rmInfo.mMode = BeMCRMMode_Invalid;
  5229. return;
  5230. }
  5231. rmInfo.mMode = BeMCRMMode_Direct;
  5232. return ValidateRMResult(operand, rmInfo, doValidate);
  5233. }
  5234. GetRMParams(OperandToAddr(operand), rmInfo, doValidate);
  5235. if (rmInfo.mMode == BeMCRMMode_Invalid)
  5236. return;
  5237. BF_ASSERT(rmInfo.mMode == BeMCRMMode_Direct);
  5238. rmInfo.mMode = BeMCRMMode_Deref;
  5239. return ValidateRMResult(OperandToAddr(operand), rmInfo, doValidate);
  5240. }
  5241. // Fall through
  5242. }
  5243. else if (operand.mKind == BeMCOperandKind_VRegAddr)
  5244. {
  5245. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5246. if (vregInfo->mIsExpr)
  5247. {
  5248. if (vregInfo->IsDirectRelToAny())
  5249. {
  5250. if (vregInfo->mRelTo.mKind == BeMCOperandKind_VReg)
  5251. return GetRMParams(BeMCOperand::FromVRegAddr(vregInfo->mRelTo.mVRegIdx), rmInfo, doValidate);
  5252. else if (vregInfo->mRelTo.mKind == BeMCOperandKind_VRegLoad)
  5253. return GetRMParams(BeMCOperand::FromVReg(vregInfo->mRelTo.mVRegIdx), rmInfo, doValidate);
  5254. }
  5255. rmInfo.mErrorVReg = operand.mVRegIdx;
  5256. rmInfo.mMode = BeMCRMMode_Invalid;
  5257. return;
  5258. }
  5259. BF_ASSERT(!vregInfo->mIsExpr);
  5260. BF_ASSERT(vregInfo->mReg == X64Reg_None);
  5261. X64CPURegister reg = X64Reg_None;
  5262. if ((vregInfo->mIsRetVal) && (mCompositeRetVRegIdx != -1) && (mCompositeRetVRegIdx != operand.mVRegIdx))
  5263. {
  5264. return GetRMParams(BeMCOperand::FromVReg(mCompositeRetVRegIdx), rmInfo, doValidate);
  5265. }
  5266. reg = mUseBP ? X64Reg_RBP : X64Reg_RSP;
  5267. rmInfo.mDisp = mStackSize + vregInfo->mFrameOffset;
  5268. if (rmInfo.mRegA == X64Reg_None)
  5269. rmInfo.mRegA = reg;
  5270. else if (rmInfo.mRegB == X64Reg_None)
  5271. rmInfo.mRegB = reg;
  5272. rmInfo.mMode = BeMCRMMode_Direct;
  5273. return ValidateRMResult(operand, rmInfo, doValidate);
  5274. }
  5275. else if (operand.mKind == BeMCOperandKind_VRegLoad)
  5276. {
  5277. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5278. if (!vregInfo->mIsExpr)
  5279. {
  5280. auto reg = vregInfo->mReg;
  5281. if (reg == X64Reg_None)
  5282. {
  5283. rmInfo.mErrorVReg = operand.mVRegIdx;
  5284. rmInfo.mMode = BeMCRMMode_Invalid;
  5285. return;
  5286. }
  5287. if (rmInfo.mRegA == X64Reg_None)
  5288. rmInfo.mRegA = reg;
  5289. else if (rmInfo.mRegB == X64Reg_None)
  5290. rmInfo.mRegB = reg;
  5291. rmInfo.mMode = BeMCRMMode_Deref;
  5292. return ValidateRMResult(operand, rmInfo, doValidate);
  5293. }
  5294. }
  5295. else
  5296. {
  5297. rmInfo.mMode = BeMCRMMode_Invalid;
  5298. return;
  5299. }
  5300. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5301. BF_ASSERT(vregInfo->mIsExpr);
  5302. if (vregInfo->mRelTo)
  5303. {
  5304. auto oldRegA = rmInfo.mRegA;
  5305. GetRMParams(vregInfo->mRelTo, rmInfo, false);
  5306. if (rmInfo.mMode == BeMCRMMode_Invalid)
  5307. {
  5308. if (rmInfo.mErrorVReg == -1)
  5309. {
  5310. rmInfo.mErrorVReg = operand.mVRegIdx;
  5311. }
  5312. return;
  5313. }
  5314. if (rmInfo.mMode == BeMCRMMode_Deref)
  5315. {
  5316. // A deref can only stand alone, and no double-derefs
  5317. if ((vregInfo->mRelOffset) || (vregInfo->mRelOffsetScale != 1) || (operand.mKind == BeMCOperandKind_VRegLoad))
  5318. {
  5319. BF_ASSERT(vregInfo->mRelTo.IsVRegAny());
  5320. rmInfo.mErrorVReg = vregInfo->mRelTo.mVRegIdx;
  5321. // For some reason we had changed this to:
  5322. //*errorVReg = operand.mVRegIdx;
  5323. // This doesn't work, it's the deref that we want to isolate, otherwise we just end up creating another invalid expression
  5324. rmInfo.mMode = BeMCRMMode_Invalid;
  5325. return;
  5326. }
  5327. if (operand.mKind == BeMCOperandKind_VRegAddr)
  5328. {
  5329. rmInfo.mMode = BeMCRMMode_Direct;
  5330. return ValidateRMResult(vregInfo->mRelTo, rmInfo, doValidate);
  5331. }
  5332. else if (operand.mKind == BeMCOperandKind_VReg)
  5333. {
  5334. rmInfo.mMode = BeMCRMMode_Deref;
  5335. return ValidateRMResult(vregInfo->mRelTo, rmInfo, doValidate);
  5336. }
  5337. else
  5338. NotImpl();
  5339. }
  5340. }
  5341. if (vregInfo->mRelOffset)
  5342. {
  5343. if (vregInfo->mRelOffsetScale != 1)
  5344. rmInfo.mVRegWithScaledOffset = operand.mVRegIdx;
  5345. bool relToComplicated = (rmInfo.mRegB != X64Reg_None) || (rmInfo.mBScale != 1);
  5346. GetRMParams(vregInfo->mRelOffset, rmInfo, false);
  5347. if (rmInfo.mMode == BeMCRMMode_Invalid)
  5348. {
  5349. // Pick the "most complicated" between relOffset and relTo?
  5350. if (relToComplicated)
  5351. {
  5352. BF_ASSERT(vregInfo->mRelTo.IsVRegAny());
  5353. rmInfo.mErrorVReg = vregInfo->mRelTo.mVRegIdx;
  5354. }
  5355. else
  5356. {
  5357. BF_ASSERT(vregInfo->mRelOffset.IsVRegAny());
  5358. rmInfo.mErrorVReg = vregInfo->mRelOffset.mVRegIdx;
  5359. }
  5360. rmInfo.mMode = BeMCRMMode_Invalid;
  5361. return;
  5362. }
  5363. if (rmInfo.mMode == BeMCRMMode_Deref) // Deref only allowed on relTo
  5364. {
  5365. BF_ASSERT(vregInfo->mRelOffset.IsVRegAny());
  5366. rmInfo.mErrorVReg = vregInfo->mRelOffset.mVRegIdx;
  5367. rmInfo.mMode = BeMCRMMode_Invalid;
  5368. return;
  5369. }
  5370. }
  5371. bool success = true;
  5372. if (vregInfo->mRelOffsetScale != 1)
  5373. {
  5374. if (rmInfo.mBScale != 1)
  5375. success = false;
  5376. rmInfo.mBScale = vregInfo->mRelOffsetScale;
  5377. if ((rmInfo.mBScale != 2) && (rmInfo.mBScale != 4) && (rmInfo.mBScale != 8))
  5378. success = false;
  5379. if (rmInfo.mRegB == X64Reg_None)
  5380. {
  5381. rmInfo.mRegB = rmInfo.mRegA;
  5382. rmInfo.mRegA = X64Reg_None;
  5383. }
  5384. }
  5385. if (!success)
  5386. {
  5387. if (rmInfo.mErrorVReg == -1)
  5388. rmInfo.mErrorVReg = operand.mVRegIdx;
  5389. }
  5390. if (success)
  5391. {
  5392. if (operand.mKind == BeMCOperandKind_VRegLoad)
  5393. {
  5394. rmInfo.mMode = BeMCRMMode_Deref;
  5395. return ValidateRMResult(vregInfo->mRelOffset, rmInfo, doValidate);
  5396. }
  5397. else
  5398. {
  5399. rmInfo.mMode = BeMCRMMode_Direct;
  5400. return ValidateRMResult(vregInfo->mRelOffset, rmInfo, doValidate);
  5401. }
  5402. }
  5403. else
  5404. {
  5405. rmInfo.mMode = BeMCRMMode_Invalid;
  5406. return;
  5407. }
  5408. }
  5409. bool BeMCContext::HasImmediateTarget(const BeMCOperand& operand)
  5410. {
  5411. if (operand.IsImmediate())
  5412. return true;
  5413. auto vregInfo = GetVRegInfo(operand);
  5414. if (vregInfo == NULL)
  5415. return false;
  5416. if (vregInfo->mRelTo)
  5417. return HasImmediateTarget(vregInfo->mRelTo);
  5418. return false;
  5419. }
  5420. void BeMCContext::DisableRegister(const BeMCOperand& operand, X64CPURegister reg)
  5421. {
  5422. auto vregInfo = GetVRegInfo(operand);
  5423. if (vregInfo == NULL)
  5424. return;
  5425. switch (reg)
  5426. {
  5427. case X64Reg_RAX:
  5428. vregInfo->mDisableRAX = true;
  5429. break;
  5430. case X64Reg_RDX:
  5431. vregInfo->mDisableRDX = true;
  5432. break;
  5433. case X64Reg_SIL:
  5434. vregInfo->mDisableEx = true;
  5435. break;
  5436. default:
  5437. NotImpl();
  5438. }
  5439. if (vregInfo->mRelTo)
  5440. DisableRegister(vregInfo->mRelTo, reg);
  5441. if (vregInfo->mRelOffset)
  5442. DisableRegister(vregInfo->mRelOffset, reg);
  5443. }
  5444. void BeMCContext::MarkInvalidRMRegs(const BeMCOperand& operand)
  5445. {
  5446. if (!operand.IsVRegAny())
  5447. return;
  5448. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5449. if (vregInfo->mReg == X64Reg_R12) // Can't allow a scaled R12, it's illegal in ModRM
  5450. vregInfo->mDisableR12 = true;
  5451. if (vregInfo->mReg == X64Reg_R13) // Can't allow a base R13, it's illegal in ModRM
  5452. vregInfo->mDisableR13 = true;
  5453. MarkInvalidRMRegs(vregInfo->mRelTo);
  5454. MarkInvalidRMRegs(vregInfo->mRelOffset);
  5455. }
  5456. void BeMCContext::GetUsedRegs(const BeMCOperand& operand, X64CPURegister& regA, X64CPURegister& regB)
  5457. {
  5458. BeRMParamsInfo rmInfo;
  5459. GetRMParams(operand, rmInfo);
  5460. if (rmInfo.mRegA != X64Reg_None)
  5461. regA = GetFullRegister(rmInfo.mRegA);
  5462. if (rmInfo.mRegB != X64Reg_None)
  5463. regB = GetFullRegister(rmInfo.mRegB);
  5464. }
  5465. BeMCRMMode BeMCContext::GetRMForm(const BeMCOperand& operand, bool& isMulti)
  5466. {
  5467. BeRMParamsInfo rmInfo;
  5468. GetRMParams(operand, rmInfo);
  5469. isMulti = (rmInfo.mRegB != X64Reg_None) || (rmInfo.mDisp != 0);
  5470. return rmInfo.mMode;
  5471. }
  5472. void BeMCContext::GetValAddr(const BeMCOperand& operand, X64CPURegister& reg, int& offset)
  5473. {
  5474. if (operand.IsNativeReg())
  5475. {
  5476. reg = operand.mReg;
  5477. return;
  5478. }
  5479. BF_ASSERT(operand.IsVRegAny());
  5480. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5481. if (operand.mKind == BeMCOperandKind_VReg)
  5482. {
  5483. if (vregInfo->mRelTo)
  5484. {
  5485. GetValAddr(vregInfo->mRelTo, reg, offset);
  5486. if (vregInfo->mRelOffset)
  5487. {
  5488. BF_ASSERT(vregInfo->mRelOffset.IsImmediateInt());
  5489. offset += vregInfo->mRelOffset.mImmediate;
  5490. }
  5491. return;
  5492. }
  5493. BF_ASSERT(vregInfo->mType->IsPointer());
  5494. BF_ASSERT(vregInfo->mReg != X64Reg_None);
  5495. reg = vregInfo->mReg;
  5496. return;
  5497. }
  5498. if ((vregInfo->mIsRetVal) && (mCompositeRetVRegIdx != -1) && (mCompositeRetVRegIdx != operand.mVRegIdx))
  5499. {
  5500. BF_ASSERT(mCompositeRetVRegIdx != -1);
  5501. GetValAddr(BeMCOperand::FromVRegAddr(mCompositeRetVRegIdx), reg, offset);
  5502. return;
  5503. }
  5504. while (vregInfo->IsDirectRelTo())
  5505. {
  5506. vregInfo = GetVRegInfo(vregInfo->mRelTo);
  5507. }
  5508. if ((mCompositeRetVRegIdx == operand.mVRegIdx) && (vregInfo->mReg != X64Reg_None))
  5509. {
  5510. reg = vregInfo->mReg;
  5511. return;
  5512. }
  5513. reg = mUseBP ? X64Reg_RBP : X64Reg_RSP;
  5514. offset = mStackSize + vregInfo->mFrameOffset;
  5515. }
  5516. int BeMCContext::GetHighestVRegRef(const BeMCOperand& operand)
  5517. {
  5518. if (!operand.IsVRegAny())
  5519. return -1;
  5520. int highestIdx = operand.mVRegIdx;
  5521. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5522. if (vregInfo->mRelTo)
  5523. highestIdx = std::max(highestIdx, GetHighestVRegRef(vregInfo->mRelTo));
  5524. if (vregInfo->mRelOffset)
  5525. highestIdx = std::max(highestIdx, GetHighestVRegRef(vregInfo->mRelOffset));
  5526. return highestIdx;
  5527. }
  5528. uint8 BeMCContext::GetJumpOpCode(BeCmpKind cmpKind, bool isLong)
  5529. {
  5530. if (isLong)
  5531. {
  5532. switch (cmpKind)
  5533. {
  5534. case BeCmpKind_None: // JMP
  5535. return 0xE9;
  5536. case BeCmpKind_SLT: // JL
  5537. return 0x8C;
  5538. case BeCmpKind_ULT: // JB
  5539. case BeCmpKind_OLT: // JB
  5540. return 0x82;
  5541. case BeCmpKind_SLE: // JLE
  5542. return 0x8E;
  5543. case BeCmpKind_ULE: // JBE
  5544. case BeCmpKind_OLE: // JBE
  5545. return 0x86;
  5546. case BeCmpKind_EQ: // JE
  5547. return 0x84;
  5548. case BeCmpKind_NE: // JNE
  5549. return 0x85;
  5550. case BeCmpKind_OEQ: // JNP
  5551. return 0x8B;
  5552. case BeCmpKind_UNE: // JP
  5553. return 0x8A;
  5554. case BeCmpKind_SGT: // JG
  5555. return 0x8F;
  5556. case BeCmpKind_UGT: // JA
  5557. case BeCmpKind_OGT: // JA
  5558. return 0x87;
  5559. case BeCmpKind_SGE: // JGE
  5560. return 0x8D;
  5561. case BeCmpKind_UGE: // JAE
  5562. case BeCmpKind_OGE: // JAE
  5563. return 0x83;
  5564. case BeCmpKind_NB: // JNB
  5565. return 0x83;
  5566. case BeCmpKind_NO: // JNO
  5567. return 0x81;
  5568. case BeCmpKind_Sign: // JS
  5569. return 0x88;
  5570. }
  5571. }
  5572. else
  5573. {
  5574. switch (cmpKind)
  5575. {
  5576. case BeCmpKind_None: // JMP
  5577. return 0xEB;
  5578. case BeCmpKind_SLT: // JL
  5579. return 0x7C;
  5580. case BeCmpKind_ULT: // JB
  5581. case BeCmpKind_OLT: // JB
  5582. return 0x72;
  5583. case BeCmpKind_SLE: // JLE
  5584. return 0x7E;
  5585. case BeCmpKind_ULE: // JBE
  5586. case BeCmpKind_OLE: // JBE
  5587. return 0x76;
  5588. case BeCmpKind_EQ: // JE
  5589. return 0x74;
  5590. case BeCmpKind_NE: // JNE
  5591. return 0x75;
  5592. case BeCmpKind_OEQ: // JNP
  5593. return 0x7B;
  5594. case BeCmpKind_UNE: // JP
  5595. return 0x7A;
  5596. case BeCmpKind_SGT: // JG
  5597. return 0x7F;
  5598. case BeCmpKind_UGT: // JA
  5599. case BeCmpKind_OGT: // JA
  5600. return 0x77;
  5601. case BeCmpKind_SGE: // JGE
  5602. return 0x7D;
  5603. case BeCmpKind_UGE: // JAE
  5604. case BeCmpKind_OGE: // JAE
  5605. return 0x73;
  5606. case BeCmpKind_NB: // JNB
  5607. return 0x73;
  5608. case BeCmpKind_NO: // JNO
  5609. return 0x71;
  5610. case BeCmpKind_Sign: // JS
  5611. return 0x78;
  5612. }
  5613. }
  5614. return 0;
  5615. }
  5616. void BeMCContext::Emit(uint8 val)
  5617. {
  5618. mOut.Write(val);
  5619. }
  5620. void BeMCContext::EmitModRM(int mod, int reg, int rm)
  5621. {
  5622. mOut.Write((uint8)((mod << 6) | (reg << 3) | rm));
  5623. }
  5624. void BeMCContext::EmitModRMRel(int rx, X64CPURegister regA, X64CPURegister regB, int bScale, int relOffset)
  5625. {
  5626. if (regB != X64Reg_None)
  5627. {
  5628. // We can't encode regA as RBP in the SIB base
  5629. BF_ASSERT(regA != X64Reg_RBP);
  5630. // We can't encode RegB as RSP in the SIB index
  5631. BF_ASSERT(regB != X64Reg_RSP);
  5632. }
  5633. uint8 modRM = (rx << 3);
  5634. if ((regB == X64Reg_None) && (regA != X64Reg_RSP) && (regA != X64Reg_R12)) // RSP/R12 can't be encoded without a SIB
  5635. {
  5636. if ((relOffset == 0) && (regA != X64Reg_RBP) && (regA != X64Reg_R13)) // RBP/R13 can't be encoded with Mod0
  5637. {
  5638. modRM |= (0x0 << 6) | (EncodeRegNum(regA)); // [regA]
  5639. mOut.Write(modRM);
  5640. return;
  5641. }
  5642. else if ((relOffset >= -0x80) && (relOffset <= 0x7F))
  5643. {
  5644. modRM |= (0x1 << 6) | (EncodeRegNum(regA)); // [regA]+disp8
  5645. mOut.Write(modRM);
  5646. mOut.Write((uint8)relOffset);
  5647. return;
  5648. }
  5649. else
  5650. {
  5651. modRM |= (0x2 << 6) | (EncodeRegNum(regA)); // [regA]+disp32
  5652. mOut.Write(modRM);
  5653. mOut.Write((int32)relOffset);
  5654. return;
  5655. }
  5656. }
  5657. else if (regA == X64Reg_None) // The only option is disp32
  5658. {
  5659. modRM |= (0x0 << 6) | (0x4); // [--][--]+disp32
  5660. mOut.Write(modRM);
  5661. uint8 sib = ((bScale == 2) ? 1 : (bScale == 4) ? 2 : (bScale == 8) ? 3 : 0) << 6;
  5662. sib |= (EncodeRegNum(regB) << 3) | 5;
  5663. mOut.Write(sib);
  5664. mOut.Write((int32)relOffset);
  5665. return;
  5666. }
  5667. else
  5668. {
  5669. // Do no-offset version UNLESS we have a base of R13, which has its representation stolen by '[--][--]+disp32',
  5670. // so we must use the longer +disp8 version in that case
  5671. if ((relOffset == 0) && (regA != X64Reg_R13))
  5672. {
  5673. modRM |= (0x0 << 6) | (0x4); // [--][--]
  5674. mOut.Write(modRM);
  5675. uint8 sib = ((bScale == 2) ? 1 : (bScale == 4) ? 2 : (bScale == 8) ? 3 : 0) << 6;
  5676. sib |= (EncodeRegNum(regB) << 3) | EncodeRegNum(regA);
  5677. mOut.Write(sib);
  5678. return;
  5679. }
  5680. else if ((relOffset >= -0x80) && (relOffset <= 0x7F))
  5681. {
  5682. modRM |= (0x1 << 6) | (0x4); // [--][--]+disp8
  5683. mOut.Write(modRM);
  5684. uint8 sib = ((bScale == 2) ? 1 : (bScale == 4) ? 2 : (bScale == 8) ? 3 : 0) << 6;
  5685. sib |= (EncodeRegNum(regB) << 3) | EncodeRegNum(regA);
  5686. mOut.Write(sib);
  5687. mOut.Write((uint8)relOffset);
  5688. return;
  5689. }
  5690. else
  5691. {
  5692. modRM |= (0x2 << 6) | (0x4); // [--][--]+disp32
  5693. mOut.Write(modRM);
  5694. uint8 sib = ((bScale == 2) ? 1 : (bScale == 4) ? 2 : (bScale == 8) ? 3 : 0) << 6;
  5695. sib |= (EncodeRegNum(regB) << 3) | EncodeRegNum(regA);
  5696. mOut.Write(sib);
  5697. mOut.Write((int32)relOffset);
  5698. return;
  5699. }
  5700. }
  5701. }
  5702. void BeMCContext::EmitModRMRelStack(int rx, int regOffset, int scale)
  5703. {
  5704. EmitModRMRel(rx, mUseBP ? X64Reg_RBP : X64Reg_RSP, X64Reg_None, scale, regOffset);
  5705. }
  5706. void BeMCContext::EmitModRM(int rx, BeMCOperand rm, int relocOfs)
  5707. {
  5708. uint8 modRM = (rx << 3);
  5709. //int relocIdx = -1;
  5710. if (rm.IsImmediateFloat())
  5711. {
  5712. EmitModRM_XMM_IMM(rx, rm);
  5713. return;
  5714. }
  5715. else if ((rm.mKind == BeMCOperandKind_Symbol) || (rm.mKind == BeMCOperandKind_SymbolAddr))
  5716. {
  5717. BeMCRelocation reloc;
  5718. auto sym = mCOFFObject->mSymbols[rm.mSymbolIdx];
  5719. if (sym->mIsTLS)
  5720. {
  5721. auto vregInfo = mVRegInfo[mTLSVRegIdx];
  5722. modRM |= (2 << 6) | EncodeRegNum(vregInfo->mReg);
  5723. reloc.mKind = BeMCRelocationKind_SECREL;
  5724. relocOfs = 0;
  5725. }
  5726. else
  5727. {
  5728. modRM |= 0x5; // RIP + <X>
  5729. reloc.mKind = BeMCRelocationKind_REL32;
  5730. }
  5731. Emit(modRM);
  5732. reloc.mOffset = mOut.GetPos();
  5733. reloc.mSymTableIdx = rm.mSymbolIdx;
  5734. mCOFFObject->mTextSect.mRelocs.push_back(reloc);
  5735. mTextRelocs.push_back((int)mCOFFObject->mTextSect.mRelocs.size() - 1);
  5736. //relocIdx = (int)mOut.GetSize();
  5737. mOut.Write((int32)relocOfs);
  5738. return;
  5739. }
  5740. else if (rm.mKind == BeMCOperandKind_NativeReg)
  5741. {
  5742. modRM |= (0x3 << 6) | (EncodeRegNum(rm.mReg));
  5743. }
  5744. else if (rm.IsVRegAny())
  5745. {
  5746. auto vregInfo = mVRegInfo[rm.mVRegIdx];
  5747. if ((vregInfo->mRelTo.mKind == BeMCOperandKind_SymbolAddr) &&
  5748. (vregInfo->mRelOffset.IsImmediateInt()) &&
  5749. (vregInfo->mRelOffsetScale == 1))
  5750. {
  5751. return EmitModRM(rx, vregInfo->mRelTo, relocOfs + vregInfo->mRelOffset.mImmediate);
  5752. }
  5753. if ((rm.IsVReg()) && (vregInfo->IsDirectRelToAny()))
  5754. return EmitModRM(rx, vregInfo->mRelTo, relocOfs);
  5755. BeRMParamsInfo rmInfo;
  5756. GetRMParams(rm, rmInfo);
  5757. //BF_ASSERT(resultType != BeMCRMMode_Invalid);
  5758. BF_ASSERT(rmInfo.mMode == BeMCRMMode_Deref);
  5759. EmitModRMRel(rx, rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp);
  5760. return;
  5761. }
  5762. else
  5763. {
  5764. SoftFail("Invalid rm");
  5765. }
  5766. mOut.Write(modRM);
  5767. }
  5768. void BeMCContext::EmitModRM(BeMCOperand r, BeMCOperand rm, int relocOfs)
  5769. {
  5770. uint8 modRM = 0;
  5771. BF_ASSERT(r.mKind == BeMCOperandKind_NativeReg);
  5772. EmitModRM(EncodeRegNum(r.mReg), rm, relocOfs);
  5773. }
  5774. void BeMCContext::EmitModRM_Addr(BeMCOperand r, BeMCOperand rm)
  5775. {
  5776. uint8 modRM = 0;
  5777. BF_ASSERT(r.mKind == BeMCOperandKind_NativeReg);
  5778. modRM = EncodeRegNum(r.mReg) << 3;
  5779. if (rm.mKind == BeMCOperandKind_NativeReg)
  5780. {
  5781. modRM |= (0x0 << 6) | (EncodeRegNum(rm.mReg));
  5782. }
  5783. else if ((rm.mKind == BeMCOperandKind_Symbol) || (rm.mKind == BeMCOperandKind_SymbolAddr))
  5784. {
  5785. EmitModRM(r, rm);
  5786. return;
  5787. }
  5788. else if (rm.mKind == BeMCOperandKind_VReg)
  5789. {
  5790. auto vregInfo = mVRegInfo[rm.mVRegIdx];
  5791. if (vregInfo->mIsExpr)
  5792. {
  5793. if (vregInfo->mRelOffsetScale == 1)
  5794. {
  5795. X64CPURegister relToReg = X64Reg_None;
  5796. int regOffset = 0;
  5797. if (vregInfo->mRelOffset)
  5798. {
  5799. BF_ASSERT(vregInfo->mRelOffset.IsImmediate());
  5800. regOffset = (int)vregInfo->mRelOffset.mImmediate;
  5801. }
  5802. if (vregInfo->mRelTo.IsNativeReg())
  5803. {
  5804. relToReg = vregInfo->mRelTo.mReg;
  5805. }
  5806. else if (vregInfo->mRelTo.IsVRegAny())
  5807. {
  5808. auto relVRegInfo = GetVRegInfo(vregInfo->mRelTo);
  5809. if (relVRegInfo->mRelTo)
  5810. {
  5811. BeRMParamsInfo rmInfo;
  5812. GetRMParams(rm, rmInfo);
  5813. BF_ASSERT(rmInfo.mMode != BeMCRMMode_Invalid);
  5814. EmitModRMRel(EncodeRegNum(r.mReg), rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp);
  5815. return;
  5816. }
  5817. else
  5818. NotImpl();
  5819. }
  5820. else
  5821. NotImpl();
  5822. EmitModRMRel(EncodeRegNum(r.mReg), relToReg, X64Reg_None, 1, regOffset);
  5823. return;
  5824. }
  5825. else
  5826. {
  5827. if (vregInfo->mRelOffset)
  5828. {
  5829. BF_ASSERT(vregInfo->mRelOffset.IsImmediate());
  5830. int regOffset = vregInfo->mRelOffset.mImmediate;
  5831. int scaleVal = 0;
  5832. if (vregInfo->mRelOffsetScale == 2)
  5833. scaleVal = 0x1;
  5834. else if (vregInfo->mRelOffsetScale == 4)
  5835. scaleVal = 0x2;
  5836. else if (vregInfo->mRelOffsetScale == 8)
  5837. scaleVal = 0x3;
  5838. modRM |= (0x0 << 6) | (0x4); // [--][--]
  5839. mOut.Write(modRM);
  5840. uint8 sib = (scaleVal << 6) | (EncodeRegNum(vregInfo->mRelTo.mReg) << 3) | (0x5); // [<reg>*<scale> + imm32]
  5841. mOut.Write(sib);
  5842. mOut.Write((int32)regOffset);
  5843. return;
  5844. }
  5845. }
  5846. }
  5847. else
  5848. {
  5849. SoftFail("Illegal expression in EmitModRM_Addr");
  5850. }
  5851. }
  5852. else
  5853. {
  5854. SoftFail("Invalid rm in EmitModRM_Addr");
  5855. }
  5856. mOut.Write(modRM);
  5857. }
  5858. void BeMCContext::EmitModRM_XMM_IMM(int rx, BeMCOperand& imm)
  5859. {
  5860. Emit((rx << 3) | (0x5)); // RIP + <X>
  5861. BeMCSymbol* sym = NULL;
  5862. if (imm.mKind == BeMCOperandKind_Immediate_f32)
  5863. {
  5864. String name = StrFormat("__real@%08x", *(int*)&imm.mImmF32);
  5865. sym = mCOFFObject->GetCOMDAT(name, &imm.mImmF32, 4, 4);
  5866. }
  5867. else if (imm.mKind == BeMCOperandKind_Immediate_f64)
  5868. {
  5869. String name = StrFormat("__real@%016llx", *(int64*)&imm.mImmF64);
  5870. sym = mCOFFObject->GetCOMDAT(name, &imm.mImmF64, 8, 8);
  5871. }
  5872. else if (imm.mKind == BeMCOperandKind_Immediate_f32_Packed128)
  5873. {
  5874. String name = StrFormat("__real@%08x_packed", *(int*)&imm.mImmF32);
  5875. float data[4] = { imm.mImmF32, 0, 0, 0 };
  5876. sym = mCOFFObject->GetCOMDAT(name, data, 16, 16);
  5877. }
  5878. else if (imm.mKind == BeMCOperandKind_Immediate_f64_Packed128)
  5879. {
  5880. String name = StrFormat("__real@%016llx_packed", *(int64*)&imm.mImmF64);
  5881. double data[2] = { imm.mImmF64, 0 };
  5882. sym = mCOFFObject->GetCOMDAT(name, data, 16, 16);
  5883. }
  5884. else if (imm.mKind == BeMCOperandKind_Immediate_int32x4)
  5885. {
  5886. String name = StrFormat("__real@%016llx_x4_packed", *(int*)&imm.mImmediate);
  5887. int32 data[4] = { (int32)imm.mImmediate, (int32)imm.mImmediate, (int32)imm.mImmediate, (int32)imm.mImmediate };
  5888. sym = mCOFFObject->GetCOMDAT(name, data, 16, 16);
  5889. }
  5890. else
  5891. SoftFail("Unhandled value type in EmitModRM_XMM_IMM");
  5892. BeMCRelocation reloc;
  5893. reloc.mKind = BeMCRelocationKind_REL32;
  5894. reloc.mOffset = mOut.GetPos();
  5895. reloc.mSymTableIdx = sym->mIdx;
  5896. mCOFFObject->mTextSect.mRelocs.push_back(reloc);
  5897. mTextRelocs.push_back((int)mCOFFObject->mTextSect.mRelocs.size() - 1);
  5898. mOut.Write((int32)0);
  5899. }
  5900. void BeMCContext::VRegSetInitialized(BeMCBlock* mcBlock, BeMCInst* inst, const BeMCOperand& operand, SizedArrayImpl<int>& addVec, SizedArrayImpl<int>& removeVec, bool deepSet, bool doSet)
  5901. {
  5902. if (operand.IsSymbol())
  5903. {
  5904. auto sym = mCOFFObject->mSymbols[operand.mSymbolIdx];
  5905. if (sym->mIsTLS)
  5906. {
  5907. VRegSetInitialized(mcBlock, inst, BeMCOperand::FromVReg(mTLSVRegIdx), addVec, removeVec, deepSet, doSet);
  5908. return;
  5909. }
  5910. }
  5911. if (operand.mKind == BeMCOperandKind_CmpResult)
  5912. {
  5913. auto& cmpResult = mCmpResults[operand.mCmpResultIdx];
  5914. if (cmpResult.mResultVRegIdx != -1)
  5915. VRegSetInitialized(mcBlock, inst, BeMCOperand::FromVReg(cmpResult.mResultVRegIdx), addVec, removeVec, deepSet, doSet);
  5916. return;
  5917. }
  5918. if (operand.mKind == BeMCOperandKind_VRegPair)
  5919. {
  5920. VRegSetInitialized(mcBlock, inst, BeMCOperand::FromVReg(operand.mVRegPair.mVRegIdx0), addVec, removeVec, deepSet, doSet);
  5921. VRegSetInitialized(mcBlock, inst, BeMCOperand::FromVReg(operand.mVRegPair.mVRegIdx1), addVec, removeVec, deepSet, doSet);
  5922. }
  5923. if (!operand.IsVRegAny())
  5924. return;
  5925. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  5926. if (vregInfo->mDefOnFirstUse)
  5927. {
  5928. int insertIdx = FindSafeInstInsertPos(*mInsertInstIdxRef);
  5929. AllocInst(BeMCInstKind_Def, operand, insertIdx);
  5930. vregInfo->mDefOnFirstUse = false;
  5931. if (insertIdx <= *mInsertInstIdxRef)
  5932. (*mInsertInstIdxRef)++;
  5933. }
  5934. if (vregInfo->mRelTo)
  5935. VRegSetInitialized(mcBlock, inst, vregInfo->mRelTo, addVec, removeVec, deepSet, doSet && deepSet);
  5936. if (vregInfo->mRelOffset)
  5937. VRegSetInitialized(mcBlock, inst, vregInfo->mRelOffset, addVec, removeVec, deepSet, doSet && deepSet);
  5938. if (doSet)
  5939. {
  5940. if (!removeVec.Contains(operand.mVRegIdx))
  5941. addVec.push_back(operand.mVRegIdx);
  5942. removeVec.push_back(mVRegInitializedContext.GetIdx(operand.mVRegIdx, BeTrackKind_Uninitialized));
  5943. }
  5944. }
  5945. BeMCInst* BeMCContext::FindSafePreBranchInst(BeMCBlock* mcBlock)
  5946. {
  5947. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  5948. {
  5949. auto inst = mcBlock->mInstructions[instIdx];
  5950. // We set the definition to just right before the first branch
  5951. if ((inst->mKind == BeMCInstKind_Br) || (inst->mKind == BeMCInstKind_CondBr))
  5952. {
  5953. // Don't separate out Def/Cmp/CmpToBool/Test calls
  5954. while (instIdx > 0)
  5955. {
  5956. auto checkInst = mcBlock->mInstructions[instIdx - 1];
  5957. if ((checkInst->mKind != BeMCInstKind_Def) &&
  5958. (checkInst->mKind != BeMCInstKind_DefLoad) &&
  5959. (checkInst->mKind != BeMCInstKind_Test) &&
  5960. (checkInst->mKind != BeMCInstKind_Cmp) &&
  5961. (checkInst->mKind != BeMCInstKind_CmpToBool))
  5962. return inst;
  5963. inst = checkInst;
  5964. instIdx--;
  5965. }
  5966. break;
  5967. }
  5968. }
  5969. return mcBlock->mInstructions[0];
  5970. }
  5971. void BeMCContext::InitializedPassHelper(BeMCBlock* mcBlock, BeVTrackingGenContext* genCtx, bool& modifiedBlockBefore, bool& modifiedBlockAfter)
  5972. {
  5973. modifiedBlockBefore = false;
  5974. modifiedBlockAfter = false;
  5975. genCtx->mBlocks[mcBlock->mBlockIdx].mGenerateCount++;
  5976. genCtx->mCalls++;
  5977. genCtx->mHandledCalls++;
  5978. BeMDNode* curDbgScope = NULL;
  5979. BeVTrackingList* vregsInitialized = mcBlock->mPredVRegsInitialized;
  5980. //OutputDebugStrF("InitializedPassHelper %@\n", vregsInitialized.mList);
  5981. mActiveBlock = mcBlock;
  5982. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  5983. {
  5984. genCtx->mInstructions++;
  5985. mInsertInstIdxRef = &instIdx;
  5986. auto inst = mcBlock->mInstructions[instIdx];
  5987. inst->mVRegsInitialized = vregsInitialized;
  5988. bool deepSet = false;
  5989. if (inst->mKind == BeMCInstKind_LifetimeStart)
  5990. continue;
  5991. SizedArray<int, 16> removeVec;
  5992. SizedArray<int, 16> addVec;
  5993. SizedArray<int, 16> filteredRemoveVec;
  5994. SizedArray<int, 16> filteredAddVec;
  5995. //
  5996. {
  5997. auto checkLastUseRecord = inst->mVRegLastUseRecord;
  5998. while (checkLastUseRecord != NULL)
  5999. {
  6000. removeVec.Add(checkLastUseRecord->mVRegIdx);
  6001. checkLastUseRecord = checkLastUseRecord->mNext;
  6002. }
  6003. }
  6004. if ((inst->mKind == BeMCInstKind_ValueScopeSoftEnd) || (inst->mKind == BeMCInstKind_ValueScopeHardEnd))
  6005. {
  6006. bool isSoft = inst->mKind == BeMCInstKind_ValueScopeSoftEnd;
  6007. int startVRegIdx = (int)inst->mArg0.mImmediate;
  6008. int endVRegIdx = (int)inst->mArg1.mImmediate;
  6009. int listIdx = mVRegInitializedContext.FindIndex(vregsInitialized, startVRegIdx);
  6010. if (listIdx < 0)
  6011. listIdx = ~listIdx;
  6012. for (; listIdx < vregsInitialized->mSize; listIdx++)
  6013. {
  6014. int vregIdx = vregsInitialized->mEntries[listIdx];
  6015. if (vregIdx >= endVRegIdx)
  6016. break;
  6017. auto vregInfo = mVRegInfo[vregIdx];
  6018. if (isSoft)
  6019. {
  6020. if (vregInfo->mValueScopeRetainedKind >= BeMCValueScopeRetainKind_Soft)
  6021. continue;
  6022. }
  6023. else
  6024. {
  6025. if (vregInfo->mValueScopeRetainedKind >= BeMCValueScopeRetainKind_Hard)
  6026. continue;
  6027. }
  6028. filteredRemoveVec.push_back(vregIdx);
  6029. }
  6030. }
  6031. if ((inst->mResult) && (inst->mResult.mKind == BeMCOperandKind_CmpResult))
  6032. {
  6033. auto& cmpResult = mCmpResults[inst->mResult.mCmpResultIdx];
  6034. if (cmpResult.mResultVRegIdx != -1)
  6035. {
  6036. auto cmpToBoolInst = AllocInst(BeMCInstKind_CmpToBool, BeMCOperand::FromCmpKind(cmpResult.mCmpKind), BeMCOperand(), instIdx + 1);
  6037. cmpToBoolInst->mResult = BeMCOperand::FromVReg(cmpResult.mResultVRegIdx);
  6038. if (cmpResult.mInverted)
  6039. AllocInst(BeMCInstKind_Xor, cmpToBoolInst->mResult, BeMCOperand::FromImmediate(1), instIdx + 2);
  6040. }
  6041. inst->mResult = BeMCOperand();
  6042. }
  6043. if (inst->mKind == BeMCInstKind_DbgDecl)
  6044. {
  6045. if (!mVRegInitializedContext.IsSet(vregsInitialized, inst->mArg0.mVRegIdx))
  6046. {
  6047. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  6048. if (vregInfo->mIsExpr) // For shadowed inlined params, set as initialized
  6049. addVec.push_back(inst->mArg0.mVRegIdx);
  6050. else
  6051. {
  6052. if ((vregInfo->mDbgVariable != NULL) && (vregInfo->mDbgVariable->mInitType == BfIRInitType_NotNeeded_AliveOnDecl))
  6053. addVec.push_back(inst->mArg0.mVRegIdx);
  6054. else
  6055. addVec.push_back(mVRegInitializedContext.GetIdx(inst->mArg0.mVRegIdx, BeTrackKind_Uninitialized));
  6056. }
  6057. }
  6058. vregsInitialized = mVRegInitializedContext.Modify(vregsInitialized, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  6059. continue;
  6060. }
  6061. if (inst->mKind == BeMCInstKind_Def)
  6062. {
  6063. //deepSet = true;
  6064. // If we take a pointer to a local variable before initializing it, we need to consider it as being "ambiguously initialized"
  6065. auto vregInfo = GetVRegInfo(inst->mArg0);
  6066. if (vregInfo->mRelTo.mKind == BeMCOperandKind_VRegAddr)
  6067. {
  6068. auto relVRegInfo = GetVRegInfo(vregInfo->mRelTo);
  6069. if ((relVRegInfo->mDbgVariable != NULL) && (!mVRegInitializedContext.IsSet(vregsInitialized, vregInfo->mRelTo.mVRegIdx)))
  6070. relVRegInfo->mHasAmbiguousInitialization = true;
  6071. }
  6072. }
  6073. if (inst->mKind == BeMCInstKind_LifetimeEnd)
  6074. {
  6075. // In some cases we can have a dbg variable that actually points to a global variable (due to macros/inlining/etc), so this check is for that case:
  6076. if (inst->mArg0.IsVRegAny())
  6077. {
  6078. DedupPushBack(removeVec, inst->mArg0.mVRegIdx);
  6079. DedupPushBack(removeVec, mVRegInitializedContext.GetIdx(inst->mArg0.mVRegIdx, BeTrackKind_Uninitialized));
  6080. }
  6081. vregsInitialized = mVRegInitializedContext.Modify(vregsInitialized, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  6082. continue;
  6083. }
  6084. if (inst->mKind == BeMCInstKind_DefLoad)
  6085. {
  6086. // This is also an address-taking operation, OR it's just a usage of an explicitly-undefined value
  6087. auto loadRegInfo = GetVRegInfo(inst->mArg0);
  6088. if (loadRegInfo->mRelTo.IsVRegAny())
  6089. {
  6090. auto relVRegInfo = GetVRegInfo(loadRegInfo->mRelTo);
  6091. if ((relVRegInfo->mDbgVariable != NULL) && (!mVRegInitializedContext.IsSet(vregsInitialized, loadRegInfo->mRelTo.mVRegIdx)))
  6092. relVRegInfo->mHasAmbiguousInitialization = true;
  6093. }
  6094. }
  6095. if (inst->IsMov())
  6096. {
  6097. // This is also an address-taking operation, OR it's just a usage of an explicitly-undefined value
  6098. if (inst->mArg1.IsVRegAny())
  6099. {
  6100. auto srcRegInfo = GetVRegInfo(inst->mArg1);
  6101. if ((srcRegInfo->mDbgVariable != NULL) && (!mVRegInitializedContext.IsSet(vregsInitialized, inst->mArg1.mVRegIdx)))
  6102. srcRegInfo->mHasAmbiguousInitialization = true;
  6103. }
  6104. }
  6105. int cpyDestVRegIdx = -1;
  6106. if ((inst->mKind == BeMCInstKind_MemCpy) || (inst->mKind == BeMCInstKind_MemSet))
  6107. {
  6108. cpyDestVRegIdx = inst->mArg1.mVRegPair.mVRegIdx0;
  6109. }
  6110. //deepSet = true;
  6111. auto destArg = inst->GetDest();
  6112. auto operands = { &inst->mResult, &inst->mArg0, &inst->mArg1 };
  6113. for (auto op : operands)
  6114. {
  6115. VRegSetInitialized(mcBlock, inst, *op, addVec, removeVec, deepSet || (op == destArg), true);
  6116. }
  6117. for (int removeIdx = 0; removeIdx < (int)filteredRemoveVec.size(); removeIdx++)
  6118. {
  6119. int vregIdx = filteredRemoveVec[removeIdx];
  6120. if (vregIdx >= mVRegInitializedContext.mNumItems)
  6121. continue;
  6122. auto vregInfo = mVRegInfo[vregIdx];
  6123. if (vregInfo->mChainLifetimeEnd)
  6124. {
  6125. auto mcCheck = vregInfo->mRelTo;
  6126. while (mcCheck.IsVRegAny())
  6127. {
  6128. auto checkInfo = mVRegInfo[mcCheck.mVRegIdx];
  6129. removeVec.push_back(mcCheck.mVRegIdx);
  6130. mcCheck = checkInfo->mRelTo;
  6131. }
  6132. }
  6133. }
  6134. vregsInitialized = mVRegInitializedContext.Modify(vregsInitialized, addVec, removeVec, filteredAddVec, filteredRemoveVec);
  6135. }
  6136. mInsertInstIdxRef = NULL;
  6137. mActiveBlock = NULL;
  6138. for (int succIdx = 0; succIdx < (int)mcBlock->mSuccs.size(); succIdx++)
  6139. {
  6140. BeMCBlock* succ = mcBlock->mSuccs[succIdx];
  6141. auto& entry = genCtx->mBlocks[succ->mBlockIdx];
  6142. bool forceRun = (entry.mGenerateCount == 0) && (!entry.mGenerateQueued);
  6143. //if ((!mVRegInitializedContext.HasExtraBitsSet(succ->mPredVRegsInitialized, vregsInitialized)) && (!forceRun))
  6144. //continue;
  6145. auto newPredVRegsInitialized = mVRegInitializedContext.Merge(succ->mPredVRegsInitialized, vregsInitialized);
  6146. if ((newPredVRegsInitialized == succ->mPredVRegsInitialized) && (!forceRun))
  6147. continue;
  6148. int vregIdx = -1;
  6149. /*SizedArray<int, 16> newNodes;
  6150. auto& prevDestEntry = succ->mPredVRegsInitialized;
  6151. auto& mergeFrom = vregsInitialized.mBits;
  6152. // Take nodes that were exclusive to the new set and add edges to nodes that were exclusive the old set
  6153. while (true)
  6154. {
  6155. vregIdx = mVRegInitializedContext.GetNextDiffSetIdx(prevDestEntry.mBits, mergeFrom, vregIdx);
  6156. if (vregIdx == -1)
  6157. break;
  6158. newNodes.push_back(vregIdx);
  6159. }
  6160. BF_ASSERT((!newNodes.empty()) || (forceRun));*/
  6161. /*if ((mDebugging) || (mBeFunction->mName == "?DrawEntry@DrawContext@PerfView@BeefPerf@@QEAAXPEAVGraphics@gfx@Beefy@@PEAVTrackNodeEntry@23@MM@Z"))
  6162. {
  6163. String str;
  6164. str += StrFormat("%s -> %s (Gen=%d) ", ToString(BeMCOperand::FromBlock(mcBlock)).c_str(), ToString(BeMCOperand::FromBlock(succ)).c_str(), genCtx->mBlocks[succ->mBlockIdx].mGenerateCount);
  6165. for (auto newNode : newNodes)
  6166. str += StrFormat(" %d", newNode);
  6167. str += "\r\n";
  6168. OutputDebugStringA(str.c_str());
  6169. }*/
  6170. succ->mPredVRegsInitialized = newPredVRegsInitialized;
  6171. //TODO: This was a bad comparison
  6172. // if (succ->mBlockIdx > succ->mBlockIdx)
  6173. // modifiedBlockAfter = true;
  6174. // else
  6175. // modifiedBlockBefore = true;
  6176. // What does this do?
  6177. if (succ->mBlockIdx > mcBlock->mBlockIdx)
  6178. modifiedBlockAfter = true;
  6179. else
  6180. modifiedBlockBefore = true;
  6181. entry.mGenerateQueued = true;
  6182. }
  6183. }
  6184. void BeMCContext::FixVRegInitFlags(BeMCInst* inst, const BeMCOperand& operand)
  6185. {
  6186. if (operand.IsVReg())
  6187. {
  6188. SizedArray<int, 1> addVec = { operand.mVRegIdx };
  6189. inst->mVRegsInitialized = mVRegInitializedContext.Add(inst->mVRegsInitialized, addVec, true);
  6190. }
  6191. }
  6192. void BeMCContext::SetVTrackingValue(BeMCOperand& operand, BeVTrackingValue& vTrackingValue)
  6193. {
  6194. if (!operand.IsVRegAny())
  6195. return;
  6196. vTrackingValue.mEntry->Set(operand.mVRegIdx);
  6197. auto vregInfo = mVRegInfo[operand.mVRegIdx];
  6198. SetVTrackingValue(vregInfo->mRelTo, vTrackingValue);
  6199. SetVTrackingValue(vregInfo->mRelOffset, vTrackingValue);
  6200. }
  6201. void BeMCContext::DoInitInjectionPass()
  6202. {
  6203. BP_ZONE("BeMCContext::DoInitInjectionPass");
  6204. for (auto mcBlock : mBlocks)
  6205. {
  6206. mActiveBlock = mcBlock;
  6207. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  6208. {
  6209. auto inst = mcBlock->mInstructions[instIdx];
  6210. BfIRInitType pendingInitKind = BfIRInitType_NotNeeded;
  6211. if (inst->mKind == BeMCInstKind_DbgDecl)
  6212. {
  6213. int vregIdx = inst->mArg0.mVRegIdx;
  6214. auto vregInfo = mVRegInfo[vregIdx];
  6215. auto dbgVar = vregInfo->mDbgVariable;
  6216. if (!mVRegInitializedContext.IsSet(inst->mVRegsInitialized, vregIdx))
  6217. {
  6218. if ((dbgVar->mPendingInitType != BfIRInitType_NotNeeded) && (dbgVar->mPendingInitType != BfIRInitType_NotNeeded_AliveOnDecl))
  6219. {
  6220. pendingInitKind = dbgVar->mPendingInitType;
  6221. // We don't need dynLife anymore since we're explicitly writing this
  6222. //vregInfo->mHasDynLife = false;
  6223. //AllocInst(BeMCInstKind_Def, inst->mArg0, instIdx);
  6224. }
  6225. }
  6226. }
  6227. if (pendingInitKind != BfIRInitType_NotNeeded)
  6228. {
  6229. int vregIdx = inst->mArg0.mVRegIdx;
  6230. auto dbgVar = mVRegInfo[vregIdx]->mDbgVariable;
  6231. auto varType = mVRegInfo[vregIdx]->mType;
  6232. auto initType = dbgVar->mPendingInitType;
  6233. if (varType->IsFloat())
  6234. {
  6235. BeMCOperand zeroVal;
  6236. if (varType->mTypeCode == BeTypeCode_Float)
  6237. zeroVal.mKind = BeMCOperandKind_Immediate_f32;
  6238. else
  6239. zeroVal.mKind = BeMCOperandKind_Immediate_f64;
  6240. if (initType == BfIRInitType_Uninitialized)
  6241. {
  6242. if (varType->mTypeCode == BeTypeCode_Float)
  6243. zeroVal.mImmF32 = NAN;
  6244. else
  6245. zeroVal.mImmF64 = NAN;
  6246. }
  6247. else
  6248. {
  6249. if (varType->mTypeCode == BeTypeCode_Float)
  6250. zeroVal.mImmF32 = 0;
  6251. else
  6252. zeroVal.mImmF64 = 0;
  6253. }
  6254. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(vregIdx), zeroVal, instIdx + 1);
  6255. }
  6256. else if (varType->IsNonVectorComposite())
  6257. {
  6258. SetAndRestoreValue<int*> insertPtr(mInsertInstIdxRef, &instIdx);
  6259. instIdx++;
  6260. uint8 val = (initType == BfIRInitType_Uninitialized) ? 0xCC : 0;
  6261. CreateMemSet(BeMCOperand::FromVRegAddr(vregIdx), val, varType->mSize, varType->mAlign);
  6262. instIdx--;
  6263. }
  6264. else
  6265. {
  6266. int64 val = (initType == BfIRInitType_Uninitialized) ? (int64)0xCCCCCCCCCCCCCCCCLL : 0;
  6267. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(vregIdx), BeMCOperand::FromImmediate(val), instIdx + 1);
  6268. }
  6269. dbgVar->mPendingInitType = BfIRInitType_NotNeeded;
  6270. }
  6271. }
  6272. mActiveBlock = NULL;
  6273. }
  6274. SetCurrentInst(NULL);
  6275. }
  6276. void BeMCContext::ReplaceVRegsInit(BeMCBlock* mcBlock, int startInstIdx, BeVTrackingList* prevList, BeVTrackingList* newList)
  6277. {
  6278. for (int instIdx = startInstIdx; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  6279. {
  6280. auto inst = mcBlock->mInstructions[instIdx];
  6281. if (inst->mVRegsInitialized != prevList)
  6282. return;
  6283. inst->mVRegsInitialized = newList;
  6284. }
  6285. }
  6286. // This pass does bottom-up initialization for "simple" vregs (ie: not variables)
  6287. void BeMCContext::SimpleInitializedPass()
  6288. {
  6289. }
  6290. void BeMCContext::GenerateVRegInitFlags(BeVTrackingGenContext& genCtx)
  6291. {
  6292. BP_ZONE("BeMCContext::GenerateVRegInitFlags");
  6293. mVRegInitializedContext.Clear();
  6294. mVRegInitializedContext.Init((int)mVRegInfo.size());
  6295. auto emptyList = mVRegInitializedContext.AllocEmptyList();
  6296. genCtx.mEmptyList = emptyList;
  6297. genCtx.mBlocks.Resize(mBlocks.size());
  6298. for (int blockIdx = 0; blockIdx < (int)mBlocks.size(); blockIdx++)
  6299. {
  6300. auto mcBlock = mBlocks[blockIdx];
  6301. mcBlock->mPredVRegsInitialized = emptyList;
  6302. }
  6303. SimpleInitializedPass();
  6304. bool modifiedBlockBefore;
  6305. bool modifiedBlockAfter;
  6306. InitializedPassHelper(mBlocks[0], &genCtx, modifiedBlockBefore, modifiedBlockAfter);
  6307. while (true)
  6308. {
  6309. bool didWork = false;
  6310. // Handle any outstanding pred entries
  6311. for (int blockIdx = 0; blockIdx < (int)mBlocks.size(); blockIdx++)
  6312. {
  6313. auto& entry = genCtx.mBlocks[blockIdx];
  6314. if (entry.mGenerateQueued)
  6315. {
  6316. entry.mGenerateQueued = false;
  6317. didWork = true;
  6318. auto block = mBlocks[blockIdx];
  6319. bool modifiedBlockBefore;
  6320. bool modifiedBlockAfter;
  6321. InitializedPassHelper(block, &genCtx, modifiedBlockBefore, modifiedBlockAfter);
  6322. if (modifiedBlockBefore)
  6323. break;
  6324. }
  6325. }
  6326. if (!didWork)
  6327. break;
  6328. }
  6329. // Fix up the vregsInit changes to represent changes from the last instruction in one block to the first instruction on the next block
  6330. for (int blockIdx = 0; blockIdx < (int)mBlocks.size() - 1; blockIdx++)
  6331. {
  6332. auto fromBlock = mBlocks[blockIdx];
  6333. auto toBlock = mBlocks[blockIdx + 1];
  6334. if ((fromBlock->mInstructions.empty()) || (toBlock->mInstructions.empty()))
  6335. continue;
  6336. BeMCInst* fromInst = NULL;
  6337. // Last inst may not have vregsInit set
  6338. for (int instIdx = (int)fromBlock->mInstructions.size() - 1; instIdx >= 0; instIdx--)
  6339. {
  6340. fromInst = fromBlock->mInstructions[instIdx];
  6341. if (fromInst->mVRegsInitialized != NULL)
  6342. break;
  6343. }
  6344. auto toInst = toBlock->mInstructions.front();
  6345. if ((fromInst == NULL) || (fromInst->mVRegsInitialized == NULL) ||
  6346. (toInst == NULL) || (toInst->mVRegsInitialized == NULL))
  6347. continue;
  6348. auto prevVRegsInit = toInst->mVRegsInitialized;
  6349. auto newVRegsInit = mVRegInitializedContext.SetChanges(toInst->mVRegsInitialized, fromInst->mVRegsInitialized);;
  6350. for (int instIdx = 0; instIdx < (int)toBlock->mInstructions.size(); instIdx++)
  6351. {
  6352. auto inst = toBlock->mInstructions[instIdx];
  6353. if (inst->mVRegsInitialized != prevVRegsInit)
  6354. break;
  6355. inst->mVRegsInitialized = newVRegsInit;
  6356. }
  6357. }
  6358. int instCount = 0;
  6359. for (auto block : mBlocks)
  6360. instCount += (int)block->mInstructions.size();
  6361. BpEvent("GenerateVRegInitFlags Results",
  6362. StrFormat("Blocks: %d\nInstructions: %d\nVRegs: %d\nCalls: %d\nHandled Calls: %d\nProcessed Instructions: %d\nVRegInit Bytes: %d\nTemp Bytes: %d",
  6363. (int)mBlocks.size(), instCount, (int)mVRegInfo.size(), genCtx.mCalls, genCtx.mHandledCalls, genCtx.mInstructions, mVRegInitializedContext.mAlloc.GetAllocSize(), genCtx.mAlloc.GetAllocSize()).c_str());
  6364. }
  6365. bool BeMCContext::DoInitializedPass()
  6366. {
  6367. BP_ZONE("BeMCContext::DoInitializedPass");
  6368. BeVTrackingGenContext genCtx;
  6369. GenerateVRegInitFlags(genCtx);
  6370. // Search blocks for instances of ambiguous initialization
  6371. for (int blockIdx = 0; blockIdx < (int)mBlocks.size(); blockIdx++)
  6372. {
  6373. auto mcBlock = mBlocks[blockIdx];
  6374. // If mPredVRegsInitialized is NULL, this blocks is unreachable. It could still be in some other block's preds so we don't
  6375. // completely erase it, just empty it
  6376. if (genCtx.mBlocks[mcBlock->mBlockIdx].mGenerateCount == 0)
  6377. {
  6378. // Unused block - clear almost all instructions
  6379. int newIdx = 0;
  6380. // for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  6381. // {
  6382. // auto inst = mcBlock->mInstructions[instIdx];
  6383. // if (inst->mKind == BeMCInstKind_ValueScopeHardEnd)
  6384. // {
  6385. // mcBlock->mInstructions[newIdx++] = inst;
  6386. // }
  6387. // }
  6388. mcBlock->mInstructions.mSize = newIdx;
  6389. continue;
  6390. }
  6391. for (int vregIdx : *mcBlock->mPredVRegsInitialized)
  6392. {
  6393. if (vregIdx >= mVRegInitializedContext.mNumItems)
  6394. continue;
  6395. if (mVRegInitializedContext.IsSet(mcBlock->mPredVRegsInitialized, vregIdx, BeTrackKind_Uninitialized))
  6396. {
  6397. // If a register has both the "initialized" and "uninitialized" flags set, that means that there will be debugging range
  6398. // where the variable could show an uninitialized value. We zero out the value at the def position to make
  6399. // variable viewing less confusing. This state does not affect actual program execution because we've already statically
  6400. // checked against usage of unassigned variables -- unless overridden with a "?" uninitialized expression, in which case
  6401. // one could argue that 0xCC would be a more useful value, but we go with the argument for the "less confusing" zero in this case.
  6402. auto vregInfo = mVRegInfo[vregIdx];
  6403. vregInfo->mHasAmbiguousInitialization = true;
  6404. }
  6405. }
  6406. }
  6407. bool needsReRun = false;
  6408. bool hasInits = false;
  6409. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  6410. {
  6411. auto vregInfo = mVRegInfo[vregIdx];
  6412. if (vregInfo->mDbgVariable != NULL)
  6413. {
  6414. auto initType = vregInfo->mDbgVariable->mInitType;
  6415. if (initType == BfIRInitType_NotSet)
  6416. {
  6417. if (vregInfo->mHasAmbiguousInitialization)
  6418. initType = BfIRInitType_Zero;
  6419. else
  6420. initType = BfIRInitType_NotNeeded;
  6421. }
  6422. else if ((initType == BfIRInitType_Uninitialized) && (!vregInfo->mHasAmbiguousInitialization))
  6423. {
  6424. // If we have an explicitly-uninitialized variable but it doesn't have ambiguous initialization
  6425. // then we don't need to do the 0xCC, we can control visibility in the debugger through scope
  6426. initType = BfIRInitType_NotNeeded;
  6427. }
  6428. if (vregInfo->mDbgVariable->mPendingInitType == initType)
  6429. continue;
  6430. if ((initType != BfIRInitType_NotNeeded) ||
  6431. (vregInfo->mDbgVariable->mPendingInitType == BfIRInitType_NotSet))
  6432. {
  6433. vregInfo->mDbgVariable->mPendingInitType = initType;
  6434. }
  6435. if ((initType != BfIRInitType_NotNeeded) && (initType != BfIRInitType_NotNeeded_AliveOnDecl))
  6436. {
  6437. vregInfo->mDbgVariable->mPendingInitDef = true;
  6438. BF_ASSERT(vregInfo->mHasDynLife);
  6439. vregInfo->mDoConservativeLife = true;
  6440. hasInits = true;
  6441. }
  6442. }
  6443. }
  6444. if (hasInits)
  6445. {
  6446. DoInitInjectionPass();
  6447. // We need to re-execute the init flag generation
  6448. BeVTrackingGenContext genCtx;
  6449. GenerateVRegInitFlags(genCtx);
  6450. }
  6451. return !needsReRun;
  6452. }
  6453. void BeMCContext::DoTLSSetup()
  6454. {
  6455. // Perform at the end of the 'entry' block
  6456. if (mTLSVRegIdx == -1)
  6457. return;
  6458. auto mcBlock = mBlocks[0];
  6459. mActiveBlock = mcBlock;
  6460. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  6461. {
  6462. auto inst = mcBlock->mInstructions[instIdx];
  6463. // Skip past param setup
  6464. if (inst->mKind == BeMCInstKind_Def)
  6465. continue;
  6466. if ((inst->mKind == BeMCInstKind_Mov) && (inst->mArg1.IsNativeReg()))
  6467. continue;
  6468. auto int32Type = mModule->mContext->GetPrimitiveType(BeTypeCode_Int32);
  6469. auto intType = mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  6470. auto intPtrType = mModule->mContext->GetPointerTo(intType);
  6471. auto mcTlsIndex = AllocVirtualReg(intType);
  6472. CreateDefineVReg(mcTlsIndex, instIdx++);
  6473. auto sym = mCOFFObject->GetSymbolRef("_tls_index");
  6474. if (sym->mType == NULL)
  6475. sym->mType = int32Type;
  6476. AllocInst(BeMCInstKind_Mov, mcTlsIndex, BeMCOperand::FromSymbol(sym->mIdx), instIdx++);
  6477. AllocInst(BeMCInstKind_Def, BeMCOperand::FromVReg(mTLSVRegIdx), instIdx++);
  6478. AllocInst(BeMCInstKind_TLSSetup, BeMCOperand::FromVReg(mTLSVRegIdx), BeMCOperand::FromImmediate(0), instIdx++);
  6479. auto mcTlsVal = AllocVirtualReg(intPtrType);
  6480. CreateDefineVReg(mcTlsVal, instIdx++);
  6481. auto tlsValInfo = GetVRegInfo(mcTlsVal);
  6482. tlsValInfo->mIsExpr = true;
  6483. tlsValInfo->mRelTo = BeMCOperand::FromVReg(mTLSVRegIdx);
  6484. tlsValInfo->mRelOffset = mcTlsIndex;
  6485. tlsValInfo->mRelOffsetScale = 8;
  6486. auto mcLoadVal = mcTlsVal;
  6487. mcLoadVal.mKind = BeMCOperandKind_VRegLoad;
  6488. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(mTLSVRegIdx), mcLoadVal, instIdx++);
  6489. break;
  6490. }
  6491. mActiveBlock = NULL;
  6492. }
  6493. void BeMCContext::DoChainedBlockMerge()
  6494. {
  6495. for (int blockIdx = 0; blockIdx < mBlocks.size() - 1; blockIdx++)
  6496. {
  6497. auto mcBlock = mBlocks[blockIdx];
  6498. if (mcBlock->mHasFakeBr)
  6499. continue;
  6500. auto nextMcBlock = mBlocks[blockIdx + 1];
  6501. // We only branch into one block, and the the next block only has current block as a predecessor?
  6502. if ((mcBlock->mSuccs.size() == 1) && (nextMcBlock->mPreds.size() == 1) && (nextMcBlock->mPreds[0] == mcBlock))
  6503. {
  6504. auto lastInst = mcBlock->mInstructions.back();
  6505. // Last instruction of current block is an unconditional branch to the next block
  6506. if ((lastInst->mKind == BeMCInstKind_Br) && (lastInst->mArg0.mBlock == nextMcBlock))
  6507. {
  6508. mcBlock->mSuccs.Remove(nextMcBlock);
  6509. for (auto succ : nextMcBlock->mSuccs)
  6510. {
  6511. if (!mcBlock->mSuccs.Contains(succ))
  6512. {
  6513. mcBlock->mSuccs.push_back(succ);
  6514. succ->mPreds.push_back(mcBlock);
  6515. }
  6516. succ->mPreds.Remove(nextMcBlock);
  6517. }
  6518. mcBlock->mMaxDeclBlockId = nextMcBlock->mMaxDeclBlockId;
  6519. mcBlock->mInstructions.pop_back();
  6520. for (auto inst : nextMcBlock->mInstructions)
  6521. mcBlock->mInstructions.push_back(inst);
  6522. mBlocks.RemoveAt(blockIdx + 1);
  6523. blockIdx--;
  6524. }
  6525. }
  6526. }
  6527. for (int blockIdx = 0; blockIdx < mBlocks.size(); blockIdx++)
  6528. mBlocks[blockIdx]->mBlockIdx = blockIdx;
  6529. }
  6530. void BeMCContext::DoSplitLargeBlocks()
  6531. {
  6532. Dictionary<int, int> blockEndRemap;
  6533. int splitSize = 4096;
  6534. int maxBlockSize = splitSize + splitSize / 4;
  6535. bool hadSplits = false;
  6536. for (int blockIdx = 0; blockIdx < mBlocks.size(); blockIdx++)
  6537. {
  6538. int blockBreakIdx = 0;
  6539. auto srcBlock = mBlocks[blockIdx];
  6540. if (srcBlock->mInstructions.size() < maxBlockSize)
  6541. continue;
  6542. hadSplits = true;
  6543. int extensionCount = srcBlock->mInstructions.size() / splitSize;
  6544. // Don't allow the last block to have too few instructions
  6545. if (srcBlock->mInstructions.size() % splitSize < splitSize / 4)
  6546. extensionCount--;
  6547. if (extensionCount == 0) // Shouldn't happen, really
  6548. continue;
  6549. for (int extIdx = 0; extIdx < extensionCount; extIdx++)
  6550. {
  6551. BeMCBlock* extBlock = mMCBlockAlloc.Alloc();
  6552. extBlock->mName = srcBlock->mName + StrFormat("__EXT%d", extIdx);
  6553. mBlocks.Insert(blockIdx + 1 + extIdx, extBlock);
  6554. int startIdx = (extIdx + 1) * splitSize;
  6555. int endIdx = BF_MIN((extIdx + 2) * splitSize, srcBlock->mInstructions.size());
  6556. if (extIdx == extensionCount - 1)
  6557. endIdx = (int)srcBlock->mInstructions.size();
  6558. extBlock->mInstructions.Insert(0, &srcBlock->mInstructions[startIdx], endIdx - startIdx);
  6559. extBlock->mPreds.Add(mBlocks[blockIdx + extIdx]);
  6560. if (extIdx > 0)
  6561. {
  6562. auto prevBlock = mBlocks[blockIdx + extIdx];
  6563. mActiveBlock = prevBlock;
  6564. AllocInst(BeMCInstKind_Br, BeMCOperand::FromBlock(extBlock));
  6565. mActiveBlock = NULL;
  6566. mBlocks[blockIdx + extIdx]->mSuccs.Add(extBlock);
  6567. }
  6568. if (extIdx == extensionCount - 1)
  6569. {
  6570. blockEndRemap[blockIdx] = blockIdx + extIdx + 1;
  6571. for (auto succ : srcBlock->mSuccs)
  6572. {
  6573. succ->mPreds.Remove(srcBlock);
  6574. succ->mPreds.Add(extBlock);
  6575. }
  6576. extBlock->mSuccs = srcBlock->mSuccs;
  6577. srcBlock->mSuccs.Clear();
  6578. srcBlock->mSuccs.Add(mBlocks[blockIdx + 1]);
  6579. }
  6580. }
  6581. mActiveBlock = srcBlock;
  6582. srcBlock->mInstructions.RemoveRange(splitSize, (int)srcBlock->mInstructions.size() - splitSize);
  6583. AllocInst(BeMCInstKind_Br, BeMCOperand::FromBlock(mBlocks[blockIdx + 1]));
  6584. mActiveBlock = NULL;
  6585. }
  6586. if (!hadSplits)
  6587. return;
  6588. // for (int blockIdx = 0; blockIdx < mBlocks.size(); blockIdx++)
  6589. // {
  6590. // auto mcBlock = mBlocks[blockIdx];
  6591. // for (auto inst : mcBlock->mInstructions)
  6592. // {
  6593. // if (inst->mResult.mKind == BeMCOperandKind_Phi)
  6594. // {
  6595. // inst->mResult.mP
  6596. // }
  6597. // }
  6598. // }
  6599. for (int blockIdx = 0; blockIdx < mBlocks.size(); blockIdx++)
  6600. mBlocks[blockIdx]->mBlockIdx = blockIdx;
  6601. for (auto phi : mPhiAlloc)
  6602. {
  6603. for (auto& phiValue : phi->mValues)
  6604. {
  6605. int remappedBlock = -1;
  6606. if (blockEndRemap.TryGetValue(phiValue.mBlockFrom->mBlockIdx, &remappedBlock))
  6607. {
  6608. phiValue.mBlockFrom = mBlocks[remappedBlock];
  6609. }
  6610. }
  6611. }
  6612. }
  6613. void BeMCContext::DetectLoops()
  6614. {
  6615. BP_ZONE("BeMCContext::DetectLoops");
  6616. BeMCLoopDetector loopDetector(this);
  6617. loopDetector.DetectLoops();
  6618. }
  6619. void BeMCContext::DoLastUsePassHelper(BeMCInst* inst, const BeMCOperand& operand)
  6620. {
  6621. if (operand.mKind == BeMCOperandKind_VRegPair)
  6622. {
  6623. auto mcOperand = BeMCOperand::FromEncoded(operand.mVRegPair.mVRegIdx0);
  6624. if (mcOperand.IsVRegAny())
  6625. DoLastUsePassHelper(inst, mcOperand);
  6626. mcOperand = BeMCOperand::FromEncoded(operand.mVRegPair.mVRegIdx1);
  6627. if (mcOperand.IsVRegAny())
  6628. DoLastUsePassHelper(inst, mcOperand);
  6629. return;
  6630. }
  6631. if (!operand.IsVRegAny())
  6632. return;
  6633. int vregIdx = operand.mVRegIdx;
  6634. auto vregInfo = mVRegInfo[vregIdx];
  6635. if ((vregInfo->mDbgVariable == NULL) && (!vregInfo->mHasDynLife) && (!vregInfo->mFoundLastUse))
  6636. {
  6637. vregInfo->mFoundLastUse = true;
  6638. auto lastUseRecord = mAlloc.Alloc<BeVRegLastUseRecord>();
  6639. lastUseRecord->mVRegIdx = vregIdx;
  6640. lastUseRecord->mNext = inst->mVRegLastUseRecord;
  6641. inst->mVRegLastUseRecord = lastUseRecord;
  6642. }
  6643. if (vregInfo->mRelTo)
  6644. DoLastUsePassHelper(inst, vregInfo->mRelTo);
  6645. if (vregInfo->mRelOffset)
  6646. DoLastUsePassHelper(inst, vregInfo->mRelOffset);
  6647. }
  6648. void BeMCContext::DoLastUsePass()
  6649. {
  6650. for (int blockIdx = (int)mBlocks.size() - 1; blockIdx >= 0; blockIdx--)
  6651. {
  6652. auto mcBlock = mBlocks[blockIdx];
  6653. for (int instIdx = (int)mcBlock->mInstructions.size() - 1; instIdx >= 0; instIdx--)
  6654. {
  6655. auto inst = mcBlock->mInstructions[instIdx];
  6656. auto operands = { &inst->mResult, &inst->mArg0, &inst->mArg1 };
  6657. for (auto operand : operands)
  6658. {
  6659. DoLastUsePassHelper(inst, *operand);
  6660. }
  6661. }
  6662. }
  6663. }
  6664. void BeMCContext::RefreshRefCounts()
  6665. {
  6666. for (auto vregInfo : mVRegInfo)
  6667. {
  6668. if (vregInfo->mMustExist)
  6669. vregInfo->mRefCount = 1;
  6670. else
  6671. vregInfo->mRefCount = 0;
  6672. vregInfo->mAssignCount = 0;
  6673. }
  6674. for (auto mcBlock : mBlocks)
  6675. {
  6676. for (auto& inst : mcBlock->mInstructions)
  6677. {
  6678. if (inst->IsDef())
  6679. continue;
  6680. auto operands = { inst->mResult, inst->mArg0, inst->mArg1 };
  6681. for (auto op : operands)
  6682. {
  6683. if (op.IsVRegAny())
  6684. mVRegInfo[op.mVRegIdx]->mRefCount++;
  6685. if (op.mKind == BeMCOperandKind_VRegPair)
  6686. {
  6687. auto vreg0 = BeMCOperand::FromEncoded(op.mVRegPair.mVRegIdx0);
  6688. if (vreg0.IsVRegAny())
  6689. {
  6690. mVRegInfo[vreg0.mVRegIdx]->mRefCount++;
  6691. mVRegInfo[vreg0.mVRegIdx]->mAssignCount++;
  6692. }
  6693. auto vreg1 = BeMCOperand::FromEncoded(op.mVRegPair.mVRegIdx1);
  6694. if (vreg1.IsVRegAny())
  6695. mVRegInfo[vreg1.mVRegIdx]->mRefCount++;
  6696. }
  6697. }
  6698. if (inst->IsAssignment())
  6699. {
  6700. if (inst->mArg0.IsVRegAny())
  6701. mVRegInfo[inst->mArg0.mVRegIdx]->mAssignCount++;
  6702. }
  6703. if (inst->mResult)
  6704. {
  6705. if (inst->mResult.IsVRegAny())
  6706. mVRegInfo[inst->mResult.mVRegIdx]->mAssignCount++;
  6707. }
  6708. }
  6709. }
  6710. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  6711. {
  6712. auto vregInfo = mVRegInfo[vregIdx];
  6713. if (vregInfo->mRefCount > 0)
  6714. AddRelRefs(BeMCOperand::FromVReg(vregIdx), 0);
  6715. }
  6716. }
  6717. bool BeMCContext::CheckVRegEqualityRange(BeMCBlock* mcBlock, int instIdx, const BeMCOperand& vreg0, const BeMCOperand& vreg1, BeMCRemapper& regRemaps, bool onlyCheckFirstLifetime)
  6718. {
  6719. // The algorithm here is that we check to see if either of these registers are ever written to
  6720. // up to the point that one of them dies. If not, we can map them to each other
  6721. bool foundSplitPoint = false;
  6722. bool hadPointerDeref = false;
  6723. for (int checkInstIdx = instIdx + 1; checkInstIdx < (int)mcBlock->mInstructions.size(); checkInstIdx++)
  6724. {
  6725. auto checkInst = mcBlock->mInstructions[checkInstIdx];
  6726. if ((!IsLive(checkInst->mLiveness, vreg0.mVRegIdx, regRemaps)) ||
  6727. ((!onlyCheckFirstLifetime) && (!IsLive(checkInst->mLiveness, vreg1.mVRegIdx, regRemaps))))
  6728. {
  6729. return true;
  6730. }
  6731. if ((checkInst->mKind == BeMCInstKind_LifetimeEnd) &&
  6732. ((checkInst->mArg0.mVRegIdx == vreg0.mVRegIdx) || (checkInst->mArg0.mVRegIdx == vreg1.mVRegIdx)))
  6733. {
  6734. if (checkInstIdx == (int)mcBlock->mInstructions.size() - 1)
  6735. {
  6736. // There are cases where this might be the last instruction in a block, so we won't see the mLiveness change
  6737. // so this catches that case
  6738. return true;
  6739. }
  6740. // We have cases like when have a "defer delete val;", val will get used by the dtor after the LifetimeEnd
  6741. return false;
  6742. }
  6743. if (hadPointerDeref)
  6744. {
  6745. // Pointer deref is okay as long as liveness ends one instruction afterwards
  6746. return false;
  6747. }
  6748. // Anything can happen during a call...
  6749. if (checkInst->mKind == BeMCInstKind_Call)
  6750. return false;
  6751. auto mcDest = checkInst->GetDest();
  6752. if ((mcDest != NULL) && (mcDest->mKind == BeMCOperandKind_VReg))
  6753. {
  6754. // Any assignment to either of these regs disqualifies them
  6755. if ((mcDest->mVRegIdx == vreg0.mVRegIdx) || (mcDest->mVRegIdx == vreg1.mVRegIdx))
  6756. return false;
  6757. // Any writing into memory could be aliased, so we don't replace across that boundary
  6758. if (HasPointerDeref(*mcDest))
  6759. {
  6760. // Delay the 'return false' until we determine if our liveness ends on the next instruction.
  6761. // If it does, then this is okay because it didn't affect the source of any operations
  6762. hadPointerDeref = true;
  6763. }
  6764. }
  6765. }
  6766. return false;
  6767. }
  6768. void BeMCContext::DoInstCombinePass()
  6769. {
  6770. BP_ZONE("BeMCContext::DoInstCombinePass");
  6771. BeMCRemapper regRemaps;
  6772. regRemaps.Init((int)mVRegInfo.size());
  6773. BeVTrackingValue vTrackingVal((int)mVRegInfo.size());
  6774. HashSet<int> wantUnwrapVRegs;
  6775. for (auto mcBlock : mBlocks)
  6776. {
  6777. BeDbgLoc* curDbgLoc = NULL;
  6778. int safeMergeLocStart = 0;
  6779. bool lastDefDidMerge = false;
  6780. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  6781. {
  6782. bool reprocessInst = false;
  6783. auto inst = mcBlock->mInstructions[instIdx];
  6784. BeMCInst* prevInst = NULL;
  6785. if (instIdx > 0)
  6786. prevInst = mcBlock->mInstructions[instIdx - 1];
  6787. if ((inst->mDbgLoc != curDbgLoc) && (inst->mDbgLoc != NULL))
  6788. {
  6789. safeMergeLocStart = instIdx;
  6790. curDbgLoc = inst->mDbgLoc;
  6791. }
  6792. if ((inst->mKind == BeMCInstKind_Call) || (inst->mKind == BeMCInstKind_MemCpy) || (inst->mKind == BeMCInstKind_MemSet))
  6793. {
  6794. // Don't merge anything across callsites or memory writes
  6795. safeMergeLocStart = instIdx;
  6796. }
  6797. auto instDest = inst->GetDest();
  6798. if ((instDest != NULL) && (HasPointerDeref(*instDest)))
  6799. {
  6800. // Memory aliasing may invalidate any accesses, don't merge expressions across here
  6801. safeMergeLocStart = instIdx + 1;
  6802. }
  6803. SetCurrentInst(inst);
  6804. //TODO: Remove
  6805. // if we've remapped other vregs onto this that aren't dead, then we can't end the value lifetime but we
  6806. // must end the lifetime of the visible debug variable
  6807. /*if (inst->mKind == BeMCInstKind_LifetimeEnd)
  6808. {
  6809. bool extendValue = false;
  6810. int vregIdx = inst->mArg0.mVRegIdx;
  6811. int checkIdx = regRemaps.GetHead(vregIdx);
  6812. while (checkIdx != -1)
  6813. {
  6814. if ((checkIdx != vregIdx) && (mLivenessContext.IsSet(inst->mLiveness, checkIdx)))
  6815. extendValue = true;
  6816. checkIdx = regRemaps.GetNext(checkIdx);
  6817. }
  6818. if (extendValue)
  6819. {
  6820. //Throw error
  6821. }
  6822. //inst->mKind = BeMCInstKind_LifetimeDbgEnd;
  6823. }*/
  6824. // Handle movs into .addrs
  6825. if (inst->mKind == BeMCInstKind_Mov)
  6826. {
  6827. auto vregInfoDest = GetVRegInfo(inst->mArg0);
  6828. if (vregInfoDest != NULL)
  6829. {
  6830. auto vregInfoSrc = GetVRegInfo(inst->mArg1);
  6831. if ((vregInfoSrc != NULL) && (vregInfoSrc->mForceMerge))
  6832. {
  6833. vregInfoSrc->mForceMerge = false;
  6834. if ((vregInfoDest->mDbgVariable != NULL) && (vregInfoSrc->CanEliminate()))
  6835. {
  6836. BF_ASSERT(!mVRegInitializedContext.IsSet(inst->mVRegsInitialized, inst->mArg0.mVRegIdx));
  6837. AddRegRemap(inst->mArg1.mVRegIdx, inst->mArg0.mVRegIdx, regRemaps);
  6838. // Remove any LifetimeStarts since our source may have been alive before us
  6839. //if (inst->mArg1.mVRegIdx < inst->mArg0.mVRegIdx)
  6840. //removeLifetimeStarts.insert(inst->mArg0.mVRegIdx);
  6841. }
  6842. }
  6843. }
  6844. }
  6845. // The following code only applies when we aren't on the last instruction
  6846. int nextIdx = instIdx + 1;
  6847. BeMCInst* nextInst = NULL;
  6848. while (true)
  6849. {
  6850. if (nextIdx >= (int)mcBlock->mInstructions.size())
  6851. break;
  6852. nextInst = mcBlock->mInstructions[nextIdx];
  6853. if (!nextInst->IsDef())
  6854. break;
  6855. nextIdx++;
  6856. }
  6857. if (nextInst == NULL)
  6858. continue;
  6859. // The following code only applies when we have at least 2 more instructions
  6860. // Find a non-Def instruction
  6861. int nextNextIdx = nextIdx + 1;
  6862. BeMCInst* nextNextInst = NULL;
  6863. while (true)
  6864. {
  6865. if (nextNextIdx >= (int)mcBlock->mInstructions.size())
  6866. break;
  6867. nextNextInst = mcBlock->mInstructions[nextNextIdx];
  6868. if (!nextNextInst->IsDef())
  6869. break;
  6870. nextNextIdx++;
  6871. }
  6872. auto result = RemapOperand(inst->mResult, regRemaps);
  6873. auto arg0 = RemapOperand(inst->mArg0, regRemaps);
  6874. auto arg1 = RemapOperand(inst->mArg1, regRemaps);
  6875. if (inst->IsDef())
  6876. {
  6877. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  6878. if (vregInfo->mIsExpr)
  6879. {
  6880. for (int checkInstIdx = safeMergeLocStart; checkInstIdx < instIdx; checkInstIdx++)
  6881. {
  6882. auto checkInst = mcBlock->mInstructions[checkInstIdx];
  6883. if (checkInst->IsDef())
  6884. {
  6885. auto checkArg0 = RemapOperand(checkInst->mArg0, regRemaps);
  6886. auto checkVRegInfo = mVRegInfo[checkArg0.mVRegIdx];
  6887. if (checkVRegInfo->mIsExpr)
  6888. {
  6889. // Remap to use the earlier definition of this expression
  6890. if (AreOperandsEquivalent(inst->mArg0, checkArg0, regRemaps))
  6891. {
  6892. vTrackingVal.Clear();
  6893. SetVTrackingValue(checkArg0, vTrackingVal);
  6894. // Check to make sure we didn't do any writes to any of the values used in the expression. This happens
  6895. // during "iVal++", for example.
  6896. bool hadBadWrite = false;
  6897. for (int verifyInstIdx = safeMergeLocStart; verifyInstIdx < instIdx; verifyInstIdx++)
  6898. {
  6899. auto verifyInst = mcBlock->mInstructions[verifyInstIdx];
  6900. auto verifyDest = verifyInst->GetDest();
  6901. if ((verifyDest != NULL) && (verifyDest->IsVRegAny()) && (vTrackingVal.mEntry->IsSet(verifyDest->mVRegIdx)))
  6902. {
  6903. hadBadWrite = true;
  6904. break;
  6905. }
  6906. }
  6907. if (!hadBadWrite)
  6908. {
  6909. auto vregInfoFrom = GetVRegInfo(inst->mArg0);
  6910. if (vregInfoFrom->mDbgVariable == NULL)
  6911. AddRegRemap(inst->mArg0.mVRegIdx, checkArg0.mVRegIdx, regRemaps, true);
  6912. }
  6913. break;
  6914. }
  6915. }
  6916. }
  6917. }
  6918. }
  6919. }
  6920. BeMCOperand origNextDestOperand;
  6921. if (nextInst->IsAssignment())
  6922. origNextDestOperand = nextInst->mArg0;
  6923. else if (nextInst->mResult)
  6924. origNextDestOperand = nextInst->mResult;
  6925. BeMCOperand nextDestOperand = RemapOperand(origNextDestOperand, regRemaps);
  6926. if (inst->mKind == BeMCInstKind_DefLoad)
  6927. {
  6928. auto vregInfoDest = GetVRegInfo(inst->mArg0);
  6929. auto mcLoaded = inst->mArg0;
  6930. auto mcAddr = vregInfoDest->mRelTo;
  6931. // We can only do a regRemap replace if we aren't the product of a pointer dereference
  6932. bool mapToDef = false;
  6933. if (mcAddr.mKind == BeMCOperandKind_VReg)
  6934. {
  6935. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  6936. // We do 'onlyCheckFirstLifetime' because we have cases like when have a "defer delete val;", because val will get used by the dtor after the LiftimeEnd
  6937. if ((vregInfo->IsDirectRelTo()) && (vregInfo->mDbgVariable == NULL) && (CheckVRegEqualityRange(mcBlock, instIdx, mcLoaded, mcAddr, regRemaps, true)))
  6938. {
  6939. // If the source value doesn't change in the lifetime of the loaded value then
  6940. // we can just map directly to the source - no copy needed.
  6941. inst->mKind = BeMCInstKind_Def;
  6942. AddRegRemap(inst->mArg0.mVRegIdx, vregInfoDest->mRelTo.mVRegIdx, regRemaps, true);
  6943. continue;
  6944. }
  6945. }
  6946. else if (mcAddr.mKind == BeMCOperandKind_SymbolAddr)
  6947. {
  6948. // No actual load needed - just keep it as a Direct reference
  6949. inst->mKind = BeMCInstKind_Def;
  6950. //wantUnwrapVRegs.Add(inst->mArg0.mVRegIdx);
  6951. continue;
  6952. }
  6953. else if (mcAddr.mKind == BeMCOperandKind_Symbol)
  6954. {
  6955. //TODO: We used to have this as a 'inst->mKind = BeMCInstKind_Def;' case also, but that messed up post-increment (ie: gVal++) values
  6956. continue;
  6957. }
  6958. bool hadPointerDeref = false;
  6959. for (int checkInstIdx = instIdx + 1; checkInstIdx < (int)mcBlock->mInstructions.size(); checkInstIdx++)
  6960. {
  6961. auto checkInst = mcBlock->mInstructions[checkInstIdx];
  6962. BF_ASSERT(mcLoaded.IsVRegAny());
  6963. if (!IsLive(checkInst->mLiveness, mcLoaded.mVRegIdx, regRemaps))
  6964. {
  6965. break;
  6966. }
  6967. if ((mcAddr.IsVRegAny()) && (!IsLive(checkInst->mLiveness, mcAddr.mVRegIdx, regRemaps)))
  6968. break;
  6969. if (hadPointerDeref)
  6970. {
  6971. // Pointer deref is okay as long as liveness ends one instruction afterwards
  6972. break;
  6973. }
  6974. auto mcDest = checkInst->GetDest();
  6975. if ((mcDest != NULL) &&
  6976. ((mcDest->mKind == BeMCOperandKind_VReg) || (mcDest->mKind == mcAddr.mKind)))
  6977. {
  6978. // Any assignment to either of these regs ends our remapping loop
  6979. if ((mcDest->mVRegIdx == mcLoaded.mVRegIdx) || (mcDest->mVRegIdx == mcAddr.mVRegIdx))
  6980. {
  6981. break;
  6982. }
  6983. // Any writing into memory could be aliased, so we don't replace across that boundary
  6984. if (HasPointerDeref(*mcDest))
  6985. {
  6986. // Delay the 'return false' until we determine if our liveness ends on the next instruction.
  6987. // If it does, then this is okay because it didn't affect the source of any operations
  6988. hadPointerDeref = true;
  6989. }
  6990. }
  6991. if (checkInst->IsInformational())
  6992. continue;
  6993. if (checkInst->mKind == BeMCInstKind_Call) // Anything can happen during a call
  6994. break;
  6995. if (checkInst->mDbgLoc != inst->mDbgLoc)
  6996. break;
  6997. auto operands = { &checkInst->mArg0, &checkInst->mArg1 };
  6998. for (auto& operand : operands)
  6999. {
  7000. // If we reference the loaded version during the range where the value hasn't changed,
  7001. // then just refer back to the original value. This is useful in cases like i++,
  7002. // where we want to be able to still do an INC on the source variable rather than
  7003. // "loadedI = i; i = loadedI + 1; ... use loadedI ..."
  7004. if ((operand->mKind == BeMCOperandKind_VReg) && (operand->mVRegIdx == mcLoaded.mVRegIdx))
  7005. {
  7006. *operand = mcAddr;
  7007. }
  7008. if ((operand->mKind == BeMCOperandKind_VRegLoad) && (operand->mVRegIdx == mcLoaded.mVRegIdx))
  7009. {
  7010. if (mcAddr.mKind == BeMCOperandKind_VReg)
  7011. {
  7012. operand->mVRegIdx = mcAddr.mVRegIdx;
  7013. }
  7014. }
  7015. }
  7016. }
  7017. }
  7018. // Check for the special case of a single move to a SRET dereference - if so, then make the source composite be the source value
  7019. if ((inst->mKind == BeMCInstKind_Mov) && (inst->mArg0.mKind == BeMCOperandKind_VRegLoad) && (inst->mArg1.mKind == BeMCOperandKind_VReg))
  7020. {
  7021. auto vregInfoDest = GetVRegInfo(inst->mArg0);
  7022. auto vregInfoSrc = GetVRegInfo(inst->mArg1);
  7023. if ((vregInfoDest->IsDirectRelTo()) && (vregInfoSrc->IsDirectRelTo()))
  7024. {
  7025. auto destRelTo = GetVRegInfo(vregInfoDest->mRelTo);
  7026. auto srcRelTo = GetVRegInfo(vregInfoSrc->mRelTo);
  7027. if ((destRelTo->mIsRetVal) && (vregInfoDest->mAssignCount == 1) &&
  7028. (!destRelTo->mIsExpr) && (!srcRelTo->mIsExpr))
  7029. {
  7030. mCompositeRetVRegIdx = vregInfoDest->mRelTo.mVRegIdx;
  7031. RemoveInst(mcBlock, instIdx);
  7032. srcRelTo->SetRetVal();
  7033. instIdx--;
  7034. continue;
  7035. }
  7036. }
  7037. }
  7038. // For the form:
  7039. // @Def %vreg0
  7040. // [%result] = <op> %vreg0, %vreg1
  7041. // Remap %vreg0 to %vreg1 if this is the only assignment, and thus an immutable definition, and vreg1 doesn't change
  7042. if ((nextDestOperand) &&
  7043. (((inst->mKind == BeMCInstKind_Def) && (arg0 == nextDestOperand)) ||
  7044. ((inst->mKind == BeMCInstKind_LifetimeStart) && (BeMCOperand::ToLoad(arg0) == nextDestOperand))))
  7045. {
  7046. lastDefDidMerge = false;
  7047. auto vregInfoDest = GetVRegInfo(nextDestOperand);
  7048. auto checkOperands = { &nextInst->mArg0, &nextInst->mArg1 };
  7049. bool checkArg1 = true;
  7050. if (nextInst->mResult)
  7051. {
  7052. if ((nextInst->mKind == BeMCInstKind_Add) || (nextInst->mKind == BeMCInstKind_Mul))
  7053. {
  7054. // These are commutative, so we can remap to either the LHS or RHS
  7055. checkArg1 = true;
  7056. }
  7057. else
  7058. {
  7059. // We can only remap to LHS
  7060. checkArg1 = false;
  7061. }
  7062. }
  7063. for (int argCheckIdx = (nextInst->mResult ? 0 : 1); argCheckIdx < (checkArg1 ? 2 : 1); argCheckIdx++)
  7064. {
  7065. BeMCOperand origCheckOperand;
  7066. if (argCheckIdx == 0)
  7067. origCheckOperand = nextInst->mArg0;
  7068. else
  7069. origCheckOperand = nextInst->mArg1;
  7070. auto checkOperand = RemapOperand(origCheckOperand, regRemaps);
  7071. if (checkOperand.mKind != BeMCOperandKind_VReg) // Loads/Addrs not applicable
  7072. continue;
  7073. auto vregInfoCheck = GetVRegInfo(checkOperand);
  7074. if (vregInfoCheck == NULL)
  7075. continue;
  7076. if ((vregInfoDest->mIsExpr) || (vregInfoCheck->mIsExpr))
  7077. continue;
  7078. if ((vregInfoCheck->mForceMem) && (!vregInfoDest->mForceMem))
  7079. continue;
  7080. if ((!vregInfoDest->mForceMerge) && (!vregInfoDest->mIsRetVal))
  7081. {
  7082. // If we merge to a vreg that has a pointer ref, that could be a deoptimization by not allowing
  7083. // either to bind to a register
  7084. if (vregInfoDest->mForceMem != vregInfoCheck->mForceMem)
  7085. {
  7086. if (!GetType(checkOperand)->IsNonVectorComposite())
  7087. continue;
  7088. }
  7089. }
  7090. // We have a special retVal case where we can allow the __returnVal to share an address with the
  7091. // actual 'ret' param. This is most useful when returning structs by value, to avoid a copy.
  7092. // The only downside is that editing the return value in the debugger will also show the value
  7093. // of the ret param changing, which is normally not acceptable but in this case we allow it.
  7094. bool allowCoexist = ((vregInfoDest->mIsRetVal) ||
  7095. (vregInfoCheck->CanEliminate()) || (vregInfoDest->CanEliminate()));
  7096. // Only vregs of the same types can be merged
  7097. if (!AreTypesEquivalent(vregInfoCheck->mType, vregInfoDest->mType))
  7098. continue;
  7099. bool canMerge = false;
  7100. if ((nextInst->IsMov()) && (allowCoexist))
  7101. {
  7102. if ((vregInfoCheck->mForceMerge) || (vregInfoCheck->mIsRetVal))
  7103. canMerge = true; // This is to allow the incoming vregs to combine with the ".addr" debug variables
  7104. else // Always use orig operands for liveness checking
  7105. canMerge = CheckVRegEqualityRange(mcBlock, nextIdx, origCheckOperand, origNextDestOperand, regRemaps);
  7106. }
  7107. else
  7108. {
  7109. // If we're dealing with a non-Mov instruction, then the values of both of these vregs will
  7110. // be different after the instruction, so we can only map them if the incoming vreg will be dead
  7111. // afterwards. If both are dbgVariables then we can't allow them to coeexist because the user
  7112. // could edit one of the values in the debugger and we don't want the other one changing.
  7113. if ((nextNextInst != NULL) &&
  7114. (!IsLive(nextNextInst->mLiveness, origCheckOperand.mVRegIdx, regRemaps)) &&
  7115. (!IsLive(nextNextInst->mLiveness, checkOperand.mVRegIdx, regRemaps)))
  7116. canMerge = true;
  7117. }
  7118. if (canMerge)
  7119. {
  7120. if (vregInfoCheck->mForceMerge)
  7121. vregInfoCheck->mForceMerge = false; // Only allow one merge
  7122. lastDefDidMerge = true;
  7123. if (vregInfoDest->mIsRetVal)
  7124. {
  7125. // We don't do a remap for this return value optimization, because we want both vregs
  7126. // to still show in the debugger - even though they will point to the same address
  7127. // now (in the case of a struct return)
  7128. vregInfoCheck->SetRetVal();
  7129. //BF_ASSERT(mCompositeRetVRegIdx != -1);
  7130. //vregInfoCheck->mRelTo = BeMCOperand::FromVReg(mCompositeRetVRegIdx);
  7131. break;
  7132. }
  7133. if ((vregInfoCheck->mDbgVariable != NULL) && (vregInfoDest->mDbgVariable != NULL))
  7134. {
  7135. // We want separate vreg info so we can track the mDbgVariable separately, but we do want
  7136. // the same location
  7137. vregInfoDest->mIsExpr = true;
  7138. vregInfoDest->mRelTo.mKind = BeMCOperandKind_VReg;
  7139. vregInfoDest->mRelTo.mVRegIdx = checkOperand.mVRegIdx;
  7140. }
  7141. else
  7142. {
  7143. // Replace %vreg0 with %vreg1?
  7144. if (vregInfoDest->CanEliminate())
  7145. {
  7146. bool canReplace = true;
  7147. if (canReplace)
  7148. {
  7149. AddRegRemap(nextDestOperand.mVRegIdx, checkOperand.mVRegIdx, regRemaps);
  7150. break;
  7151. }
  7152. }
  7153. // Or go the other way, and make %vreg1 use our variable directly since we're just handing off anyway
  7154. if (vregInfoCheck->CanEliminate())
  7155. {
  7156. if (vregInfoCheck->mIsRetVal)
  7157. {
  7158. vregInfoDest->SetRetVal();
  7159. //BF_ASSERT(mCompositeRetVRegIdx != -1);
  7160. //vregInfoCheck->mRelTo = BeMCOperand::FromVReg(mCompositeRetVRegIdx);
  7161. }
  7162. AddRegRemap(checkOperand.mVRegIdx, nextDestOperand.mVRegIdx, regRemaps);
  7163. if (vregInfoDest->mNaturalReg == X64Reg_None)
  7164. vregInfoDest->mNaturalReg = vregInfoCheck->mNaturalReg;
  7165. break;
  7166. }
  7167. }
  7168. }
  7169. }
  7170. }
  7171. //if (mDebugging)
  7172. {
  7173. // For the form
  7174. // Def %vreg0
  7175. // Mov %vreg0, %vreg1
  7176. // <Op> <Arg>, %vreg0
  7177. // Where vreg0 has no other references, convert to
  7178. // <Op> <arg>, &vreg1
  7179. if ((inst->mKind == BeMCInstKind_Def) && (nextInst->mKind == BeMCInstKind_Mov) && (nextNextInst != NULL))
  7180. {
  7181. auto vregInfo = GetVRegInfo(inst->mArg0);
  7182. if ((vregInfo->mRefCount == 2) && (inst->mArg0 == nextInst->mArg0) && (!nextInst->mArg1.IsNativeReg()) &&
  7183. (nextNextInst->mArg0 != nextInst->mArg0) && (nextNextInst->mArg1 == nextInst->mArg0) &&
  7184. (AreTypesEquivalent(GetType(inst->mArg0), GetType(nextInst->mArg1))))
  7185. {
  7186. nextNextInst->mArg1 = nextInst->mArg1;
  7187. RemoveInst(mcBlock, instIdx);
  7188. RemoveInst(mcBlock, instIdx);
  7189. instIdx--;
  7190. continue;
  7191. }
  7192. }
  7193. }
  7194. // For the form
  7195. // <BinOp> %vreg0, %vreg1
  7196. // Test %vreg0, 1
  7197. // Remove test, because the BinOp will already set the correct flags
  7198. if ((nextInst->mKind == BeMCInstKind_Test) && (nextInst->mArg1.IsImmediateInt()) && (nextInst->mArg1.mImmediate == 1) &&
  7199. (nextInst->mArg0 == inst->mArg0))
  7200. {
  7201. if ((inst->mKind == BeMCInstKind_Xor) || (inst->mKind == BeMCInstKind_Or) || (inst->mKind == BeMCInstKind_And))
  7202. {
  7203. BF_ASSERT(!inst->mResult);
  7204. RemoveInst(mcBlock, instIdx + 1);
  7205. instIdx--;
  7206. continue;
  7207. }
  7208. }
  7209. if (inst->mResult)
  7210. {
  7211. auto nextArg0 = RemapOperand(nextInst->mArg0, regRemaps);
  7212. auto nextArg1 = RemapOperand(nextInst->mArg1, regRemaps);
  7213. // For the form:
  7214. // %vreg2 = <op> %vreg0, %vreg1
  7215. // Mov %vreg3, %vreg2
  7216. // If %vreg2 has no other references, combine into
  7217. // %vreg3 = <op> %vreg0, %vreg1
  7218. if ((nextInst->IsMov()) && (result == nextArg1))
  7219. {
  7220. // We purposely check inst->mResult here rather than 'result', because that ref count is our
  7221. // indication of whether the result was actually used again
  7222. auto vregInfo = GetVRegInfo(inst->mResult);
  7223. if ((vregInfo != NULL) && (vregInfo->mRefCount == 2) && (vregInfo->CanEliminate()))
  7224. {
  7225. // Make sure the MOV isn't a conversion
  7226. auto movArg0Type = GetType(nextInst->mArg0);
  7227. auto movArg1Type = GetType(nextInst->mArg1);
  7228. if (AreTypesEquivalent(movArg0Type, movArg1Type))
  7229. {
  7230. // We actually eliminate the current inst incase there was a Def between the two,
  7231. // otherwise the Def would occur AFTER the assignment which would be bad
  7232. nextInst->mResult = nextInst->mArg0;
  7233. nextInst->mKind = inst->mKind;
  7234. nextInst->mArg0 = inst->mArg0;
  7235. nextInst->mArg1 = inst->mArg1;
  7236. RemoveInst(mcBlock, instIdx);
  7237. instIdx--;
  7238. continue;
  7239. }
  7240. }
  7241. }
  7242. if (inst->IsCommutable())
  7243. {
  7244. if ((nextInst->IsMov()) && (arg1 == nextArg0) && (result == nextArg1))
  7245. {
  7246. break;
  7247. }
  7248. }
  7249. // For the form:
  7250. // %vreg2 = <op> %vreg0, %vreg1
  7251. // If %vreg0 ends its life on this instruction,
  7252. // Replace all instances of %vreg2 with %vreg0
  7253. if ((inst->mResult.mKind == BeMCOperandKind_VReg) && (inst->mArg0.mKind == BeMCOperandKind_VReg) &&
  7254. (inst->mResult != inst->mArg0))
  7255. {
  7256. // Check liveness against both the orig arg0 and the remap
  7257. //if ((!nextInst->mLiveness->IsSet(inst->mArg0.mVRegIdx)) && (!nextInst->mLiveness->IsSet(arg0.mVRegIdx)))
  7258. if (!IsLive(nextInst->mLiveness, inst->mArg0.mVRegIdx, regRemaps))
  7259. {
  7260. // Only do a full regRemap if this is the def initialization
  7261. if ((prevInst != NULL) && (prevInst->mKind == BeMCInstKind_Def) && (prevInst->mArg0.mVRegIdx == inst->mResult.mVRegIdx))
  7262. {
  7263. auto vregInfoResult = GetVRegInfo(result);
  7264. auto vregInfo0 = GetVRegInfo(arg0);
  7265. if ((vregInfoResult->CanEliminate()) && (!vregInfo0->mIsExpr))
  7266. {
  7267. AddRegRemap(inst->mResult.mVRegIdx, arg0.mVRegIdx, regRemaps);
  7268. }
  7269. }
  7270. }
  7271. }
  7272. }
  7273. if (inst->mKind == BeMCInstKind_RestoreVolatiles)
  7274. {
  7275. if (mcBlock == mBlocks.back())
  7276. {
  7277. bool hadInvalidInst = false;
  7278. for (int checkInstIdx = instIdx + 1; checkInstIdx < (int)mcBlock->mInstructions.size(); checkInstIdx++)
  7279. {
  7280. auto checkInst = mcBlock->mInstructions[checkInstIdx];
  7281. if ((checkInst->mKind != BeMCInstKind_DbgDecl) &&
  7282. (checkInst->mKind != BeMCInstKind_EnsureInstructionAt) &&
  7283. (checkInst->mKind != BeMCInstKind_Ret))
  7284. {
  7285. hadInvalidInst = true;
  7286. break;
  7287. }
  7288. }
  7289. if (!hadInvalidInst)
  7290. {
  7291. // We return after this, we don't need to restore volatiles
  7292. inst->mArg1 = BeMCOperand::FromPreserveFlag(BeMCPreserveFlag_NoRestore);
  7293. continue;
  7294. }
  7295. }
  7296. }
  7297. if (nextNextInst == NULL)
  7298. {
  7299. if (reprocessInst)
  7300. instIdx--;
  7301. continue;
  7302. }
  7303. // For the form
  7304. // Xor &vreg0, 1
  7305. // CondBr <label>, <op>
  7306. // If vreg0 dies at the end, convert to to:
  7307. // Test &veg0, 1
  7308. // CondBr <label>, ~<op>
  7309. // This is advantageous because Test is a non-modifying instruction so we can hopefully elide a copy
  7310. if ((inst->mKind == BeMCInstKind_Xor) &&
  7311. (nextInst->mKind == BeMCInstKind_CondBr))
  7312. {
  7313. auto vregInfo = GetVRegInfo(inst->mArg0);
  7314. if ((vregInfo != NULL) && (!IsLive(nextNextInst->mLiveness, inst->mArg0.mVRegIdx, regRemaps)))
  7315. {
  7316. inst->mKind = BeMCInstKind_Test;
  7317. BF_ASSERT(nextInst->mArg1.mKind == BeMCOperandKind_CmpKind);
  7318. nextInst->mArg1.mCmpKind = BeModule::InvertCmp(nextInst->mArg1.mCmpKind);
  7319. if ((instIdx >= 2) && (!lastDefDidMerge))
  7320. {
  7321. auto defInst = mcBlock->mInstructions[instIdx - 2];
  7322. if ((defInst->mKind == BeMCInstKind_Def) && (defInst->mArg0 == inst->mArg0))
  7323. {
  7324. // Go back and try to eliminate this vreg
  7325. instIdx -= 3;
  7326. continue;
  7327. }
  7328. }
  7329. }
  7330. }
  7331. // For the form:
  7332. // %vreg0 = CmpToBool <cmpType>
  7333. // Test %vreg0, %vreg0
  7334. // CondBr %label, eq
  7335. // If %vreg0 has no other references, convert to:
  7336. // CondBr %label, <cmpType>
  7337. if ((inst->mKind == BeMCInstKind_CmpToBool) &&
  7338. (nextInst->mKind == BeMCInstKind_Test) && (nextInst->mArg0 == nextInst->mArg1) &&
  7339. (nextNextInst->mKind == BeMCInstKind_CondBr) &&
  7340. (nextInst->mArg0 == inst->mResult))
  7341. {
  7342. auto vregInfo = GetVRegInfo(inst->mResult);
  7343. if ((vregInfo != NULL) && (vregInfo->mRefCount == 3))
  7344. {
  7345. BF_ASSERT(nextNextInst->mArg1.mKind == BeMCOperandKind_CmpKind);
  7346. nextNextInst->mArg1.mCmpKind = inst->mArg0.mCmpKind;
  7347. RemoveInst(mcBlock, nextIdx);
  7348. RemoveInst(mcBlock, instIdx);
  7349. instIdx--;
  7350. }
  7351. }
  7352. if (reprocessInst)
  7353. instIdx--;
  7354. }
  7355. }
  7356. SetCurrentInst(NULL);
  7357. Dictionary<int, int> regRemapMap;
  7358. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  7359. {
  7360. int remapIdx = regRemaps.GetHead(vregIdx);
  7361. if (remapIdx != vregIdx)
  7362. regRemapMap[vregIdx] = remapIdx;
  7363. }
  7364. // We want to keep the earliest @Def and remap (in necessary) the later one
  7365. // For remaps, the liveVRregIdx will always indicate which @Def occurs earlier,
  7366. // unless there's a case I haven't considered
  7367. Dictionary<int, int> defMap;
  7368. for (auto& remapPair : regRemapMap)
  7369. {
  7370. int remapFrom = remapPair.mKey;
  7371. int remapTo = remapPair.mValue;
  7372. if (remapTo < 0)
  7373. continue;
  7374. auto vregInfoFrom = mVRegInfo[remapFrom];
  7375. auto vregInfoTo = mVRegInfo[remapTo];
  7376. int bestVRegIdx = remapTo;
  7377. //auto itr = defMap.find(remapTo);
  7378. //if (itr != defMap.end())
  7379. int* prevBestVRegIdxPtr = NULL;
  7380. if (defMap.TryGetValue(remapTo, &prevBestVRegIdxPtr))
  7381. {
  7382. }
  7383. else
  7384. defMap[remapTo] = bestVRegIdx;
  7385. }
  7386. struct _RemapEntry
  7387. {
  7388. int mVRegFrom;
  7389. int mVRegTo;
  7390. };
  7391. Array<_RemapEntry> initRemaps;
  7392. // We have a many-to-one relation so we have to use both a
  7393. HashSet<int> keepDefs;
  7394. for (auto& defPair : defMap)
  7395. keepDefs.Add(defPair.mValue);
  7396. HashSet<int> removeDefs;
  7397. for (auto& remapPair : regRemapMap)
  7398. {
  7399. removeDefs.Add(remapPair.mKey);
  7400. keepDefs.Add(remapPair.mValue);
  7401. auto vregInfoFrom = mVRegInfo[remapPair.mKey];
  7402. auto vregInfoTo = mVRegInfo[remapPair.mValue];
  7403. if (vregInfoFrom->mHasDynLife)
  7404. {
  7405. vregInfoTo->mHasDynLife = true;
  7406. _RemapEntry remapEntry = { remapPair.mKey, remapPair.mValue };
  7407. initRemaps.Add(remapEntry);
  7408. }
  7409. if (vregInfoFrom->mForceMem)
  7410. vregInfoTo->mForceMem = true;
  7411. }
  7412. auto _RemapVReg = [&](int& vregIdx)
  7413. {
  7414. int* regIdxPtr = NULL;
  7415. if (regRemapMap.TryGetValue(vregIdx, &regIdxPtr))
  7416. {
  7417. int regIdx = *regIdxPtr;
  7418. if (regIdx < 0)
  7419. {
  7420. Fail("Invalid reg remap");
  7421. }
  7422. else
  7423. vregIdx = regIdx;
  7424. }
  7425. };
  7426. auto _RemapOperand = [&](BeMCOperand* operand)
  7427. {
  7428. if (operand->IsVRegAny())
  7429. {
  7430. int* regIdxPtr = NULL;
  7431. if (regRemapMap.TryGetValue(operand->mVRegIdx, &regIdxPtr))
  7432. {
  7433. int regIdx = *regIdxPtr;
  7434. if (regIdx < 0)
  7435. {
  7436. if (operand->mKind == BeMCOperandKind_VRegLoad)
  7437. operand->mKind = BeMCOperandKind_VReg;
  7438. else
  7439. NotImpl();
  7440. operand->mVRegIdx = -regIdx;
  7441. }
  7442. else
  7443. {
  7444. operand->mVRegIdx = regIdx;
  7445. }
  7446. }
  7447. }
  7448. if (operand->mKind == BeMCOperandKind_VRegPair)
  7449. {
  7450. _RemapVReg(operand->mVRegPair.mVRegIdx0);
  7451. _RemapVReg(operand->mVRegPair.mVRegIdx1);
  7452. }
  7453. if (operand->IsVRegAny())
  7454. {
  7455. if (wantUnwrapVRegs.Contains(operand->mVRegIdx))
  7456. {
  7457. // Replace any rel-to symbol info with direct references to those symbols
  7458. auto vregInfo = mVRegInfo[operand->mVRegIdx];
  7459. BF_ASSERT(vregInfo->IsDirectRelToAny());
  7460. if (operand->mKind == BeMCOperandKind_VReg)
  7461. *operand = vregInfo->mRelTo;
  7462. else if (operand->mKind == BeMCOperandKind_VRegAddr)
  7463. *operand = OperandToAddr(vregInfo->mRelTo);
  7464. }
  7465. }
  7466. };
  7467. Dictionary<BeVTrackingList*, BeVTrackingList*> initRemapMap;
  7468. if ((!regRemapMap.IsEmpty()) || (!wantUnwrapVRegs.IsEmpty()))
  7469. {
  7470. for (auto mcBlock : mBlocks)
  7471. {
  7472. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  7473. {
  7474. auto inst = mcBlock->mInstructions[instIdx];
  7475. if (inst->IsDef())
  7476. {
  7477. if ((removeDefs.Contains(inst->mArg0.mVRegIdx)) || (wantUnwrapVRegs.Contains(inst->mArg0.mVRegIdx)))
  7478. {
  7479. RemoveInst(mcBlock, instIdx);
  7480. instIdx--;
  7481. continue;
  7482. }
  7483. }
  7484. auto operands = { &inst->mResult, &inst->mArg0, &inst->mArg1 };
  7485. for (auto& operand : operands)
  7486. {
  7487. _RemapOperand(operand);
  7488. }
  7489. if ((!initRemaps.IsEmpty()) && (inst->mVRegsInitialized != NULL))
  7490. {
  7491. BeVTrackingList** vregsInitializedValuePtr = NULL;
  7492. if (initRemapMap.TryAdd(inst->mVRegsInitialized, NULL, &vregsInitializedValuePtr))
  7493. {
  7494. SizedArray<int, 16> removeVec;
  7495. SizedArray<int, 16> addVec;
  7496. SizedArray<int, 16> filteredRemoveVec;
  7497. SizedArray<int, 16> filteredAddVec;
  7498. for (auto& entry : initRemaps)
  7499. {
  7500. if (mVRegInitializedContext.IsSet(inst->mVRegsInitialized, entry.mVRegFrom))
  7501. {
  7502. removeVec.Add(entry.mVRegFrom);
  7503. if (!mVRegInitializedContext.IsSet(inst->mVRegsInitialized, entry.mVRegTo))
  7504. addVec.Add(entry.mVRegTo);
  7505. }
  7506. }
  7507. BeVTrackingList* vregsInitialized = mVRegInitializedContext.Modify(inst->mVRegsInitialized, addVec, removeVec, filteredAddVec, filteredRemoveVec, true);
  7508. *vregsInitializedValuePtr = vregsInitialized;
  7509. }
  7510. inst->mVRegsInitialized = *vregsInitializedValuePtr;
  7511. }
  7512. }
  7513. }
  7514. for (auto vregInfo : mVRegInfo)
  7515. {
  7516. auto operands = { &vregInfo->mRelTo, &vregInfo->mRelOffset };
  7517. for (auto& operand : operands)
  7518. {
  7519. if (operand->IsVRegAny())
  7520. {
  7521. RemapOperand(*operand, regRemaps);
  7522. _RemapOperand(operand);
  7523. }
  7524. }
  7525. }
  7526. }
  7527. }
  7528. void BeMCContext::DoRegAssignPass()
  7529. {
  7530. BP_ZONE("BeMCContext::DoRegAssignPass");
  7531. bool generateLiveness = true;
  7532. //
  7533. if (generateLiveness)
  7534. {
  7535. for (auto& node : mColorizer.mNodes)
  7536. {
  7537. node.mEdges.Clear();
  7538. }
  7539. mColorizer.Prepare();
  7540. GenerateLiveness();
  7541. }
  7542. else
  7543. {
  7544. mColorizer.Prepare();
  7545. }
  7546. mColorizer.GenerateRegCosts();
  7547. //
  7548. {
  7549. BP_ZONE("BeMCContext::DoRegAssignPass:ints");
  7550. mColorizer.AssignRegs(BeMCColorizer::RegKind_Ints);
  7551. }
  7552. //
  7553. {
  7554. BP_ZONE("BeMCContext::DoRegAssignPass:float");
  7555. mColorizer.AssignRegs(BeMCColorizer::RegKind_Floats);
  7556. }
  7557. #ifdef _DEBUG
  7558. BF_ASSERT(mColorizer.Validate());
  7559. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  7560. {
  7561. auto vregInfo = mVRegInfo[vregIdx];
  7562. if (vregInfo->mRefCount == 0)
  7563. continue;
  7564. if ((vregInfo->mIsRetVal) && (mCompositeRetVRegIdx != -1) && (vregIdx != mCompositeRetVRegIdx))
  7565. continue;
  7566. if ((vregInfo->mForceReg) && (!vregInfo->mRelTo) && (vregInfo->mReg == X64Reg_None))
  7567. {
  7568. SoftFail("Failed to assign register to ForceReg vreg");
  7569. }
  7570. if (vregInfo->mDisableRAX)
  7571. {
  7572. BF_ASSERT(ResizeRegister(vregInfo->mReg, 8) != X64Reg_RAX);
  7573. }
  7574. if (vregInfo->mDisableRDX)
  7575. {
  7576. BF_ASSERT(ResizeRegister(vregInfo->mReg, 8) != X64Reg_RDX);
  7577. }
  7578. if (vregInfo->mDisableR11)
  7579. {
  7580. BF_ASSERT(ResizeRegister(vregInfo->mReg, 8) != X64Reg_R11);
  7581. }
  7582. if (vregInfo->mDisableR12)
  7583. {
  7584. BF_ASSERT(ResizeRegister(vregInfo->mReg, 8) != X64Reg_R12);
  7585. }
  7586. if (vregInfo->mDisableR13)
  7587. {
  7588. BF_ASSERT(ResizeRegister(vregInfo->mReg, 8) != X64Reg_R13);
  7589. }
  7590. }
  7591. #endif
  7592. }
  7593. void BeMCContext::DoFrameObjPass()
  7594. {
  7595. BF_ASSERT(mBlocks.size() == 1);
  7596. SetCurrentInst(NULL);
  7597. // MS x64 ABI requires a "home address" of 4 intptrs when we call a function, plus whatever
  7598. // we need for calls with more than 4 params.
  7599. // If we're doing UseBP, we have to allocate these at call time
  7600. int homeSize = BF_ALIGN(BF_MAX(mMaxCallParamCount, 4) * 8, 16);
  7601. if (mMaxCallParamCount == -1)
  7602. homeSize = 0;
  7603. mStackSize = 0;
  7604. if (mUseBP)
  7605. mUsedRegs.Add(X64Reg_RBP);
  7606. int regStackOffset = 0;
  7607. int xmmRegStackSize = 0;
  7608. for (auto usedReg : mUsedRegs)
  7609. {
  7610. if (!IsVolatileReg(usedReg))
  7611. {
  7612. BF_ASSERT(usedReg != X64Reg_RSP);
  7613. if (IsXMMReg(usedReg))
  7614. xmmRegStackSize += 16;
  7615. else
  7616. regStackOffset += 8;
  7617. }
  7618. }
  7619. int xmmStackOffset = -1;
  7620. if (xmmRegStackSize > 0)
  7621. {
  7622. int align = 16;
  7623. int alignOffset = regStackOffset + 8;
  7624. int alignedPosition = (mStackSize + alignOffset + (align - 1)) & ~(align - 1);
  7625. mStackSize = alignedPosition - alignOffset;
  7626. mStackSize += xmmRegStackSize;
  7627. xmmStackOffset = -mStackSize - regStackOffset;
  7628. }
  7629. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  7630. {
  7631. auto vregInfo = mVRegInfo[vregIdx];
  7632. if (vregInfo->mRelOffset.mKind == BeMCOperandKind_Immediate_HomeSize)
  7633. {
  7634. vregInfo->mRelOffset = BeMCOperand::FromImmediate(homeSize);
  7635. }
  7636. if ((vregInfo->mIsRetVal) && (vregIdx != mCompositeRetVRegIdx))
  7637. continue;
  7638. if ((vregInfo->mRefCount > 0) && (!vregInfo->mIsExpr) && (vregInfo->mReg == X64Reg_None) && (vregInfo->mFrameOffset == INT_MIN))
  7639. {
  7640. BF_ASSERT(vregInfo->mAlign != -1);
  7641. int align = BF_MAX(vregInfo->mAlign, 1);
  7642. int alignOffset = regStackOffset + 8;
  7643. int alignedPosition = (mStackSize + alignOffset + (align - 1)) & ~(align - 1);
  7644. mStackSize = alignedPosition - alignOffset;
  7645. //vregInfo->mFrameOffset = -mStackSize - regStackOffset - 8;
  7646. mStackSize += BF_ALIGN(vregInfo->mType->mSize, vregInfo->mAlign);
  7647. vregInfo->mFrameOffset = -mStackSize - regStackOffset;
  7648. }
  7649. }
  7650. // If we have dynamic stack resizing then we have a stack frame and must be 16-byte aligned
  7651. // even if we're a leaf function
  7652. bool mHasFramePointer = false;
  7653. // Potentially pull off retaddr bytes if alignment allows --
  7654. // Stack must be aligned to 16 bytes and retaddr offsets us by 8,
  7655. // so we must always offset by 0x?8
  7656. int align = 8;
  7657. mStackSize = (mStackSize + (align - 1)) & ~(align - 1);
  7658. //if (!mUseBP)
  7659. {
  7660. // MS x64 ABI requires a "home address" of 4 intptrs when we call a function, plus whatever
  7661. // we need for calls with more than 4 params.
  7662. // If we're doing UseBP, we have to allocate these at call time
  7663. if (mMaxCallParamCount != -1)
  7664. {
  7665. mStackSize += homeSize;
  7666. // Even param counts, to align to 16 bytes.
  7667. // Minimum is space for 4 params
  7668. //int paramSpace = (std::max(mMaxCallParamCount, 4) + 1) & ~1;
  7669. //mStackSize += paramSpace * 8;
  7670. }
  7671. }
  7672. int stackAdjust = mStackSize;
  7673. mStackSize += regStackOffset;
  7674. if ((mStackSize != 0) && (mStackSize % 16 == 0))
  7675. {
  7676. mStackSize += 8;
  7677. stackAdjust += 8;
  7678. }
  7679. mActiveBlock = mBlocks[0];
  7680. if (mUseBP)
  7681. {
  7682. AllocInst(BeMCInstKind_Unwind_SetBP, 0);
  7683. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RBP), BeMCOperand::FromReg(X64Reg_RSP), 0);
  7684. }
  7685. int xmmRegIdx = 0;
  7686. for (auto usedReg : mUsedRegs)
  7687. {
  7688. if ((!IsVolatileReg(usedReg)) && (IsXMMReg(usedReg)))
  7689. {
  7690. AllocInst(BeMCInstKind_Unwind_SaveXMM, BeMCOperand::FromReg(usedReg), BeMCOperand::FromImmediate(mStackSize + xmmStackOffset + xmmRegIdx * 16), 0);
  7691. AllocInst(BeMCInstKind_Push, BeMCOperand::FromReg(usedReg), BeMCOperand::FromImmediate(mStackSize + xmmStackOffset + xmmRegIdx * 16), 0);
  7692. xmmRegIdx++;
  7693. }
  7694. }
  7695. if (stackAdjust > 0)
  7696. {
  7697. AllocInst(BeMCInstKind_Unwind_Alloc, BeMCOperand::FromImmediate(stackAdjust), 0);
  7698. AllocInst(BeMCInstKind_Sub, BeMCOperand::FromReg(X64Reg_RSP), BeMCOperand::FromImmediate(stackAdjust), 0);
  7699. if (stackAdjust >= 4096)
  7700. {
  7701. BeMCOperand mcFunc;
  7702. mcFunc.mKind = BeMCOperandKind_SymbolAddr;
  7703. mcFunc.mSymbolIdx = mCOFFObject->GetSymbolRef("__chkstk")->mIdx;
  7704. AllocInst(BeMCInstKind_Call, mcFunc, 0);
  7705. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand::FromImmediate(stackAdjust), 0);
  7706. }
  7707. }
  7708. for (auto usedReg : mUsedRegs)
  7709. {
  7710. if ((!IsVolatileReg(usedReg)) && (!IsXMMReg(usedReg)))
  7711. {
  7712. AllocInst(BeMCInstKind_Unwind_PushReg, BeMCOperand::FromReg(usedReg), 0);
  7713. AllocInst(BeMCInstKind_Push, BeMCOperand::FromReg(usedReg), 0);
  7714. }
  7715. }
  7716. if (mHasVAStart)
  7717. {
  7718. for (int i = 0; i < 4; i++)
  7719. {
  7720. auto regSaveVReg = AllocVirtualReg(mModule->mContext->GetPointerTo(mModule->mContext->GetPrimitiveType(BeTypeCode_Int64)));
  7721. auto vregInfo = GetVRegInfo(regSaveVReg);
  7722. vregInfo->mRelTo = BeMCOperand::FromReg(X64Reg_RSP);
  7723. vregInfo->mRelOffset = BeMCOperand::FromImmediate(i * 8 + 8);
  7724. vregInfo->mIsExpr = true;
  7725. X64CPURegister reg;
  7726. switch (i)
  7727. {
  7728. case 0:
  7729. reg = X64Reg_RCX;
  7730. break;
  7731. case 1:
  7732. reg = X64Reg_RDX;
  7733. break;
  7734. case 2:
  7735. reg = X64Reg_R8;
  7736. break;
  7737. case 3:
  7738. reg = X64Reg_R9;
  7739. break;
  7740. }
  7741. AllocInst(BeMCInstKind_Mov, BeMCOperand::ToLoad(regSaveVReg), BeMCOperand::FromReg(reg), 0);
  7742. }
  7743. }
  7744. BeMCOperand restoreBPVal;
  7745. if (mUseBP)
  7746. {
  7747. auto nativeType = mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  7748. restoreBPVal = AllocVirtualReg(nativeType);
  7749. auto restoreBPVReg = mVRegInfo[restoreBPVal.mVRegIdx];
  7750. restoreBPVReg->mIsExpr = true;
  7751. restoreBPVReg->mRelTo = BeMCOperand::FromReg(X64Reg_RBP);
  7752. restoreBPVReg->mRelOffset = BeMCOperand::FromImmediate(stackAdjust);
  7753. restoreBPVal.mKind = BeMCOperandKind_VReg;
  7754. }
  7755. for (int instIdx = 0; instIdx < (int)mActiveBlock->mInstructions.size(); instIdx++)
  7756. {
  7757. auto checkInst = mActiveBlock->mInstructions[instIdx];
  7758. if (checkInst->mKind == BeMCInstKind_Ret)
  7759. {
  7760. mCurDbgLoc = checkInst->mDbgLoc;
  7761. int insertIdx = instIdx;
  7762. int xmmRegIdx = 0;
  7763. for (auto usedReg : mUsedRegs)
  7764. {
  7765. if ((!IsVolatileReg(usedReg)) && (IsXMMReg(usedReg)))
  7766. {
  7767. AllocInst(BeMCInstKind_Pop, BeMCOperand::FromReg(usedReg), BeMCOperand::FromImmediate(mStackSize + xmmStackOffset + xmmRegIdx * 16), insertIdx++);
  7768. xmmRegIdx++;
  7769. }
  7770. }
  7771. if (mUseBP)
  7772. {
  7773. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RSP), restoreBPVal, insertIdx++);
  7774. }
  7775. else if (stackAdjust > 0)
  7776. {
  7777. AllocInst(BeMCInstKind_Add, BeMCOperand::FromReg(X64Reg_RSP), BeMCOperand::FromImmediate(stackAdjust), insertIdx++);
  7778. instIdx++;
  7779. }
  7780. for (auto usedReg : mUsedRegs)
  7781. {
  7782. if ((!IsVolatileReg(usedReg)) && (!IsXMMReg(usedReg)))
  7783. AllocInst(BeMCInstKind_Pop, BeMCOperand::FromReg(usedReg), insertIdx++);
  7784. }
  7785. instIdx = insertIdx;
  7786. }
  7787. }
  7788. }
  7789. static int gLegalizeIdx = 0;
  7790. void BeMCContext::DoActualization()
  7791. {
  7792. while (true)
  7793. {
  7794. bool hasMoreWork = false;
  7795. for (auto mcBlock : mBlocks)
  7796. {
  7797. mActiveBlock = mcBlock;
  7798. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  7799. {
  7800. auto inst = mcBlock->mInstructions[instIdx];
  7801. SetCurrentInst(inst);
  7802. bool forceMove = false;
  7803. if (inst->mKind == BeMCInstKind_DbgDecl)
  7804. {
  7805. auto vregInfo = GetVRegInfo(inst->mArg0);
  7806. if (vregInfo->mWantsExprActualize)
  7807. {
  7808. if (inst->mArg0.mKind == BeMCOperandKind_VReg)
  7809. {
  7810. if (vregInfo->mRelTo.IsSymbol())
  7811. {
  7812. forceMove = true;
  7813. }
  7814. else if (vregInfo->mDbgVariable->mIsValue)
  7815. {
  7816. vregInfo->mDbgVariable->mIsValue = false;
  7817. inst->mArg0.mKind = BeMCOperandKind_VRegAddr;
  7818. }
  7819. else
  7820. {
  7821. inst->mArg0.mKind = BeMCOperandKind_VRegAddr;
  7822. vregInfo->mDbgVariable->mType = mModule->mDbgModule->CreateReferenceType(BeValueDynCast<BeDbgType>(vregInfo->mDbgVariable->mType));
  7823. }
  7824. }
  7825. vregInfo->mWantsExprActualize = false;
  7826. }
  7827. }
  7828. if ((inst->IsDef()) || (forceMove))
  7829. {
  7830. int vregIdx = inst->mArg0.mVRegIdx;
  7831. auto vregInfo = GetVRegInfo(inst->mArg0);
  7832. if (vregInfo->mWantsExprOffsetActualize)
  7833. {
  7834. vregInfo->mWantsExprOffsetActualize = false;
  7835. auto offsetType = GetType(vregInfo->mRelOffset);
  7836. auto scratchReg = AllocVirtualReg(offsetType, 2, false);
  7837. CreateDefineVReg(scratchReg, instIdx++);
  7838. AllocInst(BeMCInstKind_Mov, scratchReg, vregInfo->mRelOffset, instIdx++);
  7839. AllocInst(BeMCInstKind_IMul, scratchReg, BeMCOperand::FromImmediate(vregInfo->mRelOffsetScale), instIdx++);
  7840. vregInfo->mRelOffset = scratchReg;
  7841. vregInfo->mRelOffsetScale = 1;
  7842. }
  7843. if ((vregInfo->mWantsExprActualize) || (forceMove))
  7844. {
  7845. if (vregInfo->mDbgVariable != NULL)
  7846. {
  7847. // Wait until dbgDecl so we can potentially set to VRegAddr
  7848. }
  7849. else
  7850. {
  7851. vregInfo->mWantsExprActualize = false;
  7852. }
  7853. BF_ASSERT((!vregInfo->IsDirectRelTo()) || (vregInfo->mDbgVariable != NULL));
  7854. if (vregInfo->mIsExpr)
  7855. {
  7856. if (vregInfo->IsDirectRelToAny())
  7857. {
  7858. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(vregIdx), vregInfo->mRelTo, instIdx++ + 1);
  7859. vregInfo->mIsExpr = false;
  7860. vregInfo->mRelTo = BeMCOperand();
  7861. vregInfo->mRelOffset = BeMCOperand();
  7862. vregInfo->mRelOffsetScale = 1;
  7863. }
  7864. else
  7865. {
  7866. // Create a new reg with the expression, and then load the value into the old non-expressionized reg
  7867. // This has the advantage of fixing any other references to this expr, too
  7868. auto scratchReg = AllocVirtualReg(vregInfo->mType, 2, false);
  7869. auto scratchVRegInfo = mVRegInfo[scratchReg.mVRegIdx];
  7870. scratchVRegInfo->mIsExpr = true;
  7871. scratchVRegInfo->mRelTo = vregInfo->mRelTo;
  7872. scratchVRegInfo->mRelOffset = vregInfo->mRelOffset;
  7873. scratchVRegInfo->mRelOffsetScale = vregInfo->mRelOffsetScale;
  7874. vregInfo->mIsExpr = false;
  7875. vregInfo->mRelTo = BeMCOperand();
  7876. vregInfo->mRelOffset = BeMCOperand();
  7877. vregInfo->mRelOffsetScale = 1;
  7878. CreateDefineVReg(scratchReg, instIdx++ + 1);
  7879. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(vregIdx), scratchReg, instIdx++ + 1);
  7880. }
  7881. }
  7882. }
  7883. }
  7884. }
  7885. }
  7886. if (!hasMoreWork)
  7887. break;
  7888. }
  7889. }
  7890. void BeMCContext::DoLoads()
  7891. {
  7892. for (auto mcBlock : mBlocks)
  7893. {
  7894. mActiveBlock = mcBlock;
  7895. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  7896. {
  7897. auto inst = mcBlock->mInstructions[instIdx];
  7898. SetCurrentInst(inst);
  7899. if (inst->mKind == BeMCInstKind_DefLoad)
  7900. {
  7901. // Almost all DefLoads should be eliminated during InstCombine, but for 'persistent loads', we need
  7902. // to actualize them now
  7903. inst->mKind = BeMCInstKind_Def;
  7904. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  7905. if (vregInfo->mRefCount == 0)
  7906. continue;
  7907. auto relLoaded = vregInfo->mRelTo;
  7908. auto loadTo = inst->mArg0;
  7909. BF_ASSERT(loadTo.mKind == BeMCOperandKind_VReg);
  7910. AllocInst(BeMCInstKind_Mov, loadTo, relLoaded, instIdx++ + 1);
  7911. vregInfo->mIsExpr = false;
  7912. vregInfo->mRelTo = BeMCOperand();
  7913. }
  7914. if (inst->mKind == BeMCInstKind_Def)
  7915. {
  7916. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  7917. if (vregInfo->mRefCount == 0)
  7918. continue;
  7919. if (vregInfo->mRelTo.mKind == BeMCOperandKind_VRegAddr)
  7920. {
  7921. auto vregRelInfo = mVRegInfo[vregInfo->mRelTo.mVRegIdx];
  7922. vregRelInfo->mForceMem = true;
  7923. CheckForce(vregInfo);
  7924. }
  7925. }
  7926. }
  7927. }
  7928. }
  7929. static int sLegalizationIdx = 0;
  7930. // Returning false means we generated some new vregs that need to be assigned to registers
  7931. bool BeMCContext::DoLegalization()
  7932. {
  7933. BP_ZONE("BeMCContext::DoLegalization");
  7934. bool debugging = false;
  7935. //if (mBeFunction->mName == "?DrawEntry@DrawContext@PerfView@BeefPerf@@QEAAXPEAVGraphics@gfx@Beefy@@PEAVTrackNodeEntry@23@MM@Z")
  7936. //debugging = true;
  7937. if (debugging)
  7938. OutputDebugStrF("DoLegalization\n");
  7939. bool isFinalRun = true;
  7940. int startingVRegCount = (int)mVRegInfo.size();
  7941. HashSet<int> vregExprChangeSet;
  7942. bool hasPendingActualizations = false;
  7943. SetCurrentInst(NULL);
  7944. int regPreserveDepth = 0;
  7945. for (auto mcBlock : mBlocks)
  7946. {
  7947. mActiveBlock = mcBlock;
  7948. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  7949. {
  7950. BfIRInitType pendingInitKind = BfIRInitType_NotNeeded;
  7951. gLegalizeIdx++;
  7952. auto inst = mcBlock->mInstructions[instIdx];
  7953. SetCurrentInst(inst);
  7954. mActiveInst = inst;
  7955. if (inst->mKind == BeMCInstKind_Mov)
  7956. {
  7957. // Useless mov, remove it
  7958. if (OperandsEqual(inst->mArg0, inst->mArg1, true))
  7959. {
  7960. // We leave the instructions alone here and remove them in DoRegFinalization.
  7961. // There are some cases with DirectRelTo vregs where we need to preserve liveness that a mov
  7962. // generates. We just want to keep from inserting any additional legalization instructions here (ie: movs to temps)
  7963. continue;
  7964. }
  7965. }
  7966. BeMCInst* nextInst = NULL;
  7967. if (instIdx < mcBlock->mInstructions.size() - 1)
  7968. {
  7969. // It's important that we do this at the top before any instructions might get split, so liveness of the next instruction can be analyzed
  7970. nextInst = mcBlock->mInstructions[instIdx + 1];
  7971. }
  7972. auto arg0 = GetFixedOperand(inst->mArg0);
  7973. auto arg1 = GetFixedOperand(inst->mArg1);
  7974. auto arg0Type = GetType(inst->mArg0);
  7975. auto arg1Type = GetType(inst->mArg1);
  7976. if ((arg0Type != NULL) && (arg0Type->mSize == 0) && (!inst->IsPsuedo()))
  7977. {
  7978. RemoveInst(mcBlock, instIdx);
  7979. instIdx--;
  7980. continue;
  7981. }
  7982. if ((inst->mKind == BeMCInstKind_Sub) && (OperandsEqual(inst->mResult, inst->mArg1)))
  7983. {
  7984. // From: b = Sub a, b
  7985. // To: Neg b
  7986. // Add b, a
  7987. AllocInst(BeMCInstKind_Add, inst->mResult, inst->mArg0, instIdx + 1);
  7988. inst->mKind = BeMCInstKind_Neg;
  7989. inst->mArg0 = inst->mResult;
  7990. inst->mArg1 = BeMCOperand();
  7991. inst->mResult = BeMCOperand();
  7992. isFinalRun = false;
  7993. instIdx++;
  7994. continue;
  7995. }
  7996. // Check operands
  7997. if ((!inst->IsPsuedo()) && (arg0Type != NULL) && (!arg0Type->IsNonVectorComposite()))
  7998. {
  7999. bool replacedOpr = false;
  8000. for (int oprIdx = 0; oprIdx < 2; oprIdx++)
  8001. {
  8002. BeMCOperand& origOperand = (oprIdx == 0) ? inst->mArg0 : inst->mArg1;
  8003. BeMCOperand& remappedOperand = (oprIdx == 0) ? arg0 : arg1;
  8004. if (remappedOperand.IsVRegAny())
  8005. {
  8006. bool isMulti = false;
  8007. BeRMParamsInfo rmInfo;
  8008. GetRMParams(remappedOperand, rmInfo);
  8009. bool badOperand = false;
  8010. bool scratchForceReg = false;
  8011. if (rmInfo.mMode == BeMCRMMode_Invalid)
  8012. {
  8013. badOperand = true;
  8014. scratchForceReg = true;
  8015. }
  8016. // We can only allow a "complex direct" expression such as (RAX+RBX) on the SRC side of
  8017. // a mov, where it can be emitted as a LEA. All other uses are invalid.
  8018. bool allowComplexDirect = (inst->mKind == BeMCInstKind_Mov) && (oprIdx == 1);
  8019. if (allowComplexDirect)
  8020. {
  8021. auto vregInfo = GetVRegInfo(remappedOperand);
  8022. if ((vregInfo->mRelTo.mKind == BeMCOperandKind_SymbolAddr) &&
  8023. (vregInfo->mRelOffset.IsImmediateInt()) &&
  8024. (vregInfo->mRelOffsetScale == 1))
  8025. {
  8026. badOperand = false;
  8027. }
  8028. }
  8029. else
  8030. {
  8031. if ((rmInfo.mMode == BeMCRMMode_Direct) &&
  8032. ((rmInfo.mRegB != X64Reg_None) || (rmInfo.mBScale != 1) || (rmInfo.mDisp != 0)))
  8033. {
  8034. badOperand = true;
  8035. }
  8036. // bool isSimple = (rmForm == BeMCRMMode_Direct) && (regB == X64Reg_None) && (bScale == 1) && (disp == 0);
  8037. // if ((!isSimple) && (oprIdx == 1) && (!arg0isSimple))
  8038. // {
  8039. // NOP;
  8040. // }
  8041. // if (oprIdx == 0)
  8042. // arg0isSimple = isSimple;
  8043. }
  8044. if (badOperand)
  8045. {
  8046. BeMCOperand checkOperand = remappedOperand;
  8047. while (checkOperand.mKind == BeMCOperandKind_VReg)
  8048. {
  8049. auto vregInfo = GetVRegInfo(checkOperand);
  8050. if (!vregInfo->IsDirectRelToAny())
  8051. break;
  8052. checkOperand = vregInfo->mRelTo;
  8053. }
  8054. if (inst->mKind == BeMCInstKind_Mov)
  8055. {
  8056. if ((checkOperand.mKind == BeMCOperandKind_Symbol) || (checkOperand.mKind == BeMCOperandKind_SymbolAddr))
  8057. {
  8058. // SymbolAddr can be emitted as an LEA
  8059. badOperand = false;
  8060. }
  8061. if (checkOperand.mKind == BeMCOperandKind_VRegAddr)
  8062. {
  8063. auto vregInfo = GetVRegInfo(checkOperand);
  8064. if ((vregInfo->IsDirectRelToAny()) && (vregInfo->mRelTo.mKind == BeMCOperandKind_Symbol))
  8065. {
  8066. badOperand = false;
  8067. }
  8068. }
  8069. }
  8070. }
  8071. if (badOperand)
  8072. {
  8073. if ((rmInfo.mErrorVReg == -2) && (rmInfo.mVRegWithScaledOffset != -1))
  8074. {
  8075. auto offsetVRegInfo = mVRegInfo[rmInfo.mVRegWithScaledOffset];
  8076. offsetVRegInfo->mWantsExprOffsetActualize = true;
  8077. hasPendingActualizations = true;
  8078. }
  8079. else if (!vregExprChangeSet.Contains(rmInfo.mErrorVReg))
  8080. {
  8081. auto remappedVRegInfo = mVRegInfo[remappedOperand.mVRegIdx];
  8082. BeMCOperand savedOperand = BeMCOperand::FromVReg(remappedOperand.mVRegIdx);
  8083. BeMCOperand newOperand;
  8084. bool replaceLoad = origOperand.mKind == BeMCOperandKind_VRegLoad;
  8085. // If we have a load on both the target and a load on a failed arg then do *not* keep the load-ness of the replaced arg
  8086. if ((replaceLoad) &&
  8087. ((inst->mArg0 == origOperand) || (inst->mArg1 == origOperand)) &&
  8088. (HasLoad(inst->mResult)) && (HasLoad((origOperand))))
  8089. {
  8090. // This can not only fix further 'bad operand' passes, it can fix aliasing problems
  8091. replaceLoad = false;
  8092. }
  8093. if (replaceLoad)
  8094. {
  8095. // Loads keep their load-ness
  8096. savedOperand = BeMCOperand::FromVReg(remappedOperand.mVRegIdx);
  8097. newOperand = origOperand;
  8098. }
  8099. else
  8100. {
  8101. savedOperand = origOperand;
  8102. newOperand.mKind = BeMCOperandKind_VReg;
  8103. }
  8104. auto type = GetType(savedOperand);
  8105. //TODO: Are there circumstances when this needs to be forceReg?
  8106. BeMCOperand scratchReg = AllocVirtualReg(type, 2, scratchForceReg);
  8107. int insertPos = FindSafeInstInsertPos(instIdx);
  8108. CreateDefineVReg(scratchReg, insertPos);
  8109. AllocInst(BeMCInstKind_Mov, scratchReg, savedOperand, insertPos + 1);
  8110. newOperand.mVRegIdx = scratchReg.mVRegIdx;
  8111. IntroduceVRegs(scratchReg, mcBlock, insertPos, insertPos + 1);
  8112. auto scratchVRegInfo = mVRegInfo[scratchReg.mVRegIdx];
  8113. // If a scratch vreg needs to preserve a register like the remapped vreg
  8114. scratchVRegInfo->mDisableR11 |= remappedVRegInfo->mDisableR11;
  8115. scratchVRegInfo->mDisableR12 |= remappedVRegInfo->mDisableR12;
  8116. scratchVRegInfo->mDisableR13 |= remappedVRegInfo->mDisableR13;
  8117. scratchVRegInfo->mDisableRAX |= remappedVRegInfo->mDisableRAX;
  8118. scratchVRegInfo->mDisableRDX |= remappedVRegInfo->mDisableRDX;
  8119. scratchVRegInfo->mDisableEx |= remappedVRegInfo->mDisableEx;
  8120. if ((insertPos == instIdx) || (!scratchForceReg))
  8121. {
  8122. // Only allow short-lived forceRegs
  8123. instIdx += 2;
  8124. //origOperand.mVRegIdx = scratchReg.mVRegIdx;
  8125. origOperand = newOperand;
  8126. }
  8127. else
  8128. {
  8129. // .. otherwise we need yet another layer of indirection for the load
  8130. auto loadedType = GetType(inst->mArg0);
  8131. BeMCOperand loadedReg = AllocVirtualReg(loadedType, 2, false);
  8132. CreateDefineVReg(loadedReg, insertPos + 2);
  8133. AllocInst(BeMCInstKind_Mov, loadedReg, origOperand, insertPos + 3);
  8134. IntroduceVRegs(scratchReg, mcBlock, insertPos + 2, insertPos + 3);
  8135. instIdx += 4;
  8136. origOperand = loadedReg;
  8137. }
  8138. replacedOpr = true;
  8139. isFinalRun = false;
  8140. if (debugging)
  8141. OutputDebugStrF(" BadOperand %d\n", rmInfo.mErrorVReg);
  8142. }
  8143. }
  8144. }
  8145. }
  8146. if (replacedOpr)
  8147. {
  8148. //instIdx--;
  8149. continue;
  8150. }
  8151. }
  8152. if (!inst->IsPsuedo())
  8153. {
  8154. if ((HasSymbolAddr(inst->mArg0)) && (inst->mKind != BeMCInstKind_Call))
  8155. {
  8156. ReplaceWithNewVReg(inst->mArg0, instIdx, true, false, true);
  8157. isFinalRun = false;
  8158. // if (debugging)
  8159. // OutputDebugStrF(" SymbolAddr0\n");
  8160. }
  8161. // We can only allow symbol addresses on a MOV where they can be made into a LEA
  8162. if ((HasSymbolAddr(inst->mArg1)) && (inst->mKind != BeMCInstKind_Mov))
  8163. {
  8164. ReplaceWithNewVReg(inst->mArg1, instIdx, true, false);
  8165. isFinalRun = false;
  8166. // if (debugging)
  8167. // OutputDebugStrF(" SymbolAddr1\n");
  8168. }
  8169. }
  8170. if (!inst->IsPsuedo())
  8171. {
  8172. if ((arg0.IsImmediate()) && (arg1) && (inst->IsCommutable()))
  8173. {
  8174. // Always make the immediate be the second arg
  8175. BF_SWAP(inst->mArg0, inst->mArg1);
  8176. BF_SWAP(arg0, arg1);
  8177. }
  8178. bool isIntMul = (inst->IsMul()) && (arg0Type->IsInt());
  8179. bool isIntDiv = (((inst->mKind == BeMCInstKind_IDiv) || (inst->mKind == BeMCInstKind_Div))) && (arg0Type->IsInt());
  8180. bool isMov_R64_IMM64 = ((inst->mKind == BeMCInstKind_Mov) && (inst->mArg0.IsNativeReg()) && (inst->mArg1.IsImmediateInt()));
  8181. bool doStdCheck = true;
  8182. if (isMov_R64_IMM64)
  8183. doStdCheck = false;
  8184. bool is3FormMul = false;
  8185. bool badOps = false;
  8186. // Two immediates are never allowed. This can happen in cases like '(a % 1) * 2'
  8187. if ((arg0.IsImmediate()) && (arg1.IsImmediate()))
  8188. {
  8189. badOps = true;
  8190. doStdCheck = true;
  8191. }
  8192. else if ((isIntMul) && (inst->mResult))
  8193. {
  8194. is3FormMul = true;
  8195. if (inst->mArg1.IsImmediateInt())
  8196. {
  8197. bool isLegal = true;
  8198. int64 immediateInt = inst->mArg1.GetImmediateInt();
  8199. if (inst->mKind == BeMCInstKind_Mul)
  8200. {
  8201. if (immediateInt > 0xFFFFFFFFLL)
  8202. isLegal = false;
  8203. }
  8204. else
  8205. {
  8206. if ((immediateInt < -0x80000000LL) || (immediateInt > 0x7FFFFFFF))
  8207. isLegal = false;
  8208. }
  8209. if (isLegal)
  8210. doStdCheck = false;
  8211. }
  8212. }
  8213. // Int 3-form mul does not follow these rules
  8214. if (doStdCheck)
  8215. {
  8216. bool isIncOrDec = false;
  8217. isIncOrDec = (((inst->mKind == BeMCInstKind_Add) || (inst->mKind == BeMCInstKind_Sub)) &&
  8218. (arg1.IsImmediateInt()) && (arg1.mImmediate == 1));
  8219. if ((!isIncOrDec) && (!isIntMul) && (!isIntDiv))
  8220. {
  8221. if ((arg0.MayBeMemory()) && (arg1.MayBeMemory()))
  8222. {
  8223. if (arg0 == arg1)
  8224. {
  8225. if (inst->mKind != BeMCInstKind_Mov)
  8226. {
  8227. badOps = true;
  8228. }
  8229. }
  8230. else
  8231. badOps = true;
  8232. }
  8233. //TODO: For what instructions was this true? CMP, MOV, ADD, etc... seems to have it
  8234. if ((arg1Type != NULL) && (arg1Type->IsFloatOrVector()))
  8235. {
  8236. // MOV is allowed on '<r/m>, <imm>'
  8237. if ((inst->mKind != BeMCInstKind_Mov) && (arg0.MayBeMemory()) && (arg1.IsImmediate()))
  8238. badOps = true;
  8239. }
  8240. }
  8241. // "MOV <r>, <imm64>" is the only instruction that allows an IMM64 immediate
  8242. if (((inst->mKind != BeMCInstKind_Mov) || (!arg0.IsNativeReg())) &&
  8243. (arg1.mKind == BeMCOperandKind_Immediate_i64))
  8244. {
  8245. if ((arg1.mImmediate < -0x80000000LL) || (arg1.mImmediate > 0x7FFFFFFF))
  8246. badOps = true;
  8247. }
  8248. if (badOps)
  8249. {
  8250. // On X64 we can never have an instruction where both args are memory so we create a short-lived scratch vreg
  8251. // and run another reg pass to generate register access
  8252. // From: <op> a, b
  8253. // To: @Def scratch
  8254. // mov scratch, b
  8255. // <op> a, scratch
  8256. auto targetType = GetType(inst->mArg0);
  8257. if ((targetType->IsFloat()) && (!inst->mResult) && (arg0.IsVReg()) && (arg1.IsImmediateFloat()))
  8258. {
  8259. auto vregInfo0 = GetVRegInfo(arg0);
  8260. if (!vregInfo0->mIsExpr)
  8261. {
  8262. // Convert from "<op> %reg0, %reg1"
  8263. // To
  8264. // Mov %vreg2<reg>, %vreg0
  8265. // <op> %vreg2<reg>, %vreg1
  8266. // Mov %vreg0, %vreg2<reg>
  8267. auto prevDest = inst->mArg0;
  8268. ReplaceWithNewVReg(inst->mArg0, instIdx, true);
  8269. AllocInst(BeMCInstKind_Mov, prevDest, inst->mArg0, instIdx + 1);
  8270. IntroduceVRegs(inst->mArg0, mcBlock, instIdx, instIdx + 2);
  8271. instIdx++;
  8272. isFinalRun = false;
  8273. continue;
  8274. }
  8275. }
  8276. if (!targetType->IsNonVectorComposite())
  8277. {
  8278. auto scratchType = GetType(inst->mArg1);
  8279. BeMCOperand scratchReg = AllocVirtualReg(scratchType, 2, true);
  8280. CreateDefineVReg(scratchReg, instIdx);
  8281. AllocInst(BeMCInstKind_Mov, scratchReg, inst->mArg1, instIdx + 1);
  8282. inst->mArg1 = scratchReg;
  8283. IntroduceVRegs(scratchReg, mcBlock, instIdx, instIdx + 2);
  8284. // Don't process the changed instructions until after another reg pass
  8285. instIdx += 2;
  8286. isFinalRun = false;
  8287. if (debugging)
  8288. OutputDebugStrF(" BadOps\n");
  8289. continue;
  8290. }
  8291. }
  8292. if (inst->mResult)
  8293. {
  8294. if (inst->mResult == inst->mArg0)
  8295. {
  8296. inst->mResult = BeMCOperand();
  8297. }
  8298. else if ((arg0Type->IsFloatOrVector()) && (!CouldBeReg(inst->mResult)))
  8299. {
  8300. // We need a REG on the dest for sure, so just create a scratch here, otherwise we end up
  8301. // requiring additional scratch vregs later
  8302. // FROM:
  8303. // %vreg0 = op %vreg1, %vreg2
  8304. // To:
  8305. // Mov %scratch<reg>, %vreg1
  8306. // Op %scratch<reg>, %vreg2
  8307. // Mov %result, %scatch<reg>
  8308. ReplaceWithNewVReg(inst->mArg0, instIdx, true);
  8309. AllocInst(BeMCInstKind_Mov, inst->mResult, inst->mArg0, instIdx + 1);
  8310. IntroduceVRegs(inst->mArg0, mcBlock, instIdx, instIdx + 2);
  8311. inst->mResult = BeMCOperand();
  8312. isFinalRun = false;
  8313. if (debugging)
  8314. OutputDebugStrF(" Float RegOnDest\n");
  8315. instIdx++;
  8316. continue;
  8317. }
  8318. else if ((inst->mResult == inst->mArg1) &&
  8319. ((inst->mKind == BeMCInstKind_Add) || (inst->mKind == BeMCInstKind_Mul) || (inst->mKind == BeMCInstKind_IMul)))
  8320. {
  8321. BF_SWAP(inst->mArg0, inst->mArg1);
  8322. BF_SWAP(arg0, arg1);
  8323. inst->mResult = BeMCOperand();
  8324. }
  8325. else
  8326. {
  8327. bool addCanBeLEA =
  8328. ((inst->mKind == BeMCInstKind_Sub) || (inst->mKind == BeMCInstKind_Add)) && (!inst->mDisableShortForm) &&
  8329. (arg0Type->mSize >= 2) && // Int8s don't have an LEA
  8330. (GetFixedOperand(inst->mResult).IsNativeReg()) && (arg0.IsNativeReg()) && (arg1.IsImmediateInt());
  8331. if (addCanBeLEA)
  8332. {
  8333. // LEA is add-only
  8334. if (inst->mKind == BeMCInstKind_Sub)
  8335. {
  8336. BF_ASSERT(inst->mArg1.IsImmediate());
  8337. inst->mKind = BeMCInstKind_Add;
  8338. inst->mArg1.mImmediate = -inst->mArg1.mImmediate;
  8339. }
  8340. }
  8341. else
  8342. {
  8343. bool handled = false;
  8344. // Check for exact quality or potential aliasing
  8345. if ((OperandsEqual(inst->mResult, inst->mArg1)) ||
  8346. ((HasLoad(inst->mResult)) && (HasLoad(inst->mArg1))))
  8347. {
  8348. // We need a scratch reg for this
  8349. ReplaceWithNewVReg(inst->mArg1, instIdx, true);
  8350. IntroduceVRegs(inst->mArg1, mcBlock, instIdx, instIdx + 2);
  8351. }
  8352. if (!handled)
  8353. {
  8354. // From: result = <op> a, b
  8355. // To: mov result, a
  8356. // <op> result, b
  8357. AllocInst(BeMCInstKind_Mov, inst->mResult, inst->mArg0, instIdx++);
  8358. inst->mArg0 = inst->mResult;
  8359. inst->mResult = BeMCOperand();
  8360. FixVRegInitFlags(inst, inst->mArg0);
  8361. if (debugging)
  8362. OutputDebugStrF(" Result\n");
  8363. isFinalRun = false;
  8364. }
  8365. }
  8366. continue;
  8367. }
  8368. }
  8369. }
  8370. if ((inst->mKind != BeMCInstKind_Cmp) &&
  8371. (inst->mKind != BeMCInstKind_CmpToBool) &&
  8372. (inst->mKind != BeMCInstKind_Mov))
  8373. {
  8374. if ((arg0Type != NULL) && (arg0Type->IsFloatOrVector()))
  8375. {
  8376. // <r/m>, <xmm> is not valid, <xmm>, <r/m>
  8377. if (!arg0.IsNativeReg())
  8378. {
  8379. if ((arg1.IsNativeReg()) && (inst->mArg1.IsVReg()) && (nextInst != NULL) && (nextInst->mLiveness != NULL) && (inst->IsCommutable()))
  8380. {
  8381. int underlyingVRegIdx = GetUnderlyingVReg(inst->mArg1.mVRegIdx);
  8382. if (!mLivenessContext.IsSet(nextInst->mLiveness, underlyingVRegIdx))
  8383. {
  8384. // If Arg1 is a temporary vreg, then we can use that as our destination so we don't need
  8385. // to create another temporary vreg
  8386. BF_SWAP(inst->mArg0, inst->mArg1);
  8387. AllocInst(BeMCInstKind_Mov, inst->mArg1, inst->mArg0, instIdx + 1);
  8388. continue;
  8389. }
  8390. }
  8391. BeMCOperand prevDest;
  8392. auto prevDestPtr = inst->GetDest();
  8393. if (prevDestPtr != NULL)
  8394. prevDest = *prevDestPtr;
  8395. ReplaceWithNewVReg(inst->mArg0, instIdx, true);
  8396. isFinalRun = false;
  8397. if (debugging)
  8398. OutputDebugStrF(" Float Arg0\n");
  8399. if (prevDest)
  8400. {
  8401. // This is a modifying instruction so we need to store the result afterward
  8402. AllocInst(BeMCInstKind_Mov, prevDest, inst->mArg0, instIdx + 1);
  8403. IntroduceVRegs(inst->mArg0, mcBlock, instIdx, instIdx + 2);
  8404. instIdx++; // Skip this instruction until next pass
  8405. }
  8406. else
  8407. {
  8408. IntroduceVRegs(inst->mArg0, mcBlock, instIdx, instIdx + 1);
  8409. }
  8410. continue;
  8411. }
  8412. }
  8413. }
  8414. if ((!inst->IsMov()) && (arg0.IsVReg()) && (arg0Type->IsFloatOrVector()))
  8415. {
  8416. BF_ASSERT(!inst->mResult);
  8417. // XMM instructions (besides MOVs) require native register destinations
  8418. if ((inst->IsCommutable()) && (arg1.IsNativeReg()) && (inst->mArg1.IsVReg()))
  8419. {
  8420. auto nextInst = mcBlock->mInstructions[instIdx + 1];
  8421. int underlyingVRegIdx = GetUnderlyingVReg(inst->mArg1.mVRegIdx);
  8422. if (!mLivenessContext.IsSet(nextInst->mLiveness, underlyingVRegIdx))
  8423. {
  8424. // Convert a "Mul %vreg0, %vreg1<reg>"
  8425. // To
  8426. // Mul %reg1<reg>, %vreg0
  8427. // Mov %vreg0, %reg1<reg>
  8428. // This only works if %reg1 dies after this instruction and this is
  8429. // cheaper than the more general case below
  8430. AllocInst(BeMCInstKind_Mov, inst->mArg0, inst->mArg1, instIdx++ + 1);
  8431. BF_SWAP(inst->mArg0, inst->mArg1);
  8432. BF_SWAP(arg0, arg1);
  8433. continue;
  8434. }
  8435. }
  8436. // Convert from "<op> %reg0, %reg1"
  8437. // To
  8438. // Mov %vreg2<reg>, %vreg0
  8439. // <op> %vreg2<reg>, %vreg1
  8440. // Mov %vreg0, %vreg2<reg>
  8441. auto prevDest = inst->mArg0;
  8442. ReplaceWithNewVReg(inst->mArg0, instIdx, true);
  8443. AllocInst(BeMCInstKind_Mov, prevDest, inst->mArg0, instIdx + 1);
  8444. IntroduceVRegs(inst->mArg0, mcBlock, instIdx, instIdx + 2);
  8445. instIdx++;
  8446. isFinalRun = false;
  8447. if (debugging)
  8448. OutputDebugStrF(" Float reg dest\n");
  8449. continue;
  8450. }
  8451. }
  8452. if (inst->mKind == BeMCInstKind_Call)
  8453. {
  8454. // Convert from
  8455. // Mov %reg0, <x>
  8456. // ..
  8457. // Call %reg0
  8458. // To "Call <x>"
  8459. // This is a common case for virtual dispatch where complex address expressions get actualized
  8460. // but then we end up with an 'extra' vreg
  8461. if (inst->mArg0.IsVReg())
  8462. {
  8463. auto vregInfo = GetVRegInfo(inst->mArg0);
  8464. if (vregInfo->mRefCount == 1)
  8465. {
  8466. for (int checkInstIdx = instIdx - 1; checkInstIdx >= 0; checkInstIdx--)
  8467. {
  8468. auto checkInst = mcBlock->mInstructions[checkInstIdx];
  8469. if ((checkInst->mKind == BeMCInstKind_Mov) && (checkInst->mArg0 == inst->mArg0))
  8470. {
  8471. // We can't extend down any ForceRegs, those must be confined to very short intervals
  8472. if (!HasForceRegs(checkInst->mArg1))
  8473. {
  8474. inst->mArg0 = checkInst->mArg1;
  8475. RemoveInst(mcBlock, checkInstIdx);
  8476. instIdx--;
  8477. continue;
  8478. }
  8479. }
  8480. }
  8481. }
  8482. }
  8483. }
  8484. switch (inst->mKind)
  8485. {
  8486. case BeMCInstKind_PreserveVolatiles:
  8487. regPreserveDepth++;
  8488. break;
  8489. case BeMCInstKind_RestoreVolatiles:
  8490. regPreserveDepth--;
  8491. break;
  8492. case BeMCInstKind_DefLoad:
  8493. {
  8494. SoftFail("DefLoad- DoLoads should have removed these");
  8495. }
  8496. break;
  8497. case BeMCInstKind_Def:
  8498. {
  8499. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  8500. // Legalize vreg exprs
  8501. if ((vregInfo->mRefCount > 0) && (vregInfo->mRelTo) && (!vregInfo->IsDirectRelToAny()))
  8502. {
  8503. if (vregInfo->mRelTo.mKind == BeMCOperandKind_SymbolAddr)
  8504. continue; // Just leave it
  8505. if (vregInfo->mRelTo.mKind == BeMCOperandKind_Symbol)
  8506. {
  8507. continue;
  8508. }
  8509. if (vregInfo->IsDirectRelTo())
  8510. {
  8511. if (vregInfo->mRelTo.IsVRegAny())
  8512. {
  8513. // Propagate change to the directRel
  8514. //auto itr = vregExprChangeSet.find(vregInfo->mRelTo.mVRegIdx);
  8515. //if (itr != vregExprChangeSet.end())
  8516. if (vregExprChangeSet.Contains(vregInfo->mRelTo.mVRegIdx))
  8517. {
  8518. BF_ASSERT(!isFinalRun);
  8519. vregExprChangeSet.Add(inst->mArg0.mVRegIdx);
  8520. isFinalRun = false;
  8521. }
  8522. }
  8523. }
  8524. else
  8525. {
  8526. if (vregInfo->mRelOffset)
  8527. {
  8528. auto relOfsType = GetType(vregInfo->mRelOffset);
  8529. // We can only do offsets by int64s
  8530. if (relOfsType->mTypeCode != BeTypeCode_Int64)
  8531. {
  8532. bool didExtend = false;
  8533. if (vregInfo->mRelOffset.IsVReg())
  8534. {
  8535. // First we try to resize the original vreg, which we are allowed to do
  8536. // if there's only one mov and the vreg dies by the end of the block
  8537. int relVRegIdx = vregInfo->mRelOffset.mVRegIdx;
  8538. bool foundDef = false;
  8539. BeMCInst* setInst = NULL;
  8540. for (int checkIdx = 0; checkIdx < (int)mcBlock->mInstructions.size(); checkIdx++)
  8541. {
  8542. auto checkInst = mcBlock->mInstructions[checkIdx];
  8543. if ((checkInst->mKind == BeMCInstKind_Def) && (checkInst->mArg0.mVRegIdx == relVRegIdx))
  8544. foundDef = true;
  8545. if (foundDef)
  8546. {
  8547. if ((checkInst->mKind == BeMCInstKind_Mov) && (checkInst->mArg0 == vregInfo->mRelOffset))
  8548. {
  8549. if (setInst != NULL)
  8550. break; // Only one set is allowed
  8551. setInst = checkInst;
  8552. }
  8553. else
  8554. {
  8555. // If we also use this reg for a non-offset reason then we can't just extend the size
  8556. bool hasNonOffsetRef = false;
  8557. auto operands = { &checkInst->mResult, &checkInst->mArg0, &checkInst->mArg1 };
  8558. for (auto operand : operands)
  8559. if (ContainsNonOffsetRef(*operand, vregInfo->mRelOffset))
  8560. hasNonOffsetRef = true;
  8561. if (hasNonOffsetRef)
  8562. {
  8563. break;
  8564. }
  8565. }
  8566. if (!mLivenessContext.IsSet(checkInst->mLiveness, relVRegIdx))
  8567. {
  8568. if (setInst != NULL)
  8569. {
  8570. didExtend = true;
  8571. auto relVRegInfo = mVRegInfo[relVRegIdx];
  8572. setInst->mKind = BeMCInstKind_MovSX;
  8573. relVRegInfo->mType = mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  8574. relVRegInfo->mAlign = relVRegInfo->mType->mAlign;
  8575. if (debugging)
  8576. OutputDebugStrF(" Def MovSX\n");
  8577. isFinalRun = false;
  8578. }
  8579. break;
  8580. }
  8581. }
  8582. }
  8583. }
  8584. if (!didExtend)
  8585. {
  8586. auto relOfs64 = AllocVirtualReg(mModule->mContext->GetPrimitiveType(BeTypeCode_Int64), 2);
  8587. CreateDefineVReg(relOfs64, instIdx);
  8588. AllocInst(BeMCInstKind_MovSX, relOfs64, vregInfo->mRelOffset, instIdx + 1);
  8589. IntroduceVRegs(inst->mArg0, mcBlock, instIdx, instIdx + 2);
  8590. instIdx += 2;
  8591. vregInfo->mRelOffset = relOfs64;
  8592. vregExprChangeSet.Add(relOfs64.mVRegIdx);
  8593. if (debugging)
  8594. OutputDebugStrF(" Def MovSX 2\n");
  8595. isFinalRun = false;
  8596. }
  8597. }
  8598. }
  8599. BeRMParamsInfo rmInfo;
  8600. GetRMParams(inst->mArg0, rmInfo);
  8601. bool isValid = rmInfo.mMode != BeMCRMMode_Invalid;
  8602. if (!isValid)
  8603. {
  8604. if (rmInfo.mErrorVReg == -1)
  8605. {
  8606. BF_ASSERT(!vregInfo->mRelOffset);
  8607. if (vregInfo->mType->IsPointer())
  8608. {
  8609. // This must be a cast like "(void*)gGlobalVar". We need to actualize the symbol address
  8610. // in a native register now.
  8611. vregExprChangeSet.Add(inst->mArg0.mVRegIdx);
  8612. AllocInst(BeMCInstKind_Mov, inst->mArg0, vregInfo->mRelTo, instIdx++ + 1);
  8613. vregInfo->mIsExpr = false;
  8614. vregInfo->mRelTo = BeMCOperand();
  8615. vregInfo->mForceReg = true;
  8616. CheckForce(vregInfo);
  8617. isFinalRun = false;
  8618. if (debugging)
  8619. OutputDebugStrF(" Symbol Addr\n");
  8620. break;
  8621. }
  8622. else
  8623. {
  8624. // This could be a special case like
  8625. // MOV (int64)floatVal, reg64
  8626. // For two-step loading of immediates into memory without using .rdata
  8627. continue;
  8628. }
  8629. }
  8630. else if (rmInfo.mErrorVReg == -2)
  8631. {
  8632. vregExprChangeSet.Add(inst->mArg0.mVRegIdx);
  8633. MarkInvalidRMRegs(inst->mArg0);
  8634. if (debugging)
  8635. OutputDebugStrF(" GetRMParams invalid indices\n");
  8636. isFinalRun = false;
  8637. break;
  8638. }
  8639. //if (vregExprChangeSet.find(errorVRegIdx) != vregExprChangeSet.end())
  8640. if (vregExprChangeSet.Contains(rmInfo.mErrorVReg))
  8641. {
  8642. // This means we have already modified some dependent vregs, so we may be legalized already.
  8643. // Wait till next iteration to determine that.
  8644. BF_ASSERT(!isFinalRun);
  8645. isValid = true; //
  8646. }
  8647. }
  8648. // The only valid form is [<reg>*<1/2/4/8>+<imm>]
  8649. // If we violate that then we have to break it up
  8650. //if ((vregInfo->mRelOffsetScale != 1) && (vregInfo->mRelOffsetScale != 2) && (vregInfo->mRelOffsetScale != 4) && (vregInfo->mRelOffsetScale != 8))
  8651. if ((!isValid) && (rmInfo.mErrorVReg == inst->mArg0.mVRegIdx))
  8652. {
  8653. vregExprChangeSet.Add(rmInfo.mErrorVReg);
  8654. if ((vregInfo->mRelOffsetScale != 1) && (vregInfo->mRelOffsetScale != 2) && (vregInfo->mRelOffsetScale != 4) && (vregInfo->mRelOffsetScale != 8))
  8655. {
  8656. auto relOffsetType = GetType(vregInfo->mRelOffset);
  8657. BeMCOperand scratchReg = AllocVirtualReg(relOffsetType, 2, true);
  8658. CreateDefineVReg(scratchReg, instIdx++);
  8659. auto newInst = AllocInst(BeMCInstKind_IMul, vregInfo->mRelOffset, BeMCOperand::FromImmediate(vregInfo->mRelOffsetScale), instIdx++);
  8660. newInst->mResult = scratchReg;
  8661. vregInfo->mRelOffset = scratchReg;
  8662. vregInfo->mRelOffsetScale = 1;
  8663. if (debugging)
  8664. OutputDebugStrF(" Invalid RM combo\n");
  8665. vregExprChangeSet.Add(scratchReg.mVRegIdx);
  8666. }
  8667. else if (!vregInfo->mRelTo.IsVRegAny())
  8668. {
  8669. vregInfo->mWantsExprActualize = true;
  8670. hasPendingActualizations = true;
  8671. }
  8672. isFinalRun = false;
  8673. }
  8674. else if (!isValid)
  8675. {
  8676. auto errorVRegInfo = mVRegInfo[rmInfo.mErrorVReg];
  8677. if ((errorVRegInfo->mIsExpr) && (!errorVRegInfo->IsDirectRelTo()))
  8678. {
  8679. errorVRegInfo->mWantsExprActualize = true;
  8680. hasPendingActualizations = true;
  8681. vregExprChangeSet.Add(rmInfo.mErrorVReg);
  8682. isFinalRun = false;
  8683. if (debugging)
  8684. OutputDebugStrF(" RM not valid, actualize\n");
  8685. }
  8686. else
  8687. {
  8688. // We don't want to have too many concurrent ForceReg vregs at once, since that causes too much register pressure and
  8689. // can cause register allocation to fail at the extreme end. The scratchReg adds another ForceReg for the lifetime
  8690. // of the def vreg, so if the def vreg doesn't immediately die and there are already too many ForceRegs active then
  8691. // we need to actualize ourselves
  8692. bool actualizeSelf = false;
  8693. if (instIdx < mcBlock->mInstructions.size() - 2)
  8694. {
  8695. auto checkInst = mcBlock->mInstructions[instIdx + 2];
  8696. if (mLivenessContext.IsSet(checkInst->mLiveness, inst->mArg0.mVRegIdx))
  8697. {
  8698. actualizeSelf = true;
  8699. }
  8700. }
  8701. else
  8702. {
  8703. actualizeSelf = true;
  8704. }
  8705. if (actualizeSelf)
  8706. {
  8707. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  8708. // DirectRel cannot actualize
  8709. if (vregInfo->IsDirectRelTo())
  8710. actualizeSelf = false;
  8711. }
  8712. if (actualizeSelf)
  8713. {
  8714. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  8715. vregInfo->mWantsExprActualize = true;
  8716. hasPendingActualizations = true;
  8717. }
  8718. else
  8719. {
  8720. // This may be a local variable that failed to be assigned to a reg, create a scratch local with a forced reg
  8721. auto errorVReg = BeMCOperand::FromVReg(rmInfo.mErrorVReg);
  8722. auto errorVRegLoad = BeMCOperand::ToLoad(errorVReg);
  8723. if ((vregInfo->mRelTo == errorVReg) || (vregInfo->mRelOffset == errorVReg))
  8724. {
  8725. auto scratchReg = AllocVirtualReg(errorVRegInfo->mType, 2, false);
  8726. auto scratchVRegInfo = mVRegInfo[scratchReg.mVRegIdx];
  8727. CreateDefineVReg(scratchReg, instIdx++);
  8728. AllocInst(BeMCInstKind_Mov, scratchReg, errorVReg, instIdx++);
  8729. isFinalRun = false;
  8730. if (debugging)
  8731. OutputDebugStrF(" RM failed, scratch vreg\n");
  8732. vregExprChangeSet.Add(scratchReg.mVRegIdx);
  8733. if (vregInfo->mRelTo == errorVReg)
  8734. {
  8735. scratchVRegInfo->mForceReg = true;
  8736. CheckForce(scratchVRegInfo);
  8737. vregInfo->mRelTo.mVRegIdx = scratchReg.mVRegIdx;
  8738. }
  8739. else if (vregInfo->mRelOffset == errorVReg)
  8740. {
  8741. scratchVRegInfo->mForceReg = true;
  8742. CheckForce(scratchVRegInfo);
  8743. vregInfo->mRelOffset.mVRegIdx = scratchReg.mVRegIdx;
  8744. }
  8745. }
  8746. else if ((vregInfo->mRelTo == errorVRegLoad) || (vregInfo->mRelOffset == errorVRegLoad))
  8747. {
  8748. auto scratchType = GetType(errorVRegLoad);
  8749. auto scratchReg = AllocVirtualReg(scratchType, 2, false);
  8750. auto scratchVRegInfo = mVRegInfo[scratchReg.mVRegIdx];
  8751. CreateDefineVReg(scratchReg, instIdx++);
  8752. AllocInst(BeMCInstKind_Mov, scratchReg, errorVRegLoad, instIdx++);
  8753. isFinalRun = false;
  8754. if (debugging)
  8755. OutputDebugStrF(" RM failed, scratch vreg\n");
  8756. vregExprChangeSet.Add(scratchReg.mVRegIdx);
  8757. if (vregInfo->mRelTo == errorVRegLoad)
  8758. {
  8759. scratchVRegInfo->mForceReg = true;
  8760. CheckForce(scratchVRegInfo);
  8761. vregInfo->mRelTo = scratchReg;
  8762. }
  8763. else if (vregInfo->mRelOffset == errorVRegLoad)
  8764. {
  8765. scratchVRegInfo->mForceReg = true;
  8766. CheckForce(scratchVRegInfo);
  8767. vregInfo->mRelOffset = scratchReg;
  8768. }
  8769. }
  8770. else
  8771. {
  8772. // This should be impossible - a previous def for an inner expr should have caught this case
  8773. if ((vregExprChangeSet.IsEmpty()) && (!hasPendingActualizations))
  8774. {
  8775. SoftFail("Error legalizing vreg expression", NULL);
  8776. }
  8777. }
  8778. }
  8779. }
  8780. }
  8781. }
  8782. }
  8783. if ((vregInfo->mDbgVariable != NULL) && (vregInfo->mDbgVariable->mPendingInitType != BfIRInitType_NotNeeded))
  8784. pendingInitKind = vregInfo->mDbgVariable->mPendingInitType;
  8785. }
  8786. break;
  8787. case BeMCInstKind_DbgDecl:
  8788. {
  8789. bool isInvalid = false;
  8790. int vregIdx = inst->mArg0.mVRegIdx;
  8791. auto vregInfo = mVRegInfo[vregIdx];
  8792. auto dbgVar = vregInfo->mDbgVariable;
  8793. BeRMParamsInfo rmInfo;
  8794. if (dbgVar->mIsValue)
  8795. GetRMParams(inst->mArg0, rmInfo);
  8796. else
  8797. {
  8798. if (inst->mArg0.mKind == BeMCOperandKind_VRegAddr)
  8799. GetRMParams(BeMCOperand::ToLoad(inst->mArg0), rmInfo);
  8800. else
  8801. {
  8802. GetRMParams(inst->mArg0, rmInfo);
  8803. if ((rmInfo.mMode != BeMCRMMode_Direct) && (rmInfo.mBScale != 1) && (rmInfo.mDisp != 0))
  8804. isInvalid = true;
  8805. }
  8806. }
  8807. if (rmInfo.mMode == BeMCRMMode_Invalid)
  8808. {
  8809. if (vregInfo->mType->mSize != 0)
  8810. {
  8811. isInvalid = true;
  8812. }
  8813. }
  8814. else if (rmInfo.mMode == BeMCRMMode_Direct)
  8815. {
  8816. if ((rmInfo.mRegB != X64Reg_None) || (rmInfo.mDisp != 0) || (rmInfo.mBScale != 1))
  8817. {
  8818. if ((vregInfo->mIsExpr) && (vregInfo->mRelTo.mKind == BeMCOperandKind_Immediate_i64))
  8819. {
  8820. // Zero-sized alloc, allow
  8821. }
  8822. else
  8823. {
  8824. isInvalid = true;
  8825. }
  8826. }
  8827. }
  8828. else if (rmInfo.mMode == BeMCRMMode_Deref)
  8829. {
  8830. if ((rmInfo.mRegB != X64Reg_None) || (rmInfo.mBScale != 1))
  8831. {
  8832. isInvalid = true;
  8833. }
  8834. }
  8835. if (isInvalid)
  8836. {
  8837. isFinalRun = false;
  8838. /*ReplaceWithNewVReg(inst->mArg0, instIdx, true, false);
  8839. // The debug value is not representable, so copy it. This can only occur within mixins where we are pointing
  8840. // to composed values
  8841. BF_ASSERT(inst->mArg0.mKind == BeMCOperandKind_VReg);
  8842. inst->mArg0.mKind = BeMCOperandKind_VRegAddr;
  8843. dbgVar->mIsValue = false;
  8844. auto newVRegInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  8845. vregInfo->mDbgVariable = NULL;
  8846. newVRegInfo->mDbgVariable = dbgVar;*/
  8847. // The debug value is not representable
  8848. inst->mArg0.mKind = BeMCOperandKind_VReg;
  8849. vregInfo->mWantsExprActualize = true;
  8850. hasPendingActualizations = true;
  8851. vregExprChangeSet.Add(inst->mArg0.mVRegIdx);
  8852. isFinalRun = false;
  8853. break;
  8854. }
  8855. if (!mVRegInitializedContext.IsSet(inst->mVRegsInitialized, vregIdx))
  8856. {
  8857. if ((dbgVar->mPendingInitType != BfIRInitType_NotNeeded) && (dbgVar->mPendingInitType != BfIRInitType_NotNeeded_AliveOnDecl))
  8858. {
  8859. Fail("Shouldn't happen here anymore");
  8860. /*pendingInitKind = dbgVar->mPendingInitType;
  8861. // We don't need dynLife anymore since we're explicitly writing this
  8862. vregInfo->mHasDynLife = false;
  8863. AllocInst(BeMCInstKind_Def, inst->mArg0, instIdx);
  8864. // We don't definitively need this, it's just to make sure this doesn't cause a liveness error
  8865. isFinalRun = false;*/
  8866. }
  8867. }
  8868. }
  8869. break;
  8870. case BeMCInstKind_MemCpy:
  8871. {
  8872. if (inst->mArg1)
  8873. {
  8874. BF_ASSERT(inst->mArg1.mKind == BeMCOperandKind_VRegPair);
  8875. int* vregIndices[] = { &inst->mArg1.mVRegPair.mVRegIdx0, &inst->mArg1.mVRegPair.mVRegIdx1 };
  8876. for (int* argIdxPtr : vregIndices)
  8877. {
  8878. auto mcArg = BeMCOperand::FromEncoded(*argIdxPtr);
  8879. BeRMParamsInfo rmInfo;
  8880. GetRMParams(mcArg, rmInfo);
  8881. if ((rmInfo.mMode != BeMCRMMode_Direct) || (rmInfo.mRegB != X64Reg_None) || (rmInfo.mRegA == X64Reg_R11))
  8882. {
  8883. BeMCOperand scratchReg = AllocVirtualReg(GetType(mcArg), 2, true);
  8884. auto vregInfo = GetVRegInfo(scratchReg);
  8885. vregInfo->mDisableR11 = true;
  8886. int insertPos = FindSafeInstInsertPos(instIdx);
  8887. CreateDefineVReg(scratchReg, insertPos);
  8888. AllocInst(BeMCInstKind_Mov, scratchReg, mcArg, insertPos + 1);
  8889. *argIdxPtr = scratchReg.mVRegIdx;
  8890. instIdx += 2;
  8891. isFinalRun = false;
  8892. if (debugging)
  8893. OutputDebugStrF(" MemCpy\n");
  8894. }
  8895. }
  8896. }
  8897. }
  8898. break;
  8899. case BeMCInstKind_MemSet:
  8900. {
  8901. if (inst->mArg1)
  8902. {
  8903. BeRMParamsInfo rmInfo;
  8904. GetRMParams(inst->mArg1, rmInfo);
  8905. if ((rmInfo.mMode != BeMCRMMode_Direct) || (rmInfo.mRegB != X64Reg_None) || (rmInfo.mRegA == X64Reg_R11) || (rmInfo.mDisp != 0))
  8906. {
  8907. BeMCOperand scratchReg = AllocVirtualReg(GetType(inst->mArg1), 2, true);
  8908. auto vregInfo = GetVRegInfo(scratchReg);
  8909. vregInfo->mDisableR11 = true;
  8910. int insertPos = FindSafeInstInsertPos(instIdx);
  8911. CreateDefineVReg(scratchReg, insertPos);
  8912. AllocInst(BeMCInstKind_Mov, scratchReg, inst->mArg1, insertPos + 1);
  8913. inst->mArg1 = scratchReg;
  8914. instIdx += 2;
  8915. isFinalRun = false;
  8916. if (debugging)
  8917. OutputDebugStrF(" MemSet\n");
  8918. }
  8919. }
  8920. }
  8921. break;
  8922. case BeMCInstKind_Neg:
  8923. {
  8924. if (arg0Type->IsFloat())
  8925. {
  8926. inst->mKind = BeMCInstKind_Xor;
  8927. if (arg0Type->mTypeCode == BeTypeCode_Float)
  8928. {
  8929. inst->mArg1.mKind = BeMCOperandKind_Immediate_f32_Packed128;
  8930. inst->mArg1.mImmF32 = -0.0;
  8931. }
  8932. else
  8933. {
  8934. inst->mArg1.mKind = BeMCOperandKind_Immediate_f64_Packed128;
  8935. inst->mArg1.mImmF64 = -0.0;
  8936. }
  8937. }
  8938. }
  8939. break;
  8940. case BeMCInstKind_Div:
  8941. case BeMCInstKind_IDiv:
  8942. case BeMCInstKind_Rem:
  8943. case BeMCInstKind_IRem:
  8944. {
  8945. // Unsigned div and rem can be implemented with bitwise operations
  8946. // Negative values cannot (at least following C standards, because of rounding issues)
  8947. if ((inst->mKind == BeMCInstKind_Div) || (inst->mKind == BeMCInstKind_Rem))
  8948. {
  8949. if ((!inst->mResult) && (inst->mArg1.IsImmediateInt()) && (IsPowerOfTwo(inst->mArg1.mImmediate)))
  8950. {
  8951. if (inst->mKind == BeMCInstKind_Div)
  8952. {
  8953. int64 divVal = inst->mArg1.mImmediate;
  8954. int shiftCount = 0;
  8955. while (divVal > 1)
  8956. {
  8957. shiftCount++;
  8958. divVal >>= 1;
  8959. }
  8960. inst->mKind = BeMCInstKind_Shr;
  8961. inst->mArg1 = BeMCOperand::FromImmediate(shiftCount);
  8962. isFinalRun = false;
  8963. if (debugging)
  8964. OutputDebugStrF(" Div SHR\n");
  8965. break;
  8966. }
  8967. else if (inst->mKind == BeMCInstKind_Rem)
  8968. {
  8969. inst->mKind = BeMCInstKind_And;
  8970. inst->mArg1 = BeMCOperand::FromImmediate(inst->mArg1.mImmediate - 1);
  8971. isFinalRun = false;
  8972. if (debugging)
  8973. OutputDebugStrF(" Div REM\n");
  8974. break;
  8975. }
  8976. }
  8977. }
  8978. if (arg1.IsImmediate())
  8979. {
  8980. // Oops, must be 'rm'
  8981. ReplaceWithNewVReg(inst->mArg1, instIdx, true, false);
  8982. isFinalRun = false;
  8983. if (debugging)
  8984. OutputDebugStrF(" Div/Rem not RM\n");
  8985. }
  8986. auto arg0Type = GetType(arg0);
  8987. if (arg0Type->IsInt())
  8988. {
  8989. // We can't allow division by RDX because we need RAX:RDX for the dividend
  8990. bool needRegDisable = false;
  8991. std::function<void(BeMCOperand)> _CheckReg = [&](BeMCOperand operand)
  8992. {
  8993. if (!operand)
  8994. return;
  8995. if (operand.mKind == BeMCOperandKind_NativeReg)
  8996. {
  8997. auto divisorReg = GetFullRegister(operand.mReg);
  8998. if ((divisorReg == X64Reg_RDX) || (divisorReg == X64Reg_RAX))
  8999. needRegDisable = true;
  9000. }
  9001. auto vregInfo = GetVRegInfo(operand);
  9002. if (vregInfo != NULL)
  9003. {
  9004. auto divisorReg = GetFullRegister(vregInfo->mReg);
  9005. if ((divisorReg == X64Reg_RDX) || (divisorReg == X64Reg_RAX))
  9006. needRegDisable = true;
  9007. _CheckReg(vregInfo->mRelTo);
  9008. _CheckReg(vregInfo->mRelOffset);
  9009. if ((needRegDisable) &&
  9010. ((!vregInfo->mDisableRAX) || (!vregInfo->mDisableRDX)))
  9011. {
  9012. vregInfo->mDisableRAX = true;
  9013. vregInfo->mDisableRDX = true;
  9014. isFinalRun = false;
  9015. if (debugging)
  9016. OutputDebugStrF(" Div/Rem invalid reg\n");
  9017. }
  9018. }
  9019. };
  9020. needRegDisable = false;
  9021. _CheckReg(arg1);
  9022. if ((instIdx > 0) && (regPreserveDepth > 0))
  9023. {
  9024. auto prevInst = mcBlock->mInstructions[instIdx - 1];
  9025. if (prevInst->mKind == BeMCInstKind_Mov)
  9026. {
  9027. // Check replaced 'mov' (which is inside PreserveVolatile sections)
  9028. needRegDisable = false;
  9029. _CheckReg(prevInst->mArg1);
  9030. }
  9031. }
  9032. ///
  9033. auto checkArg1 = arg1;
  9034. if (checkArg1.mKind == BeMCOperandKind_VRegLoad)
  9035. {
  9036. // Handle '*vreg<RAX>' case
  9037. checkArg1.mKind = BeMCOperandKind_VReg;
  9038. checkArg1 = GetFixedOperand(checkArg1);
  9039. }
  9040. if (checkArg1.mKind == BeMCOperandKind_NativeReg)
  9041. {
  9042. // We can't allow division by RDX because we need RAX:RDX for the dividend
  9043. auto divisorReg = GetFullRegister(checkArg1.mReg);
  9044. if ((checkArg1.IsNativeReg()) &&
  9045. ((divisorReg == X64Reg_RDX) || (divisorReg == X64Reg_RAX)))
  9046. {
  9047. BF_ASSERT(inst->mArg1.IsVRegAny());
  9048. int vregIdx = GetUnderlyingVReg(inst->mArg1.mVRegIdx);
  9049. auto vregInfo = mVRegInfo[vregIdx];
  9050. if (vregInfo != NULL)
  9051. {
  9052. vregInfo->mDisableRAX = true;
  9053. vregInfo->mDisableRDX = true;
  9054. isFinalRun = false;
  9055. if (debugging)
  9056. OutputDebugStrF(" Div/Rem invalid reg\n");
  9057. }
  9058. }
  9059. }
  9060. else
  9061. {
  9062. auto vregInfo = GetVRegInfo(arg1);
  9063. if ((vregInfo != NULL) && (vregInfo->mIsExpr))
  9064. {
  9065. ReplaceWithNewVReg(inst->mArg1, instIdx, true);
  9066. }
  9067. }
  9068. // DIV/IDIV can only operate on the RDX:RAX pair, except for i8 divide which just uses AX
  9069. bool isRegADividend = (arg0.mReg == X64Reg_RAX) || (arg0.mReg == X64Reg_EAX) || (arg0.mReg == X64Reg_AX) || (arg0.mReg == X64Reg_AL);
  9070. if ((!arg0.IsNativeReg()) || (!isRegADividend) ||
  9071. (inst->mKind == BeMCInstKind_Rem) || (inst->mKind == BeMCInstKind_IRem))
  9072. {
  9073. bool preserveRDX = (arg0Type->mSize != 1) && (regPreserveDepth == 0);
  9074. auto mcScratch = BeMCOperand::FromReg(ResizeRegister(X64Reg_RAX, arg0Type));
  9075. BF_ASSERT(!inst->mResult);
  9076. BeMCInst* preserveRAXInst = AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX), instIdx++);
  9077. BeMCInst* preserveRDXInst = NULL;
  9078. if (preserveRDX)
  9079. preserveRDXInst = AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RDX), instIdx++);
  9080. AllocInst(BeMCInstKind_Mov, mcScratch, inst->mArg0, instIdx++);
  9081. if ((inst->mKind == BeMCInstKind_Rem) || (inst->mKind == BeMCInstKind_IRem))
  9082. {
  9083. if (inst->mKind == BeMCInstKind_Rem)
  9084. inst->mKind = BeMCInstKind_Div;
  9085. else
  9086. inst->mKind = BeMCInstKind_IDiv;
  9087. BeMCOperand mcRemaindier;
  9088. if (arg0Type->mSize == 1)
  9089. {
  9090. mcRemaindier = BeMCOperand::FromReg(ResizeRegister(X64Reg_AL, arg0Type));
  9091. preserveRAXInst->mArg1 = inst->mArg0; // RAX preserve elision exception
  9092. DisableRegister(inst->mArg0, X64Reg_SIL); // Disable Hi8
  9093. DisableRegister(inst->mArg0, X64Reg_RAX);
  9094. AllocInst(BeMCInstKind_Shr, BeMCOperand::FromReg(X64Reg_AX), BeMCOperand::FromImmediate(8), instIdx++ + 1);
  9095. AllocInst(BeMCInstKind_Mov, inst->mArg0, mcRemaindier, instIdx++ + 1);
  9096. }
  9097. else
  9098. {
  9099. mcRemaindier = BeMCOperand::FromReg(ResizeRegister(X64Reg_RDX, arg0Type));
  9100. preserveRDXInst->mArg1 = inst->mArg0; // RDX preserve elision exception
  9101. // This is to avoid overlap with PreserveRAX
  9102. DisableRegister(inst->mArg0, X64Reg_RAX);
  9103. if (preserveRDX)
  9104. DisableRegister(inst->mArg0, X64Reg_RDX);
  9105. AllocInst(BeMCInstKind_Mov, inst->mArg0, mcRemaindier, instIdx++ + 1);
  9106. }
  9107. }
  9108. else
  9109. {
  9110. preserveRAXInst->mArg1 = inst->mArg0; // RAX preserve elision exception
  9111. AllocInst(BeMCInstKind_Mov, inst->mArg0, mcScratch, instIdx++ + 1);
  9112. // This is to avoid overlap with PreserveRAX
  9113. DisableRegister(inst->mArg0, X64Reg_RAX);
  9114. if (preserveRDX)
  9115. DisableRegister(inst->mArg0, X64Reg_RDX);
  9116. }
  9117. inst->mArg0 = mcScratch;
  9118. if (preserveRDX)
  9119. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RDX), instIdx++ + 1);
  9120. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX), instIdx++ + 1);
  9121. isFinalRun = false;
  9122. if (debugging)
  9123. OutputDebugStrF(" Div/Rem Setup\n");
  9124. }
  9125. else
  9126. {
  9127. if (regPreserveDepth == 0)
  9128. {
  9129. if (arg0Type->mSize != 1)
  9130. {
  9131. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RDX), instIdx++);
  9132. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RDX), instIdx++ + 1);
  9133. isFinalRun = false; // Reassign regs
  9134. if (debugging)
  9135. OutputDebugStrF(" Div/Rem Preserve\n");
  9136. }
  9137. }
  9138. }
  9139. }
  9140. else if (inst->mKind == BeMCInstKind_IRem)
  9141. {
  9142. SetAndRestoreValue<int*> insertPtr(mInsertInstIdxRef, &instIdx);
  9143. SizedArray<BeMCOperand, 3> args = { inst->mArg0, inst->mArg1 };
  9144. auto mcFunc = BeMCOperand::FromSymbolAddr(mCOFFObject->GetSymbolRef((arg0Type->mTypeCode == BeTypeCode_Double) ? "fmod" : "fmodf")->mIdx);
  9145. auto fmodVal = CreateCall(mcFunc, args, arg0Type);
  9146. inst->mKind = BeMCInstKind_Mov;
  9147. inst->mResult = BeMCOperand();
  9148. inst->mArg1 = fmodVal;
  9149. isFinalRun = false;
  9150. if (debugging)
  9151. OutputDebugStrF(" FMod\n");
  9152. }
  9153. }
  9154. break;
  9155. case BeMCInstKind_Mul:
  9156. case BeMCInstKind_IMul:
  9157. {
  9158. bool handled = false;
  9159. if ((arg0Type->mSize == 1) && (arg0Type->IsIntable()))
  9160. {
  9161. if ((!arg0.IsNativeReg()) || (arg0.mReg != X64Reg_AL) || (inst->mResult))
  9162. {
  9163. auto srcVRegInfo = GetVRegInfo(inst->mArg0);
  9164. // Int8 multiplies can only be done on AL
  9165. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX), instIdx++);
  9166. DisableRegister(inst->mArg0, X64Reg_RAX);
  9167. DisableRegister(inst->mArg1, X64Reg_RAX);
  9168. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_AL), inst->mArg1, instIdx++);
  9169. AllocInst(BeMCInstKind_Shl, BeMCOperand::FromReg(X64Reg_AX), BeMCOperand::FromImmediate(8), instIdx++);
  9170. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_AL), inst->mArg0, instIdx++);
  9171. AllocInst(BeMCInstKind_Mov, inst->mResult ? inst->mResult : inst->mArg0, BeMCOperand::FromReg(X64Reg_AL), instIdx++ + 1);
  9172. inst->mArg0 = BeMCOperand::FromReg(X64Reg_AL);
  9173. inst->mArg1 = BeMCOperand::FromReg(X64Reg_AH);
  9174. inst->mResult = BeMCOperand();
  9175. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX), instIdx++ + 1);
  9176. isFinalRun = false;
  9177. if (debugging)
  9178. OutputDebugStrF(" Mul Size 1\n");
  9179. break;
  9180. }
  9181. if (inst->mArg1.IsImmediateInt())
  9182. {
  9183. ReplaceWithNewVReg(inst->mArg1, instIdx, true, false);
  9184. }
  9185. BF_ASSERT(!inst->mResult);
  9186. handled = true;
  9187. }
  9188. else if ((inst->mKind == BeMCInstKind_Mul) && (arg0Type->IsIntable()))
  9189. {
  9190. auto wantReg0 = ResizeRegister(X64Reg_RAX, arg0Type->mSize);
  9191. if ((!arg0.IsNativeReg()) || (arg0.mReg != wantReg0) || (inst->mResult))
  9192. {
  9193. auto srcVRegInfo = GetVRegInfo(inst->mArg0);
  9194. // unsigned multiplies can only be done on AX/EAX/RAX
  9195. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX), instIdx++);
  9196. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RDX), instIdx++);
  9197. DisableRegister(inst->mArg0, X64Reg_RAX);
  9198. DisableRegister(inst->mArg0, X64Reg_RDX);
  9199. DisableRegister(inst->mArg1, X64Reg_RAX);
  9200. DisableRegister(inst->mArg1, X64Reg_RDX);
  9201. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(wantReg0), inst->mArg0, instIdx++);
  9202. AllocInst(BeMCInstKind_Mov, inst->mResult ? inst->mResult : inst->mArg0, BeMCOperand::FromReg(wantReg0), instIdx++ + 1);
  9203. inst->mArg0 = BeMCOperand::FromReg(wantReg0);
  9204. inst->mResult = BeMCOperand();
  9205. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RDX), instIdx++ + 1);
  9206. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX), instIdx++ + 1);
  9207. isFinalRun = false;
  9208. break;
  9209. }
  9210. if (inst->mArg1.IsImmediateInt())
  9211. {
  9212. ReplaceWithNewVReg(inst->mArg1, instIdx, true, false);
  9213. }
  9214. BF_ASSERT(!inst->mResult);
  9215. handled = true;
  9216. }
  9217. if (inst->mArg0.IsNativeReg())
  9218. {
  9219. auto vregInfo1 = GetVRegInfo(inst->mArg1);
  9220. if (vregInfo1 != NULL)
  9221. {
  9222. auto arg1 = GetFixedOperand(inst->mArg1);
  9223. if ((arg1.IsNativeReg()) && (inst->mArg0.mReg == arg1.mReg) &&
  9224. ((ResizeRegister(arg1.mReg, 8) == X64Reg_RAX) || (ResizeRegister(arg1.mReg, 8) == X64Reg_RDX)))
  9225. {
  9226. DisableRegister(inst->mArg1, X64Reg_RAX);
  9227. DisableRegister(inst->mArg1, X64Reg_RDX);
  9228. isFinalRun = false;
  9229. }
  9230. }
  9231. }
  9232. if (!handled)
  9233. {
  9234. if (inst->mResult)
  9235. {
  9236. // The 3-op form of MUL must be in "reg, r/m64, imm" form
  9237. if (!inst->mArg1.IsImmediateInt())
  9238. {
  9239. SoftFail("Not supported");
  9240. //isFinalRun = false;
  9241. break;
  9242. }
  9243. }
  9244. // Convert from %vreg0 = mul %vreg1, %vreg2
  9245. // To: %scratch = mul %vreg1, %vreg2
  9246. // mov %reg0, %scratch
  9247. if (inst->mResult)
  9248. {
  9249. bool isLegal = true;
  9250. int64 immediateInt = inst->mArg1.GetImmediateInt();
  9251. // if (inst->mKind == BeMCInstKind_Mul)
  9252. // {
  9253. // if (immediateInt > 0xFFFFFFFFLL)
  9254. // isLegal = false;
  9255. // }
  9256. // else
  9257. // {
  9258. // if ((immediateInt < -0x80000000LL) || (immediateInt > 0x7FFFFFFF))
  9259. // isLegal = false;
  9260. // }
  9261. if (!GetFixedOperand(inst->mResult).IsNativeReg())
  9262. {
  9263. BeMCOperand scratchReg = AllocVirtualReg(GetType(inst->mResult), 2, true);
  9264. CreateDefineVReg(scratchReg, instIdx);
  9265. AllocInst(BeMCInstKind_Mov, inst->mResult, scratchReg, instIdx + 2);
  9266. inst->mResult = scratchReg;
  9267. if (debugging)
  9268. OutputDebugStrF(" Mul 3-form not reg\n");
  9269. isFinalRun = false;
  9270. instIdx += 2;
  9271. break;
  9272. }
  9273. }
  9274. else
  9275. {
  9276. if (!GetFixedOperand(inst->mArg0).IsNativeReg())
  9277. {
  9278. auto prevArg0 = inst->mArg0;
  9279. ReplaceWithNewVReg(inst->mArg0, instIdx, true);
  9280. AllocInst(BeMCInstKind_Mov, prevArg0, inst->mArg0, instIdx + 1);
  9281. isFinalRun = false;
  9282. if (debugging)
  9283. OutputDebugStrF(" Mul not reg\n");
  9284. instIdx++;
  9285. break;
  9286. }
  9287. }
  9288. }
  9289. }
  9290. // Fallthrough
  9291. case BeMCInstKind_Add:
  9292. case BeMCInstKind_Sub:
  9293. {
  9294. }
  9295. break;
  9296. case BeMCInstKind_Shl:
  9297. case BeMCInstKind_Shr:
  9298. case BeMCInstKind_Sar:
  9299. {
  9300. if ((!inst->mArg1.IsNativeReg()) && (!inst->mArg1.IsImmediateInt()))
  9301. {
  9302. auto mcShift = inst->mArg1;
  9303. auto shiftType = GetType(mcShift);
  9304. BF_ASSERT(shiftType->IsInt());
  9305. if (shiftType->mSize != 1)
  9306. {
  9307. auto mcShift8 = AllocVirtualReg(mModule->mContext->GetPrimitiveType(BeTypeCode_Int8), 2);
  9308. CreateDefineVReg(mcShift8, instIdx++);
  9309. auto vregInfo = GetVRegInfo(mcShift8);
  9310. vregInfo->mIsExpr = true;
  9311. vregInfo->mRelTo = mcShift;
  9312. mcShift = mcShift8;
  9313. }
  9314. // The only non-constant shift is by 'CL'
  9315. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RCX), instIdx++);
  9316. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_CL), mcShift, instIdx++);
  9317. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RCX), instIdx++ + 1);
  9318. inst->mArg1 = BeMCOperand::FromReg(X64Reg_RCX);
  9319. isFinalRun = false;
  9320. if (debugging)
  9321. OutputDebugStrF(" Shift by CL\n");
  9322. continue;
  9323. }
  9324. if (inst->mArg1.IsNativeReg())
  9325. {
  9326. BF_ASSERT(inst->mArg1.mReg == X64Reg_RCX);
  9327. X64CPURegister regs[2] = { X64Reg_None, X64Reg_None };
  9328. GetUsedRegs(inst->mArg0, regs[0], regs[1]);
  9329. if ((regs[0] == X64Reg_RCX) || (regs[1] == X64Reg_RCX))
  9330. {
  9331. // Bad reg!
  9332. auto mcTarget = inst->mArg0;
  9333. int insertIdx = instIdx - 1;
  9334. ReplaceWithNewVReg(inst->mArg0, insertIdx, true);
  9335. BF_ASSERT(mcBlock->mInstructions[instIdx + 3]->mKind == BeMCInstKind_RestoreVolatiles);
  9336. AllocInst(BeMCInstKind_Mov, mcTarget, inst->mArg0, instIdx + 4);
  9337. }
  9338. }
  9339. }
  9340. break;
  9341. case BeMCInstKind_Cmp:
  9342. {
  9343. bool needSwap = false;
  9344. // Cmp <imm>, <r/m> is not legal, so we need to swap LHS/RHS, which means also modifying the instruction that uses the result of the cmp
  9345. if (arg0.IsImmediate())
  9346. needSwap = true;
  9347. if (arg0Type->IsFloat())
  9348. {
  9349. if (!arg0.IsNativeReg())
  9350. {
  9351. // We need an <xmm> for reg0. We're not allowed to reorder SwapCmpSides due to NaN handling
  9352. ReplaceWithNewVReg(inst->mArg0, instIdx, true, true);
  9353. isFinalRun = false;
  9354. break;
  9355. }
  9356. // Cmp <r/m>, <xmm> is not valid, only Cmp <xmm>, <r/m>
  9357. if ((!arg0.IsNativeReg()) && (arg1.IsNativeReg()))
  9358. {
  9359. needSwap = true;
  9360. }
  9361. else
  9362. {
  9363. if (!arg0.IsNativeReg())
  9364. {
  9365. SoftFail("xmm reg required");
  9366. }
  9367. }
  9368. }
  9369. if (needSwap)
  9370. {
  9371. bool hasConstResult = false;
  9372. int constResult = 0;
  9373. if (inst->mArg1.IsImmediate())
  9374. {
  9375. //constResult = inst->mArg0.mImmediate - inst->mArg1.mImmediate;
  9376. // Should have been handled in the frontend
  9377. ReplaceWithNewVReg(inst->mArg0, instIdx, true);
  9378. break;
  9379. }
  9380. int checkInstIdx = instIdx + 1;
  9381. auto checkBlock = mcBlock;
  9382. for (; checkInstIdx < (int)checkBlock->mInstructions.size(); checkInstIdx++)
  9383. {
  9384. auto checkInst = checkBlock->mInstructions[checkInstIdx];
  9385. if ((checkInst->mKind == BeMCInstKind_CondBr) || (checkInst->mKind == BeMCInstKind_CmpToBool))
  9386. {
  9387. if (constResult)
  9388. {
  9389. NotImpl();
  9390. //mcBlock->RemoveInst(instIdx);
  9391. //instIdx--;
  9392. }
  9393. else
  9394. {
  9395. BeMCOperand& cmpArg = (checkInst->mKind == BeMCInstKind_CondBr) ? checkInst->mArg1 : checkInst->mArg0;
  9396. BF_SWAP(inst->mArg0, inst->mArg1);
  9397. BF_ASSERT(cmpArg.mKind == BeMCOperandKind_CmpKind);
  9398. cmpArg.mCmpKind = BeModule::SwapCmpSides(cmpArg.mCmpKind);
  9399. }
  9400. break;
  9401. }
  9402. else if (checkInst->mKind == BeMCInstKind_Br)
  9403. {
  9404. // Sometimes the matching use is in another block
  9405. FindTarget(checkInst->mArg0, checkBlock, checkInstIdx);
  9406. }
  9407. else if (
  9408. (checkInst->mKind != BeMCInstKind_Def) &&
  9409. (checkInst->mKind != BeMCInstKind_DbgDecl) &&
  9410. (checkInst->mKind != BeMCInstKind_ValueScopeSoftEnd) &&
  9411. (checkInst->mKind != BeMCInstKind_ValueScopeHardEnd) &&
  9412. (checkInst->mKind != BeMCInstKind_LifetimeEnd))
  9413. {
  9414. SoftFail("Malformed");
  9415. }
  9416. }
  9417. break;
  9418. }
  9419. }
  9420. break;
  9421. case BeMCInstKind_CmpToBool:
  9422. {
  9423. if (inst->mResult.IsVRegAny())
  9424. {
  9425. if (!mVRegInitializedContext.IsSet(mCurVRegsInit, inst->mResult.mVRegIdx))
  9426. {
  9427. SizedArray<int, 1> addVec = { inst->mResult.mVRegIdx };
  9428. mCurVRegsInit = mVRegInitializedContext.Add(mCurVRegsInit, addVec, true);
  9429. BF_ASSERT(instIdx < (int)mcBlock->mInstructions.size() - 1);
  9430. if (instIdx < (int)mcBlock->mInstructions.size() - 1)
  9431. {
  9432. auto nextInst = mcBlock->mInstructions[instIdx + 1];
  9433. if (nextInst->mVRegsInitialized != NULL)
  9434. {
  9435. auto fixedVRegsInit = mVRegInitializedContext.RemoveChange(nextInst->mVRegsInitialized, inst->mResult.mVRegIdx);
  9436. ReplaceVRegsInit(mcBlock, instIdx + 1, nextInst->mVRegsInitialized, fixedVRegsInit);
  9437. }
  9438. }
  9439. }
  9440. }
  9441. // Do this with a CMOV instead?
  9442. auto result = inst->mResult;
  9443. inst->mResult.mKind = BeMCOperandKind_None;
  9444. inst->mKind = BeMCInstKind_CondBr;
  9445. inst->mArg1 = inst->mArg0; // Move cmpType
  9446. inst->mArg0 = BeMCOperand::FromLabel(mCurLabelIdx);
  9447. int insertInstIdx = instIdx + 1;
  9448. //auto mcInst = AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromLabel(mCurLabelIdx), BeMCOperand::FromCmpKind(BeCmpKind_EQ), insertInstIdx++);
  9449. auto mcInst = AllocInst(BeMCInstKind_Mov, result, BeMCOperand::FromImmediate(0), insertInstIdx++);
  9450. mcInst = AllocInst(BeMCInstKind_Br, BeMCOperand::FromLabel(mCurLabelIdx + 1), insertInstIdx++);
  9451. CreateLabel(insertInstIdx++);
  9452. mcInst = AllocInst(BeMCInstKind_Mov, result, BeMCOperand::FromImmediate(1), insertInstIdx++);
  9453. CreateLabel(insertInstIdx++);
  9454. }
  9455. break;
  9456. case BeMCInstKind_MovSX:
  9457. case BeMCInstKind_Mov:
  9458. {
  9459. bool isSignedExt = inst->mKind == BeMCInstKind_MovSX;
  9460. auto arg0Type = GetType(inst->mArg0);
  9461. auto arg1Type = GetType(inst->mArg1);
  9462. if (inst->mArg1.mKind == BeMCOperandKind_VRegAddr)
  9463. {
  9464. auto vregInfo = mVRegInfo[inst->mArg1.mVRegIdx];
  9465. if ((!vregInfo->mIsExpr) && (!vregInfo->mForceMem))
  9466. {
  9467. SoftFail("VRegAddr used without ForceMem");
  9468. }
  9469. }
  9470. if (arg0Type->mSize == 0)
  9471. {
  9472. RemoveInst(mcBlock, instIdx);
  9473. instIdx--;
  9474. continue;
  9475. }
  9476. if (arg0Type->IsNonVectorComposite())
  9477. {
  9478. if (arg1.mKind == BeMCOperandKind_Immediate_i64)
  9479. {
  9480. // This is just a "zero initializer" marker
  9481. BF_ASSERT(arg1.mImmediate == 0);
  9482. SetAndRestoreValue<int*> insertPtr(mInsertInstIdxRef, &instIdx);
  9483. RemoveInst(mcBlock, instIdx);
  9484. CreateMemSet(OperandToAddr(arg0), 0, arg0Type->mSize, arg0Type->mAlign);
  9485. instIdx--;
  9486. isFinalRun = false;
  9487. if (debugging)
  9488. OutputDebugStrF(" Zero init\n");
  9489. break;
  9490. }
  9491. else if (arg1.mKind == BeMCOperandKind_ConstAgg)
  9492. {
  9493. bool needsSaveRegs = true;
  9494. if (instIdx > 0)
  9495. {
  9496. auto prevInst = mcBlock->mInstructions[instIdx - 1];
  9497. if ((prevInst->mKind == BeMCInstKind_PreserveVolatiles) /*&& (prevInst->mArg0.mReg == X64Reg_R11)*/)
  9498. needsSaveRegs = false;
  9499. }
  9500. auto argType = GetType(arg1);
  9501. bool useRep = argType->mSize >= BF_REP_MOV_LIMIT;
  9502. int movInstIdx = instIdx;
  9503. if (needsSaveRegs)
  9504. {
  9505. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_R11), BeMCOperand(), instIdx);
  9506. instIdx += 1;
  9507. if (useRep)
  9508. {
  9509. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand(), instIdx);
  9510. instIdx += 1;
  9511. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RDI), BeMCOperand(), instIdx);
  9512. instIdx += 1;
  9513. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RCX), BeMCOperand(), instIdx);
  9514. instIdx += 1;
  9515. }
  9516. movInstIdx = instIdx;
  9517. if (useRep)
  9518. {
  9519. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RCX), BeMCOperand(), instIdx + 1);
  9520. instIdx += 1;
  9521. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RDI), BeMCOperand(), instIdx + 1);
  9522. instIdx += 1;
  9523. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand(), instIdx + 1);
  9524. instIdx += 1;
  9525. }
  9526. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_R11), BeMCOperand(), instIdx + 1);
  9527. instIdx += 1;
  9528. isFinalRun = false;
  9529. }
  9530. BeRMParamsInfo rmInfo;
  9531. GetRMParams(inst->mArg0, rmInfo);
  9532. if (rmInfo.mMode == BeMCRMMode_Invalid)
  9533. {
  9534. if (inst->mArg0.IsSymbol())
  9535. {
  9536. // Just make it an addr so we can replace it with a temporary
  9537. inst->mArg0.mKind = BeMCOperandKind_SymbolAddr;
  9538. }
  9539. ReplaceWithNewVReg(inst->mArg0, movInstIdx, true, false, true);
  9540. auto vregInfo = GetVRegInfo(inst->mArg0);
  9541. vregInfo->mDisableR11 = true;
  9542. instIdx += 2;
  9543. isFinalRun = false;
  9544. }
  9545. continue;
  9546. }
  9547. else // Struct = Struct
  9548. {
  9549. auto arg0Addr = OperandToAddr(arg0);
  9550. auto arg1Addr = OperandToAddr(arg1);
  9551. SetAndRestoreValue<int*> insertPtr(mInsertInstIdxRef, &instIdx);
  9552. RemoveInst(mcBlock, instIdx);
  9553. CreateMemCpy(arg0Addr, arg1Addr, BF_MIN(arg0Type->mSize, arg1Type->mSize), BF_MIN(arg0Type->mAlign, arg1Type->mAlign));
  9554. instIdx--;
  9555. isFinalRun = false;
  9556. if (debugging)
  9557. OutputDebugStrF(" Mov MemCpy\n");
  9558. break;
  9559. }
  9560. }
  9561. if ((arg0Type->IsFloat()) && (arg1Type->IsIntable()))
  9562. {
  9563. if ((arg1Type->mTypeCode == BeTypeCode_Int64) && (!isSignedExt))
  9564. {
  9565. //uint64->float
  9566. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand(), instIdx);
  9567. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RAX), inst->mArg1, instIdx + 1);
  9568. inst->mArg1 = BeMCOperand::FromReg(X64Reg_RAX);
  9569. inst->mKind = BeMCInstKind_MovSX;
  9570. int labelIdx = CreateLabel(instIdx + 3);
  9571. AllocInst(BeMCInstKind_Test, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand::FromReg(X64Reg_RAX), instIdx + 3);
  9572. AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromLabel(labelIdx), BeMCOperand::FromCmpKind(BeCmpKind_SGE), instIdx + 4);
  9573. BeMCOperand mcOffset;
  9574. if (arg0Type->mSize == 8)
  9575. {
  9576. mcOffset.mKind = BeMCOperandKind_Immediate_f64;
  9577. mcOffset.mImmF64 = 1.8446744073709552e+19;
  9578. }
  9579. else
  9580. {
  9581. mcOffset.mKind = BeMCOperandKind_Immediate_f32;
  9582. mcOffset.mImmF32 = 1.8446744073709552e+19;
  9583. }
  9584. AllocInst(BeMCInstKind_Add, inst->mArg0, mcOffset, instIdx + 5);
  9585. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand(), instIdx + 7);
  9586. if (debugging)
  9587. OutputDebugStrF(" uint64->float\n");
  9588. isFinalRun = false;
  9589. instIdx--;
  9590. continue; // Rerun from PreserveVolatiles
  9591. }
  9592. else if ((!isSignedExt) || (arg1Type->mSize < 4))
  9593. {
  9594. // Int->Float conversions only work on 32 and 64-bit signed values, so we
  9595. // zero-extend into a larger scratch vreg and then we do the signed conversion on that.
  9596. // Convert from
  9597. // mov float, uint<bits>
  9598. // to
  9599. // mov scratch int<bits*2>, uint<bits>
  9600. // mov float, scratch int<bits*2>
  9601. if ((arg1.IsNativeReg()) && (arg1Type->mSize == 4))
  9602. {
  9603. // If we already have a 32-bit register, then we know that the upper 32-bits are zeroed
  9604. // so during emission we just use the 64-bit version of that register
  9605. }
  9606. else
  9607. {
  9608. auto mcScratch = AllocVirtualReg(mModule->mContext->GetPrimitiveType(BeTypeCode_Int64), 2, true);
  9609. CreateDefineVReg(mcScratch, instIdx++);
  9610. if ((isSignedExt) && (arg1Type->mSize < 8))
  9611. AllocInst(BeMCInstKind_MovSX, mcScratch, inst->mArg1, instIdx++);
  9612. else
  9613. AllocInst(BeMCInstKind_Mov, mcScratch, inst->mArg1, instIdx++);
  9614. inst->mKind = BeMCInstKind_MovSX;
  9615. inst->mArg1 = mcScratch;
  9616. isFinalRun = false;
  9617. if (debugging)
  9618. OutputDebugStrF(" MovSX\n");
  9619. }
  9620. }
  9621. }
  9622. if (!arg0.IsNativeReg())
  9623. {
  9624. // With f32s we can do a imm32 mov into memory, but there's no imm64 version of that
  9625. if ((arg0Type->mTypeCode == BeTypeCode_Double) && (arg1.IsImmediate()))
  9626. {
  9627. // One option would be to do a "movsd xmm, .rdata" to load up the immediate...
  9628. // and that would leave arg0 with the possibility of binding to a register
  9629. // in a subsequent reg pass, but we don't do that.
  9630. //ReplaceWithNewVReg(inst->mArg1, instIdx, true);
  9631. // We do an int64 load/store, so arg0 must never be allowed to be a register
  9632. if (arg0.IsVReg())
  9633. {
  9634. auto vregInfo = mVRegInfo[arg0.mVRegIdx];
  9635. vregInfo->mForceMem = true;
  9636. CheckForce(vregInfo);
  9637. }
  9638. BeMCOperand castedVReg;
  9639. if (inst->mArg0.mKind == BeMCOperandKind_VRegLoad)
  9640. {
  9641. // Maintain the load-ness (don't wrap a load)
  9642. castedVReg = AllocVirtualReg(mModule->mContext->GetPointerTo(mModule->mContext->GetPrimitiveType(BeTypeCode_Int64)), 2, false);
  9643. CreateDefineVReg(castedVReg, instIdx++);
  9644. auto castedVRegInfo = GetVRegInfo(castedVReg);
  9645. castedVRegInfo->mIsExpr = true;
  9646. castedVRegInfo->mRelTo = BeMCOperand::FromVReg(inst->mArg0.mVRegIdx);
  9647. castedVReg.mKind = BeMCOperandKind_VRegLoad;
  9648. }
  9649. else
  9650. {
  9651. castedVReg = AllocVirtualReg(mModule->mContext->GetPrimitiveType(BeTypeCode_Int64), 2, false);
  9652. CreateDefineVReg(castedVReg, instIdx++);
  9653. auto castedVRegInfo = GetVRegInfo(castedVReg);
  9654. castedVRegInfo->mIsExpr = true;
  9655. castedVRegInfo->mRelTo = inst->mArg0;
  9656. }
  9657. BeMCOperand scratchReg = AllocVirtualReg(mModule->mContext->GetPrimitiveType(BeTypeCode_Int64), 2, true);
  9658. CreateDefineVReg(scratchReg, instIdx++);
  9659. AllocInst(BeMCInstKind_Mov, scratchReg, BeMCOperand::FromImmediate(arg1.mImmediate), instIdx++);
  9660. inst->mArg0 = castedVReg;
  9661. inst->mArg1 = scratchReg;
  9662. isFinalRun = false;
  9663. if (debugging)
  9664. OutputDebugStrF(" Movsd\n");
  9665. }
  9666. // Conversion types only work in "r, rm" format
  9667. auto arg0Type = GetType(inst->mArg0);
  9668. auto arg1Type = GetType(inst->mArg1);
  9669. bool isCompat = arg0Type->mSize == arg1Type->mSize;
  9670. if ((arg0Type->IsFloat()) != arg1Type->IsFloat())
  9671. isCompat = false;
  9672. if (!isCompat)
  9673. {
  9674. ReplaceWithNewVReg(inst->mArg0, instIdx, false);
  9675. isFinalRun = false;
  9676. if (debugging)
  9677. OutputDebugStrF(" Mov float rm\n");
  9678. }
  9679. }
  9680. }
  9681. break;
  9682. case BeMCInstKind_XAdd:
  9683. {
  9684. for (int oprIdx = 0; oprIdx < 2; oprIdx++)
  9685. {
  9686. BeMCOperand& origOperand = (oprIdx == 0) ? inst->mArg0 : inst->mArg1;
  9687. if (origOperand.IsVRegAny())
  9688. {
  9689. BeRMParamsInfo rmInfo;
  9690. GetRMParams(origOperand, rmInfo);
  9691. auto vregInfo = GetVRegInfo(origOperand);
  9692. bool isValid = true;
  9693. if (oprIdx == 1)
  9694. {
  9695. if (rmInfo.mMode != BeMCRMMode_Direct)
  9696. {
  9697. auto newVReg = ReplaceWithNewVReg(origOperand, instIdx, true, true);
  9698. auto newVRegInfo = GetVRegInfo(newVReg);
  9699. newVRegInfo->mDisableRAX = true;
  9700. isFinalRun = false;
  9701. }
  9702. }
  9703. }
  9704. }
  9705. }
  9706. break;
  9707. case BeMCInstKind_CmpXChg:
  9708. {
  9709. for (int oprIdx = 0; oprIdx < 2; oprIdx++)
  9710. {
  9711. BeMCOperand& origOperand = (oprIdx == 0) ? inst->mArg0 : inst->mArg1;
  9712. if (origOperand.IsVRegAny())
  9713. {
  9714. BeRMParamsInfo rmInfo;
  9715. GetRMParams(origOperand, rmInfo);
  9716. auto vregInfo = GetVRegInfo(origOperand);
  9717. if ((ResizeRegister(rmInfo.mRegA, 8) == X64Reg_RAX) || (ResizeRegister(rmInfo.mRegB, 8) == X64Reg_RAX))
  9718. {
  9719. if (!vregInfo->mIsExpr)
  9720. {
  9721. isFinalRun = false;
  9722. vregInfo->mDisableRAX = true;
  9723. continue;
  9724. }
  9725. else
  9726. {
  9727. int safeIdx = FindSafeInstInsertPos(instIdx, true);
  9728. isFinalRun = false;
  9729. auto origType = GetType(origOperand);
  9730. BeMCOperand scratchReg = AllocVirtualReg(mModule->mContext->GetPointerTo(origType), 2, false);
  9731. CreateDefineVReg(scratchReg, safeIdx++);
  9732. AllocInst(BeMCInstKind_Mov, scratchReg, OperandToAddr(origOperand), safeIdx++);
  9733. auto newVRegInfo = GetVRegInfo(scratchReg);
  9734. newVRegInfo->mDisableRAX = true;
  9735. origOperand = scratchReg;
  9736. origOperand.mKind = BeMCOperandKind_VRegLoad;
  9737. continue;
  9738. }
  9739. }
  9740. bool isValid = true;
  9741. if (oprIdx == 1)
  9742. {
  9743. if (rmInfo.mMode != BeMCRMMode_Direct)
  9744. {
  9745. int safeIdx = FindSafeInstInsertPos(instIdx, true);
  9746. auto newVReg = ReplaceWithNewVReg(origOperand, safeIdx, true, true);
  9747. auto newVRegInfo = GetVRegInfo(newVReg);
  9748. newVRegInfo->mDisableRAX = true;
  9749. isFinalRun = false;
  9750. }
  9751. }
  9752. }
  9753. }
  9754. }
  9755. break;
  9756. case BeMCInstKind_Load:
  9757. {
  9758. // And Load gets converted to a "Load %reg0, [%reg1]"
  9759. // So both mArg0 and mArg1 must be a register
  9760. if (!IsAddressable(arg1))
  9761. {
  9762. // Convert to
  9763. // Mov %scratch, %vreg1
  9764. // Load %vreg0, [%scratch]
  9765. ReplaceWithNewVReg(inst->mArg1, instIdx, true);
  9766. arg1 = GetFixedOperand(inst->mArg1);
  9767. isFinalRun = false;
  9768. if (debugging)
  9769. OutputDebugStrF(" Load\n");
  9770. }
  9771. // Do this one second since the 'insert after' makes instIdx no longer point to the Load inst
  9772. if (!arg0.IsNativeReg())
  9773. {
  9774. // Convert to
  9775. // Load %scratch, [%vreg1]
  9776. // Mov %vreg0, %scratch
  9777. ReplaceWithNewVReg(inst->mArg0, instIdx, false);
  9778. isFinalRun = false;
  9779. if (debugging)
  9780. OutputDebugStrF(" Load 2\n");
  9781. }
  9782. }
  9783. break;
  9784. case BeMCInstKind_Store:
  9785. {
  9786. // Store gets converted to a "Store [reg], reg"
  9787. if (!IsAddressable(arg0))
  9788. {
  9789. // Convert to
  9790. // Mov %scratch, %vreg0
  9791. // Store %scratch, [%vreg1]
  9792. ReplaceWithNewVReg(inst->mArg0, instIdx, true);
  9793. isFinalRun = false;
  9794. if (debugging)
  9795. OutputDebugStrF(" Store\n");
  9796. }
  9797. if (!arg1.IsNativeReg())
  9798. {
  9799. // Convert to
  9800. // Mov %scratch, %vreg1
  9801. // Store %vreg0, [%scratch]
  9802. ReplaceWithNewVReg(inst->mArg1, instIdx, true);
  9803. isFinalRun = false;
  9804. if (debugging)
  9805. OutputDebugStrF(" Store2\n");
  9806. }
  9807. }
  9808. break;
  9809. case BeMCInstKind_DefPhi:
  9810. {
  9811. // This is from a PHI whose value was not used
  9812. RemoveInst(mcBlock, instIdx);
  9813. instIdx--;
  9814. continue;
  9815. }
  9816. break;
  9817. }
  9818. if ((pendingInitKind != BfIRInitType_NotNeeded) && (pendingInitKind != BfIRInitType_NotNeeded_AliveOnDecl))
  9819. {
  9820. Fail("Shouldn't happen here anymore");
  9821. /*int vregIdx = inst->mArg0.mVRegIdx;
  9822. auto dbgVar = mVRegInfo[vregIdx]->mDbgVariable;
  9823. auto varType = mVRegInfo[vregIdx]->mType;
  9824. auto initType = dbgVar->mPendingInitType;
  9825. if (varType->IsFloat())
  9826. {
  9827. BeMCOperand zeroVal;
  9828. if (varType->mTypeCode == BeTypeCode_Float)
  9829. zeroVal.mKind = BeMCOperandKind_Immediate_f32;
  9830. else
  9831. zeroVal.mKind = BeMCOperandKind_Immediate_f64;
  9832. if (initType == BfIRInitType_Uninitialized)
  9833. zeroVal.mImmFloat = NAN;
  9834. else
  9835. zeroVal.mImmFloat = 0;
  9836. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(vregIdx), zeroVal, instIdx + 1);
  9837. }
  9838. else if (varType->IsNonVectorComposite())
  9839. {
  9840. SetAndRestoreValue<int*> insertPtr(mInsertInstIdxRef, &instIdx);
  9841. instIdx++;
  9842. uint8 val = (initType == BfIRInitType_Uninitialized) ? 0xCC : 0;
  9843. CreateMemSet(BeMCOperand::FromVRegAddr(vregIdx), val, varType->mSize, varType->mAlign);
  9844. instIdx--;
  9845. isFinalRun = false;
  9846. }
  9847. else
  9848. {
  9849. int64 val = (initType == BfIRInitType_Uninitialized) ? (int64)0xCCCCCCCCCCCCCCCCLL : 0;
  9850. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(vregIdx), BeMCOperand::FromImmediate(val), instIdx + 1);
  9851. }
  9852. dbgVar->mPendingInitType = BfIRInitType_NotNeeded;*/
  9853. }
  9854. }
  9855. mActiveInst = NULL;
  9856. }
  9857. BF_ASSERT(regPreserveDepth == 0);
  9858. SetCurrentInst(NULL);
  9859. if (hasPendingActualizations)
  9860. {
  9861. DoActualization();
  9862. isFinalRun = false;
  9863. if (debugging)
  9864. OutputDebugStrF(" DoActualizations\n");
  9865. }
  9866. mLegalizationIterations++;
  9867. return isFinalRun;
  9868. }
  9869. void BeMCContext::DoSanityChecking()
  9870. {
  9871. for (auto mcBlock : mBlocks)
  9872. {
  9873. mActiveBlock = mcBlock;
  9874. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size() - 1; instIdx++)
  9875. {
  9876. auto inst = mcBlock->mInstructions[instIdx];
  9877. auto nextInst = mcBlock->mInstructions[instIdx + 1];
  9878. auto dest = inst->GetDest();
  9879. if (dest == NULL)
  9880. continue;
  9881. auto vregInfo = GetVRegInfo(*dest);
  9882. if ((vregInfo != NULL) && (vregInfo->mHasDynLife))
  9883. {
  9884. if ((nextInst->mVRegsInitialized != NULL) && (!mVRegInitializedContext.IsSet(nextInst->mVRegsInitialized, dest->mVRegIdx)))
  9885. {
  9886. if (nextInst->mKind != BeMCInstKind_RestoreVolatiles)
  9887. {
  9888. // It's possible during legalization to extend the usage of a dynLife value 'upward', and we need
  9889. // to ensure we also extend the init flags up as well
  9890. SoftFail("Invalid liveness - from init flag not being set properly?");
  9891. }
  9892. }
  9893. }
  9894. }
  9895. }
  9896. }
  9897. void BeMCContext::DoBlockCombine()
  9898. {
  9899. auto masterBlock = mMCBlockAlloc.Alloc();
  9900. mActiveBlock = masterBlock;
  9901. for (auto mcBlock : mBlocks)
  9902. {
  9903. mcBlock->mLabelIdx = mCurLabelIdx++;
  9904. }
  9905. for (auto mcBlock : mBlocks)
  9906. {
  9907. auto inst = AllocInst();
  9908. inst->mKind = BeMCInstKind_Label;
  9909. inst->mArg0.mKind = BeMCOperandKind_Label;
  9910. inst->mArg0.mLabelIdx = mcBlock->mLabelIdx;
  9911. for (auto blockInst : mcBlock->mInstructions)
  9912. {
  9913. if (inst->mDbgLoc == NULL)
  9914. inst->mDbgLoc = blockInst->mDbgLoc;
  9915. auto operands = { &blockInst->mArg0, &blockInst->mArg1 };
  9916. for (auto op : operands)
  9917. {
  9918. if (op->mKind == BeMCOperandKind_Block)
  9919. {
  9920. op->mKind = BeMCOperandKind_Label;
  9921. op->mLabelIdx = op->mBlock->mLabelIdx;
  9922. }
  9923. }
  9924. masterBlock->mInstructions.push_back(blockInst);
  9925. }
  9926. }
  9927. mBlocks.Clear();
  9928. mBlocks.push_back(masterBlock);
  9929. }
  9930. bool BeMCContext::DoJumpRemovePass()
  9931. {
  9932. struct LabelStats
  9933. {
  9934. public:
  9935. bool mHasRefs;
  9936. bool mForceKeepLabel;
  9937. bool mFromMultipleDbgLocs;
  9938. // If all branches share the dbgLoc with the first inst then we can potentially do a remap
  9939. BeDbgLoc* mRefDbgLoc;
  9940. public:
  9941. LabelStats()
  9942. {
  9943. mHasRefs = false;
  9944. mForceKeepLabel = false;
  9945. mFromMultipleDbgLocs = false;
  9946. mRefDbgLoc = NULL;
  9947. }
  9948. };
  9949. bool didWork = false;
  9950. SizedArray<LabelStats, 32> labelStats;
  9951. labelStats.resize(mCurLabelIdx);
  9952. Dictionary<int, int> labelRemaps;
  9953. for (auto& switchEntry : mSwitchEntries)
  9954. {
  9955. labelStats[switchEntry.mBlock->mLabelIdx].mForceKeepLabel = true;
  9956. labelStats[switchEntry.mBlock->mLabelIdx].mHasRefs = true;
  9957. }
  9958. for (auto mcBlock : mBlocks)
  9959. {
  9960. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  9961. {
  9962. auto inst = mcBlock->mInstructions[instIdx];
  9963. if (inst->mKind == BeMCInstKind_Label)
  9964. {
  9965. if (instIdx < (int)mcBlock->mInstructions.size() - 2)
  9966. {
  9967. auto nextInst = mcBlock->mInstructions[instIdx + 1];
  9968. if (nextInst->mDbgLoc != NULL)
  9969. {
  9970. auto& stats = labelStats[inst->mArg0.mLabelIdx];
  9971. if ((stats.mRefDbgLoc != NULL) && (stats.mRefDbgLoc != nextInst->mDbgLoc))
  9972. stats.mFromMultipleDbgLocs = true;
  9973. else
  9974. stats.mRefDbgLoc = nextInst->mDbgLoc;
  9975. }
  9976. }
  9977. }
  9978. else
  9979. {
  9980. auto operands = { &inst->mArg0, &inst->mArg1 };
  9981. for (auto operand : operands)
  9982. {
  9983. if (operand->mKind == BeMCOperandKind_Label)
  9984. {
  9985. labelStats[operand->mLabelIdx].mHasRefs = true;
  9986. if (inst->mDbgLoc != NULL)
  9987. {
  9988. auto& stats = labelStats[operand->mLabelIdx];
  9989. if ((stats.mRefDbgLoc != NULL) && (stats.mRefDbgLoc != inst->mDbgLoc))
  9990. stats.mFromMultipleDbgLocs = true;
  9991. else
  9992. stats.mRefDbgLoc = inst->mDbgLoc;
  9993. }
  9994. }
  9995. }
  9996. }
  9997. }
  9998. }
  9999. for (auto mcBlock : mBlocks)
  10000. {
  10001. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  10002. {
  10003. auto inst = mcBlock->mInstructions[instIdx];
  10004. bool goBackToLastBranch = false;
  10005. bool removeInst = false;
  10006. // If we're just jumping to the next label, remove jump
  10007. // It's possible there are several labels in a row here, so check against all of them
  10008. if (inst->mKind == BeMCInstKind_Br)
  10009. {
  10010. if (inst->mArg0.mKind == BeMCOperandKind_Label)
  10011. {
  10012. for (int labelInstIdx = instIdx + 1; labelInstIdx < (int)mcBlock->mInstructions.size(); labelInstIdx++)
  10013. {
  10014. auto labelInst = mcBlock->mInstructions[labelInstIdx];
  10015. if (labelInst->mKind != BeMCInstKind_Label)
  10016. {
  10017. if (labelInst->IsPsuedo())
  10018. continue;
  10019. break;
  10020. }
  10021. if (inst->mArg0 == labelInst->mArg0)
  10022. {
  10023. didWork = true;
  10024. inst->mKind = inst->mArg1 ? BeMCInstKind_Nop : BeMCInstKind_EnsureInstructionAt;
  10025. inst->mArg0 = BeMCOperand();
  10026. inst->mArg1 = BeMCOperand();
  10027. goBackToLastBranch = true;
  10028. break;
  10029. }
  10030. }
  10031. }
  10032. }
  10033. if (removeInst)
  10034. {
  10035. mcBlock->mInstructions.RemoveAt(instIdx);
  10036. instIdx--;
  10037. continue;
  10038. }
  10039. // Do we have a label that immediately branches?
  10040. if (inst->mKind == BeMCInstKind_Label)
  10041. {
  10042. // We can only remap this if the contained instructions and all references share the same dbgLoc
  10043. if ((instIdx < (int)mcBlock->mInstructions.size() - 2) && (!labelStats[inst->mArg0.mLabelIdx].mFromMultipleDbgLocs))
  10044. {
  10045. bool allowRemove = true;
  10046. auto nextInst = mcBlock->mInstructions[instIdx + 1];
  10047. auto nextNextInst = mcBlock->mInstructions[instIdx + 2];
  10048. if ((nextInst->mKind == BeMCInstKind_Br) &&
  10049. ((inst->mDbgLoc == NULL) || (inst->mDbgLoc == nextNextInst->mDbgLoc)) &&
  10050. (!labelStats[inst->mArg0.mLabelIdx].mForceKeepLabel))
  10051. {
  10052. int checkIdx = instIdx - 1;
  10053. while (checkIdx >= 0)
  10054. {
  10055. auto prevInst = mcBlock->mInstructions[checkIdx];
  10056. if (prevInst->mKind == BeMCInstKind_Label)
  10057. {
  10058. // Keep looking
  10059. }
  10060. else if (prevInst->mKind == BeMCInstKind_Br)
  10061. {
  10062. break;
  10063. }
  10064. else
  10065. {
  10066. // We flowed into here, so we can't remove this branch
  10067. allowRemove = false;
  10068. break;
  10069. }
  10070. checkIdx--;
  10071. }
  10072. /*if (labelStats[inst->mArg0.mLabelIdx].mFromMultipleDbgLocs)
  10073. allowRemove = false;*/
  10074. didWork = true;
  10075. RemoveInst(mcBlock, instIdx); // Remove label
  10076. labelRemaps.TryAdd(inst->mArg0.mLabelIdx, nextInst->mArg0.mLabelIdx);
  10077. if (allowRemove)
  10078. {
  10079. RemoveInst(mcBlock, instIdx); // Remove branch
  10080. goBackToLastBranch = true;
  10081. }
  10082. }
  10083. }
  10084. }
  10085. if (goBackToLastBranch)
  10086. {
  10087. // We may be able to remove the previous branch now
  10088. int checkIdx = instIdx - 1;
  10089. while (checkIdx >= 0)
  10090. {
  10091. auto checkInst = mcBlock->mInstructions[checkIdx];
  10092. if (checkInst->mKind == BeMCInstKind_Br)
  10093. {
  10094. instIdx = checkIdx - 1;
  10095. break;
  10096. }
  10097. else if (checkInst->mKind != BeMCInstKind_Label)
  10098. break;
  10099. checkIdx--;
  10100. }
  10101. continue;
  10102. }
  10103. }
  10104. }
  10105. if (!labelRemaps.IsEmpty())
  10106. {
  10107. while (true)
  10108. {
  10109. bool didRemapMerge = false;
  10110. for (auto& remapPair : labelRemaps)
  10111. {
  10112. //auto itr = labelRemaps.find(remapPair.second);
  10113. //if (itr != labelRemaps.end())
  10114. int* valuePtr = NULL;
  10115. if (labelRemaps.TryGetValue(remapPair.mValue, &valuePtr))
  10116. {
  10117. //remapPair.second = itr->second;
  10118. remapPair.mValue = *valuePtr;
  10119. didRemapMerge = true;
  10120. }
  10121. }
  10122. if (!didRemapMerge)
  10123. break;
  10124. }
  10125. for (auto mcBlock : mBlocks)
  10126. {
  10127. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  10128. {
  10129. auto inst = mcBlock->mInstructions[instIdx];
  10130. auto operands = { &inst->mArg0, &inst->mArg1 };
  10131. for (auto operand : operands)
  10132. {
  10133. if (operand->mKind == BeMCOperandKind_Label)
  10134. {
  10135. //auto itr = labelRemaps.find(operand->mLabelIdx);
  10136. //if (itr != labelRemaps.end())
  10137. int* valuePtr = NULL;
  10138. if (labelRemaps.TryGetValue(operand->mLabelIdx, &valuePtr))
  10139. {
  10140. labelStats[operand->mLabelIdx].mHasRefs = false;
  10141. //operand->mLabelIdx = itr->second;
  10142. operand->mLabelIdx = *valuePtr;
  10143. labelStats[operand->mLabelIdx].mHasRefs = true;
  10144. }
  10145. }
  10146. }
  10147. }
  10148. }
  10149. }
  10150. for (auto mcBlock : mBlocks)
  10151. {
  10152. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  10153. {
  10154. auto inst = mcBlock->mInstructions[instIdx];
  10155. bool goBackToLastBranch = false;
  10156. }
  10157. }
  10158. // Remove unreferenced labels
  10159. for (auto mcBlock : mBlocks)
  10160. {
  10161. bool doMorePasses = false;
  10162. bool isUnreachable = false;
  10163. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  10164. {
  10165. auto inst = mcBlock->mInstructions[instIdx];
  10166. if (inst->mKind == BeMCInstKind_Label)
  10167. {
  10168. isUnreachable = false;
  10169. if (!labelStats[inst->mArg0.mLabelIdx].mHasRefs)
  10170. {
  10171. didWork = true;
  10172. mcBlock->mInstructions.RemoveAt(instIdx);
  10173. instIdx--;
  10174. }
  10175. }
  10176. if (inst->mKind == BeMCInstKind_CondBr)
  10177. doMorePasses = true;
  10178. if (inst->mKind == BeMCInstKind_Br)
  10179. {
  10180. if (isUnreachable)
  10181. {
  10182. // We can't possible reach this branch. This is an artifact of other br removals
  10183. didWork = true;
  10184. RemoveInst(mcBlock, instIdx);
  10185. instIdx--;
  10186. continue;
  10187. }
  10188. isUnreachable = true;
  10189. }
  10190. }
  10191. if (doMorePasses)
  10192. {
  10193. for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  10194. {
  10195. auto inst = mcBlock->mInstructions[instIdx];
  10196. // For the form:
  10197. // CondBr %label0, eq
  10198. // Br %label1
  10199. // %label0:
  10200. // Invert condition and convert that to
  10201. // CondBr %label1, neq
  10202. // %label0:
  10203. // We perform this after the jump-removal process to ensure we compare against the correct
  10204. // final label positions, otherwise there are cases where we would incorrectly apply this
  10205. // transformation early and end up with worse results than just not doing it
  10206. if (inst->mKind == BeMCInstKind_CondBr)
  10207. {
  10208. if (instIdx < (int)mcBlock->mInstructions.size() - 2)
  10209. {
  10210. int nextIdx = instIdx + 1;
  10211. int nextNextIdx = instIdx + 2;
  10212. auto nextInst = mcBlock->mInstructions[nextIdx];
  10213. if (nextInst->mKind == BeMCInstKind_EnsureInstructionAt)
  10214. {
  10215. if (nextInst->mDbgLoc != inst->mDbgLoc)
  10216. continue;
  10217. nextIdx++;
  10218. nextNextIdx++;
  10219. nextInst = mcBlock->mInstructions[nextIdx];
  10220. }
  10221. auto nextNextInst = mcBlock->mInstructions[nextNextIdx];
  10222. if ((nextInst->mKind == BeMCInstKind_Br) &&
  10223. (nextNextInst->mKind == BeMCInstKind_Label) && (inst->mArg0 == nextNextInst->mArg0) &&
  10224. (!BeModule::IsCmpOrdered(inst->mArg1.mCmpKind)))
  10225. {
  10226. didWork = true;
  10227. inst->mArg0 = nextInst->mArg0;
  10228. BF_ASSERT(inst->mArg1.mKind == BeMCOperandKind_CmpKind);
  10229. inst->mArg1.mCmpKind = BeModule::InvertCmp(inst->mArg1.mCmpKind);
  10230. mcBlock->mInstructions.RemoveAt(nextIdx);
  10231. }
  10232. }
  10233. }
  10234. }
  10235. }
  10236. }
  10237. return didWork;
  10238. }
  10239. void BeMCContext::DoRegFinalization()
  10240. {
  10241. SizedArray<int, 32> savedVolatileVRegs;
  10242. mUsedRegs.Clear();
  10243. SetCurrentInst(NULL);
  10244. // Remove the Def instructions, replace vreg
  10245. for (auto mcBlock : mBlocks)
  10246. {
  10247. mActiveBlock = mcBlock;
  10248. //for (int instIdx = 0; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  10249. for (auto instEnum = BeInstEnumerator(this, mcBlock); instEnum.HasMore(); instEnum.Next())
  10250. {
  10251. auto inst = instEnum.Get();
  10252. SetCurrentInst(inst);
  10253. if (inst->IsDef())
  10254. {
  10255. auto vregInfo = GetVRegInfo(inst->mArg0);
  10256. if (vregInfo->mDbgVariable != NULL)
  10257. vregInfo->mAwaitingDbgStart = true;
  10258. instEnum.RemoveCurrent();
  10259. continue;
  10260. }
  10261. // This is similar to an optimization we have in DoLegalization, but it allows for directRelTo's to match
  10262. // We can't match those in DoLegalization because it would alter the liveness
  10263. if (inst->mKind == BeMCInstKind_Mov)
  10264. {
  10265. // Useless mov, remove it
  10266. if (OperandsEqual(inst->mArg0, inst->mArg1, false))
  10267. {
  10268. if (inst->mDbgLoc != NULL)
  10269. {
  10270. inst->mKind = BeMCInstKind_EnsureInstructionAt;
  10271. inst->mArg0 = BeMCOperand();
  10272. inst->mArg1 = BeMCOperand();
  10273. }
  10274. else
  10275. {
  10276. //RemoveInst(mcBlock, instIdx);
  10277. //instIdx--;
  10278. instEnum.RemoveCurrent();
  10279. }
  10280. continue;
  10281. }
  10282. }
  10283. if ((inst->mKind >= BeMCInstKind_Def) && (inst->mKind <= BeMCInstKind_LifetimeEnd))
  10284. continue;
  10285. if (inst->mResult == inst->mArg0)
  10286. {
  10287. // Destination is implicitly arg0
  10288. inst->mResult.mKind = BeMCOperandKind_None;
  10289. }
  10290. // Unfortunate rule:
  10291. // if we end up with:
  10292. // PreserveVolatiles RAX
  10293. // Shl vreg0<RCX>, CL
  10294. // RestoreVolatiles RAXS
  10295. // Then convert that to
  10296. // PreserveVolatiles RAX
  10297. // Shl <SavedLoc>, RAX
  10298. // RestoreVolatiles RAX
  10299. /*if ((inst->mKind == BeMCInstKind_Shl) ||
  10300. (inst->mKind == BeMCInstKind_Shr) ||
  10301. (inst->mKind == BeMCInstKind_Sar))
  10302. {
  10303. if (inst->mArg1.IsNativeReg())
  10304. {
  10305. }
  10306. }*/
  10307. auto operands = { &inst->mResult, &inst->mArg0, &inst->mArg1 };
  10308. for (auto operand : operands)
  10309. {
  10310. BeMCOperand checkOperand = *operand;
  10311. while (true)
  10312. {
  10313. auto vregInfo = GetVRegInfo(checkOperand);
  10314. if (vregInfo == NULL)
  10315. break;
  10316. // We may think that range starts should always occur on a "mov <vreg>, <value>", but
  10317. // we have some cases like un-splatting into a composite, or using an uninitialized local
  10318. // variable where we want to start the range on ANY use, not just a direct assignment
  10319. if (vregInfo->mAwaitingDbgStart)
  10320. {
  10321. //AllocInst(BeMCInstKind_DbgRangeStart, checkOperand, instIdx + 1);
  10322. //vregInfo->mAwaitingDbgStart = false;
  10323. }
  10324. if (!vregInfo->mRelTo)
  10325. break;
  10326. checkOperand = vregInfo->mRelTo;
  10327. }
  10328. FixOperand(*operand, -1);
  10329. if (operand->mKind == BeMCOperandKind_NativeReg)
  10330. {
  10331. auto nativeReg = GetFullRegister(operand->mReg);
  10332. BF_ASSERT(nativeReg != X64Reg_None);
  10333. if (nativeReg != X64Reg_RSP) // This can happen from allocas
  10334. mUsedRegs.Add(nativeReg);
  10335. }
  10336. }
  10337. if (inst->IsMov())
  10338. {
  10339. bool removeInst = false;
  10340. if (GetFixedOperand(inst->mArg0) == GetFixedOperand(inst->mArg1))
  10341. {
  10342. removeInst = true;
  10343. }
  10344. if ((inst->mArg0.IsNativeReg()) && (inst->mArg1.IsNativeReg()))
  10345. {
  10346. // Removes size-reducing moves such as "mov eax, rax"
  10347. if (GetFullRegister(inst->mArg0.mReg) == inst->mArg1.mReg)
  10348. {
  10349. removeInst = true;
  10350. }
  10351. }
  10352. if (removeInst)
  10353. {
  10354. inst->mKind = BeMCInstKind_EnsureInstructionAt;
  10355. inst->mResult = BeMCOperand();
  10356. inst->mArg0 = BeMCOperand();
  10357. inst->mArg1 = BeMCOperand();
  10358. }
  10359. }
  10360. switch (inst->mKind)
  10361. {
  10362. case BeMCInstKind_PreserveVolatiles:
  10363. {
  10364. int preserveIdx;
  10365. BeMCInst* preserveInst;
  10366. BeMCInst* restoreInst;
  10367. if (inst->mKind == BeMCInstKind_PreserveVolatiles)
  10368. {
  10369. preserveIdx = instEnum.mReadIdx;
  10370. preserveInst = inst;
  10371. restoreInst = mcBlock->mInstructions[FindRestoreVolatiles(mcBlock, instEnum.mReadIdx)];
  10372. }
  10373. else
  10374. {
  10375. preserveIdx = FindPreserveVolatiles(mcBlock, instEnum.mReadIdx);
  10376. preserveInst = mcBlock->mInstructions[preserveIdx];
  10377. restoreInst = inst;
  10378. }
  10379. int insertIdx = instEnum.mWriteIdx;
  10380. if ((inst->mArg0.IsNativeReg()) && (inst->mArg1.IsNativeReg()))
  10381. {
  10382. // If our exception (IE: div target) is set to the desired preserve reg then
  10383. if (inst->mArg0.mReg == GetFullRegister(inst->mArg1.mReg))
  10384. break;
  10385. }
  10386. // Save volatile registers
  10387. for (int liveVRegIdx : *inst->mLiveness)
  10388. {
  10389. if (liveVRegIdx >= mLivenessContext.mNumItems)
  10390. continue;
  10391. auto checkVRegIdx = GetUnderlyingVReg(liveVRegIdx);
  10392. // Already handled the underlying vreg?
  10393. if ((checkVRegIdx != liveVRegIdx) && (mLivenessContext.IsSet(preserveInst->mLiveness, checkVRegIdx)))
  10394. continue;
  10395. auto vregInfo = mVRegInfo[checkVRegIdx];
  10396. if (vregInfo->mReg != X64Reg_None)
  10397. {
  10398. // Do we specify a particular reg, or just all volatiles?
  10399. if ((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg != GetFullRegister(vregInfo->mReg)))
  10400. continue;
  10401. if (!mLivenessContext.IsSet(restoreInst->mLiveness, liveVRegIdx))
  10402. {
  10403. // This vreg doesn't survive until the PreserveRegs -- it's probably used for params or the call addr
  10404. continue;
  10405. }
  10406. if ((inst->mArg0.IsNativeReg()) || (IsVolatileReg(vregInfo->mReg)))
  10407. {
  10408. if (vregInfo->mVolatileVRegSave == -1)
  10409. {
  10410. BF_ASSERT(inst->mKind != BeMCInstKind_RestoreVolatiles); // Should have already been allocated
  10411. auto savedVReg = AllocVirtualReg(vregInfo->mType);
  10412. auto savedVRegInfo = mVRegInfo[savedVReg.mVRegIdx];
  10413. savedVRegInfo->mForceMem = true;
  10414. vregInfo->mVolatileVRegSave = savedVReg.mVRegIdx;
  10415. }
  10416. savedVolatileVRegs.push_back(checkVRegIdx);
  10417. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromVReg(vregInfo->mVolatileVRegSave), GetVReg(checkVRegIdx), insertIdx++);
  10418. if (insertIdx > instEnum.mReadIdx)
  10419. instEnum.Next();
  10420. else
  10421. instEnum.mWriteIdx++;
  10422. }
  10423. }
  10424. }
  10425. bool savingSpecificReg = false;
  10426. if (inst->mArg0)
  10427. {
  10428. if (inst->mArg0.IsNativeReg())
  10429. savingSpecificReg = true;
  10430. else
  10431. BF_ASSERT(inst->mArg0.mKind == BeMCOperandKind_PreserveFlag);
  10432. }
  10433. // Full PreserveRegs case:
  10434. // Check register crossover errors, ie:
  10435. // Mov RCX, reg0<RDX>
  10436. // Mov RDX, reg1<RCX>
  10437. if (!savingSpecificReg)
  10438. {
  10439. int preserveIdx = instEnum.mReadIdx;
  10440. bool regsFailed = false;
  10441. for (int pass = 0; true; pass++)
  10442. {
  10443. bool regStomped[X64Reg_COUNT] = { false };
  10444. bool regFailed[X64Reg_COUNT] = { false };
  10445. regsFailed = false;
  10446. bool isFinalPass = pass == 4;
  10447. int instEndIdx = -1;
  10448. for (int instIdx = preserveIdx + 1; instIdx < (int)mcBlock->mInstructions.size(); instIdx++)
  10449. {
  10450. auto inst = mcBlock->mInstructions[instIdx];
  10451. if ((inst->mKind == BeMCInstKind_RestoreVolatiles) ||
  10452. (inst->mKind == BeMCInstKind_Call))
  10453. {
  10454. instEndIdx = instIdx;
  10455. if (inst->mKind == BeMCInstKind_Call)
  10456. {
  10457. BeRMParamsInfo rmInfo;
  10458. GetRMParams(inst->mArg0, rmInfo);
  10459. if (((rmInfo.mRegA != X64Reg_None) && (rmInfo.mRegA != X64Reg_RAX) && (regStomped[rmInfo.mRegA])) ||
  10460. ((rmInfo.mRegB != X64Reg_None) && (regStomped[rmInfo.mRegB])))
  10461. {
  10462. BF_ASSERT(pass == 0);
  10463. // Target is stomped! Mov to RAX and then handle it along with the generalized reg setup
  10464. auto callTarget = BeMCOperand::FromReg(X64Reg_RAX);
  10465. AllocInst(BeMCInstKind_Mov, callTarget, inst->mArg0, preserveIdx + 1);
  10466. inst->mArg0 = callTarget;
  10467. regsFailed = true; // So we'll rerun taking into account the new RAX reg
  10468. instEndIdx++;
  10469. }
  10470. }
  10471. break;
  10472. }
  10473. if (inst->mKind == BeMCInstKind_Def)
  10474. {
  10475. // If we have a Def here, it's because of a legalization that introduced a variable in a non-safe position
  10476. SoftFail("Illegal def");
  10477. }
  10478. if (inst->mKind == BeMCInstKind_Mov)
  10479. {
  10480. X64CPURegister regs[2] = { X64Reg_None, X64Reg_None };
  10481. GetUsedRegs(inst->mArg1, regs[0], regs[1]);
  10482. bool isStomped = false;
  10483. for (auto reg : regs)
  10484. {
  10485. if (reg != X64Reg_None)
  10486. {
  10487. if (regStomped[reg])
  10488. {
  10489. regsFailed = true;
  10490. isStomped = true;
  10491. regFailed[reg] = true;
  10492. }
  10493. }
  10494. }
  10495. // Don't swap on the final pass, we need regsFailed to be accurate so this is just a
  10496. // verification pass
  10497. if ((isStomped) && (!isFinalPass))
  10498. {
  10499. // Bubble up, maybe a reordering will fix our issue
  10500. BF_SWAP(mcBlock->mInstructions[instIdx - 1], mcBlock->mInstructions[instIdx]);
  10501. }
  10502. if (inst->mArg0.IsNativeReg())
  10503. {
  10504. auto reg = GetFullRegister(inst->mArg0.mReg);
  10505. regStomped[(int)reg] = true;
  10506. }
  10507. }
  10508. }
  10509. if (!regsFailed)
  10510. break;
  10511. if (isFinalPass)
  10512. {
  10513. // We've failed to reorder
  10514. int deferredIdx = 0;
  10515. for (int instIdx = preserveIdx + 1; instIdx < instEndIdx; instIdx++)
  10516. {
  10517. auto inst = mcBlock->mInstructions[instIdx];
  10518. if ((inst->mKind == BeMCInstKind_RestoreVolatiles) ||
  10519. (inst->mKind == BeMCInstKind_Call))
  10520. break;
  10521. if (inst->mKind == BeMCInstKind_Mov)
  10522. {
  10523. if (inst->mArg0.IsNativeReg())
  10524. {
  10525. auto reg = GetFullRegister(inst->mArg0.mReg);
  10526. if (regFailed[(int)reg])
  10527. {
  10528. // Convert
  10529. // Mov <reg>, vreg0
  10530. // To
  10531. // Push vreg0
  10532. // ...
  10533. // Pop <reg>
  10534. auto mcPopReg = inst->mArg0;
  10535. auto popType = GetType(mcPopReg);
  10536. if (!popType->IsFloatOrVector())
  10537. mcPopReg.mReg = GetFullRegister(mcPopReg.mReg);
  10538. auto pushType = GetType(inst->mArg1);
  10539. auto useTypeCode = pushType->IsFloatOrVector() ? pushType->mTypeCode : BeTypeCode_Int64;
  10540. BF_ASSERT(deferredIdx < 4);
  10541. auto mcDeferred = GetCallArgVReg(deferredIdx++, useTypeCode);
  10542. CreateDefineVReg(BeMCOperand::FromVReg(mcDeferred.mVRegIdx), instIdx++);
  10543. instEndIdx++;
  10544. auto popInst = AllocInst(BeMCInstKind_Mov, mcPopReg, mcDeferred, instEndIdx);
  10545. //auto popInst = AllocInst(BeMCInstKind_Pop, mcPopReg, instEndIdx);
  10546. auto arg1 = inst->mArg1;
  10547. FixOperand(arg1);
  10548. if (arg1.IsNativeReg())
  10549. {
  10550. inst->mKind = BeMCInstKind_Mov;
  10551. inst->mArg0 = mcDeferred;
  10552. if (!popType->IsFloatOrVector())
  10553. inst->mArg1 = BeMCOperand::FromReg(GetFullRegister(arg1.mReg));
  10554. else
  10555. inst->mArg1 = BeMCOperand::FromReg(arg1.mReg);
  10556. }
  10557. else
  10558. {
  10559. // Use R11 or XMM5 as our temporary - they are the least likely volatiles to be
  10560. // allocated, so we may not need to restore them after using them
  10561. X64CPURegister scratchReg;
  10562. if (pushType->mTypeCode == BeTypeCode_Float)
  10563. scratchReg = X64Reg_XMM5_f32;
  10564. else if (pushType->mTypeCode == BeTypeCode_Double)
  10565. scratchReg = X64Reg_XMM5_f64;
  10566. else
  10567. scratchReg = X64Reg_R11;
  10568. int volatileVRegSave = -1;
  10569. for (auto vregIdx : savedVolatileVRegs)
  10570. {
  10571. auto vregInfo = mVRegInfo[vregIdx];
  10572. if (GetFullRegister(vregInfo->mReg) == scratchReg)
  10573. {
  10574. volatileVRegSave = vregInfo->mVolatileVRegSave;
  10575. }
  10576. }
  10577. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(scratchReg), inst->mArg1, instIdx++);
  10578. instEndIdx++;
  10579. inst->mKind = BeMCInstKind_Mov;
  10580. inst->mArg0 = mcDeferred;
  10581. inst->mArg1 = BeMCOperand::FromReg(scratchReg);
  10582. if (volatileVRegSave != -1)
  10583. {
  10584. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(scratchReg), BeMCOperand::FromVReg(volatileVRegSave), ++instIdx);
  10585. instEndIdx++;
  10586. }
  10587. }
  10588. }
  10589. }
  10590. }
  10591. }
  10592. break;
  10593. }
  10594. }
  10595. }
  10596. }
  10597. break;
  10598. case BeMCInstKind_RestoreVolatiles:
  10599. {
  10600. bool doRestore = true;
  10601. if ((inst->mArg0.mKind == BeMCOperandKind_PreserveFlag) && ((inst->mArg0.mPreserveFlag & BeMCPreserveFlag_NoRestore) != 0))
  10602. {
  10603. // Don't bother restore registers for NoReturns
  10604. doRestore = false;
  10605. }
  10606. if (doRestore)
  10607. {
  10608. for (auto vregIdx : savedVolatileVRegs)
  10609. {
  10610. auto vregInfo = mVRegInfo[vregIdx];
  10611. int insertIdx = instEnum.mWriteIdx;
  10612. AllocInst(BeMCInstKind_Mov, GetVReg(vregIdx), BeMCOperand::FromVReg(vregInfo->mVolatileVRegSave), insertIdx++);
  10613. if (insertIdx > instEnum.mReadIdx)
  10614. instEnum.Next();
  10615. else
  10616. instEnum.mWriteIdx++;
  10617. }
  10618. }
  10619. savedVolatileVRegs.clear();
  10620. }
  10621. break;
  10622. }
  10623. }
  10624. }
  10625. SetCurrentInst(NULL);
  10626. // Do this at the end so we can properly handle RangeStarts
  10627. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  10628. {
  10629. auto vregInfo = mVRegInfo[vregIdx];
  10630. if (vregInfo->mRelTo)
  10631. {
  10632. FixOperand(vregInfo->mRelTo);
  10633. FixOperand(vregInfo->mRelOffset);
  10634. }
  10635. }
  10636. }
  10637. BeMCInstForm BeMCContext::GetInstForm(BeMCInst* inst)
  10638. {
  10639. if (!inst->mArg0)
  10640. return BeMCInstForm_Unknown;
  10641. auto arg0 = GetFixedOperand(inst->mArg0);
  10642. auto arg1 = GetFixedOperand(inst->mArg1);
  10643. if (!arg1)
  10644. {
  10645. if (arg0.mKind == BeMCOperandKind_Symbol)
  10646. return BeMCInstForm_Symbol;
  10647. if (arg0.mKind == BeMCOperandKind_SymbolAddr)
  10648. return BeMCInstForm_SymbolAddr;
  10649. auto arg0Type = GetType(arg0);
  10650. if (arg0Type == NULL)
  10651. return BeMCInstForm_Unknown;
  10652. if (arg0.IsImmediate())
  10653. {
  10654. if ((arg0.mImmediate >= -0x80000000LL) && (arg0.mImmediate <= 0x7FFFFFFF))
  10655. return BeMCInstForm_IMM32;
  10656. }
  10657. if (arg0.mKind == BeMCOperandKind_NativeReg)
  10658. {
  10659. if (arg0Type->mSize == 4)
  10660. return BeMCInstForm_R32;
  10661. if (arg0Type->mSize == 2)
  10662. return BeMCInstForm_R16;
  10663. if (arg0Type->mSize == 1)
  10664. return BeMCInstForm_R8;
  10665. return BeMCInstForm_R64;
  10666. }
  10667. if (arg0Type->mSize == 4)
  10668. return BeMCInstForm_RM32;
  10669. if (arg0Type->mSize == 2)
  10670. return BeMCInstForm_RM16;
  10671. if (arg0Type->mSize == 1)
  10672. return BeMCInstForm_RM8;
  10673. return BeMCInstForm_RM64;
  10674. }
  10675. auto arg0Type = GetType(arg0);
  10676. auto arg1Type = GetType(arg1);
  10677. if ((arg0Type != NULL) && (arg1Type != NULL) &&
  10678. ((arg0Type->IsVector()) || (arg1Type->IsVector())))
  10679. {
  10680. if (((arg0Type->IsVector()) && (arg0Type->mSize == 8)) ||
  10681. ((arg1Type->IsVector()) && (arg1Type->mSize == 8)))
  10682. {
  10683. if (arg0.IsNativeReg())
  10684. {
  10685. return BeMCInstForm_XMM64_RM64;
  10686. }
  10687. else
  10688. {
  10689. return BeMCInstForm_FRM64_XMM64;
  10690. }
  10691. }
  10692. else
  10693. {
  10694. if (arg0.IsNativeReg())
  10695. {
  10696. return BeMCInstForm_XMM128_RM128;
  10697. }
  10698. else
  10699. {
  10700. return BeMCInstForm_FRM128_XMM128;
  10701. }
  10702. }
  10703. }
  10704. else if ((arg0Type != NULL) && (arg1Type != NULL) &&
  10705. ((arg0Type->IsFloat()) || (arg1Type->IsFloat())))
  10706. {
  10707. if (arg0.IsNativeReg())
  10708. {
  10709. if (arg0Type->mTypeCode == BeTypeCode_Double)
  10710. {
  10711. if (arg1.IsImmediate())
  10712. return BeMCInstForm_XMM64_IMM;
  10713. switch (arg1Type->mTypeCode)
  10714. {
  10715. case BeTypeCode_Float: return BeMCInstForm_XMM64_FRM32;
  10716. case BeTypeCode_Double: return BeMCInstForm_XMM64_FRM64;
  10717. case BeTypeCode_Int32: return BeMCInstForm_XMM64_RM32;
  10718. case BeTypeCode_Int64: return BeMCInstForm_XMM64_RM64;
  10719. case BeTypeCode_Pointer: return BeMCInstForm_XMM64_RM64;
  10720. default: NotImpl();
  10721. }
  10722. }
  10723. else if (arg0Type->mTypeCode == BeTypeCode_Float)
  10724. {
  10725. if (arg1.IsImmediate())
  10726. return BeMCInstForm_XMM32_IMM;
  10727. switch (arg1Type->mTypeCode)
  10728. {
  10729. case BeTypeCode_Float: return BeMCInstForm_XMM32_FRM32;
  10730. case BeTypeCode_Double: return BeMCInstForm_XMM32_FRM64;
  10731. case BeTypeCode_Int32: return BeMCInstForm_XMM32_RM32;
  10732. case BeTypeCode_Int64: return BeMCInstForm_XMM32_RM64;
  10733. case BeTypeCode_Pointer: return BeMCInstForm_XMM32_RM64;
  10734. default: NotImpl();
  10735. }
  10736. }
  10737. else if (arg1Type->mTypeCode == BeTypeCode_Double)
  10738. {
  10739. if (arg0Type->mSize == 4)
  10740. return BeMCInstForm_R32_F64;
  10741. else
  10742. return BeMCInstForm_R64_F64;
  10743. }
  10744. else if (arg1Type->mTypeCode == BeTypeCode_Float)
  10745. {
  10746. if (arg0Type->mSize == 4)
  10747. return BeMCInstForm_R32_F32;
  10748. else
  10749. return BeMCInstForm_R64_F32;
  10750. }
  10751. }
  10752. else
  10753. {
  10754. if (arg1.IsImmediate())
  10755. return BeMCInstForm_Unknown;
  10756. if (arg0Type->mTypeCode == BeTypeCode_Double)
  10757. {
  10758. //TODO: This used to INCORRECTLY say 'switch (arg0Type->mTypeCode)'
  10759. switch (arg1Type->mTypeCode)
  10760. {
  10761. case BeTypeCode_Float: return BeMCInstForm_FRM32_XMM64;
  10762. case BeTypeCode_Double: return BeMCInstForm_FRM64_XMM64;
  10763. default: NotImpl();
  10764. }
  10765. }
  10766. else if (arg0Type->mTypeCode == BeTypeCode_Float)
  10767. {
  10768. //TODO: This used to INCORRECTLY say 'switch (arg0Type->mTypeCode)'
  10769. switch (arg1Type->mTypeCode)
  10770. {
  10771. case BeTypeCode_Float: return BeMCInstForm_FRM32_XMM32;
  10772. case BeTypeCode_Double: return BeMCInstForm_FRM64_XMM32;
  10773. default: NotImpl();
  10774. }
  10775. }
  10776. else
  10777. NotImpl();
  10778. }
  10779. }
  10780. if ((arg1.IsImmediate()) && (arg0Type != NULL)) // MOV r/m64, imm32
  10781. {
  10782. int64 maxSize = 8;
  10783. if ((arg0Type->IsInt()) && (arg1.IsImmediateInt()))
  10784. maxSize = arg0Type->mSize;
  10785. if ((arg1.mKind == BeMCOperandKind_Immediate_Null) || (maxSize <= 1) ||
  10786. ((arg1.mImmediate >= -0x80) && (arg1.mImmediate <= 0x7F)))
  10787. {
  10788. switch (arg0Type->mSize)
  10789. {
  10790. case 8: return BeMCInstForm_RM64_IMM8;
  10791. case 4: return BeMCInstForm_RM32_IMM8;
  10792. case 2: return BeMCInstForm_RM16_IMM8;
  10793. case 1: return BeMCInstForm_RM8_IMM8;
  10794. }
  10795. }
  10796. else if ((maxSize <= 2) ||
  10797. ((arg1.mImmediate >= -0x8000) && (arg1.mImmediate <= 0x7FFF)))
  10798. {
  10799. switch (arg0Type->mSize)
  10800. {
  10801. case 8: return BeMCInstForm_RM64_IMM16;
  10802. case 4: return BeMCInstForm_RM32_IMM16;
  10803. case 2: return BeMCInstForm_RM16_IMM16;
  10804. case 1: return BeMCInstForm_RM8_IMM8;
  10805. }
  10806. }
  10807. else if ((maxSize <= 4) ||
  10808. ((arg1.mImmediate >= -0x80000000LL) && (arg1.mImmediate <= 0x7FFFFFFF)))
  10809. {
  10810. switch (arg0Type->mSize)
  10811. {
  10812. case 8: return BeMCInstForm_RM64_IMM32;
  10813. case 4: return BeMCInstForm_RM32_IMM32;
  10814. case 2: return BeMCInstForm_RM16_IMM16;
  10815. case 1: return BeMCInstForm_RM8_IMM8;
  10816. }
  10817. }
  10818. else
  10819. return BeMCInstForm_RM64_IMM64;
  10820. }
  10821. if ((arg0Type == NULL) || (arg1Type == NULL) || (arg0Type->mSize != arg1Type->mSize))
  10822. return BeMCInstForm_Unknown;
  10823. if (arg0.mKind == BeMCOperandKind_NativeReg)
  10824. {
  10825. switch (GetType(arg0)->mSize)
  10826. {
  10827. case 8: return BeMCInstForm_R64_RM64;
  10828. case 4: return BeMCInstForm_R32_RM32;
  10829. case 2: return BeMCInstForm_R16_RM16;
  10830. case 1: return BeMCInstForm_R8_RM8;
  10831. }
  10832. }
  10833. if (arg1.mKind == BeMCOperandKind_NativeReg)
  10834. {
  10835. switch (GetType(arg1)->mSize)
  10836. {
  10837. case 8: return BeMCInstForm_RM64_R64;
  10838. case 4: return BeMCInstForm_RM32_R32;
  10839. case 2: return BeMCInstForm_RM16_R16;
  10840. case 1: return BeMCInstForm_RM8_R8;
  10841. }
  10842. }
  10843. return BeMCInstForm_Unknown;
  10844. }
  10845. BeMCInstForm BeMCContext::ToIMM16(BeMCInstForm instForm)
  10846. {
  10847. switch (instForm)
  10848. {
  10849. case BeMCInstForm_RM16_IMM8: return BeMCInstForm_RM32_IMM16;
  10850. case BeMCInstForm_RM32_IMM8: return BeMCInstForm_RM32_IMM16;
  10851. case BeMCInstForm_RM64_IMM8: return BeMCInstForm_RM64_IMM16;
  10852. case BeMCInstForm_RM16_IMM16: return BeMCInstForm_RM32_IMM16;
  10853. case BeMCInstForm_RM32_IMM16: return BeMCInstForm_RM32_IMM16;
  10854. case BeMCInstForm_RM64_IMM16: return BeMCInstForm_RM64_IMM16;
  10855. }
  10856. NotImpl();
  10857. return instForm;
  10858. }
  10859. BeMCInstForm BeMCContext::ToIMM32OrSmaller(BeMCInstForm instForm)
  10860. {
  10861. switch (instForm)
  10862. {
  10863. case BeMCInstForm_RM8_IMM8: return BeMCInstForm_RM8_IMM8;
  10864. case BeMCInstForm_RM16_IMM8: return BeMCInstForm_RM16_IMM16;
  10865. case BeMCInstForm_RM32_IMM8: return BeMCInstForm_RM32_IMM32;
  10866. case BeMCInstForm_RM64_IMM8: return BeMCInstForm_RM64_IMM32;
  10867. case BeMCInstForm_RM16_IMM16: return BeMCInstForm_RM16_IMM16;
  10868. case BeMCInstForm_RM32_IMM16: return BeMCInstForm_RM32_IMM32;
  10869. case BeMCInstForm_RM64_IMM16: return BeMCInstForm_RM64_IMM32;
  10870. case BeMCInstForm_RM32_IMM32: return BeMCInstForm_RM32_IMM32;
  10871. case BeMCInstForm_RM64_IMM32: return BeMCInstForm_RM64_IMM32;
  10872. }
  10873. NotImpl();
  10874. return instForm;
  10875. }
  10876. void BeMCContext::SetCurrentInst(BeMCInst* inst)
  10877. {
  10878. if (inst == NULL)
  10879. {
  10880. mCurDbgLoc = NULL;
  10881. mCurVRegsInit = NULL;
  10882. mCurVRegsLive = NULL;
  10883. }
  10884. else
  10885. {
  10886. mCurDbgLoc = inst->mDbgLoc;
  10887. mCurVRegsInit = inst->mVRegsInitialized;
  10888. mCurVRegsLive = inst->mLiveness;
  10889. }
  10890. }
  10891. int BeMCContext::FindPreserveVolatiles(BeMCBlock* mcBlock, int instIdx)
  10892. {
  10893. for (int checkIdx = instIdx - 1; checkIdx >= 0; checkIdx--)
  10894. {
  10895. BeMCInst* checkInst = mcBlock->mInstructions[checkIdx];
  10896. if (checkInst->mKind == BeMCInstKind_PreserveVolatiles)
  10897. return checkIdx;
  10898. }
  10899. return -1;
  10900. }
  10901. int BeMCContext::FindRestoreVolatiles(BeMCBlock* mcBlock, int instIdx)
  10902. {
  10903. for (int checkIdx = instIdx + 1; checkIdx < (int)mcBlock->mInstructions.size(); checkIdx++)
  10904. {
  10905. BeMCInst* checkInst = mcBlock->mInstructions[checkIdx];
  10906. if (checkInst->mKind == BeMCInstKind_RestoreVolatiles)
  10907. return checkIdx;
  10908. }
  10909. return mcBlock->mInstructions.size() - 1; // Ret
  10910. }
  10911. void BeMCContext::EmitInstPrefix(BeMCInstForm instForm, BeMCInst* inst)
  10912. {
  10913. switch (instForm)
  10914. {
  10915. case BeMCInstForm_R16_RM16:
  10916. case BeMCInstForm_R16_RM64_ADDR:
  10917. case BeMCInstForm_RM16_R16:
  10918. case BeMCInstForm_RM16_IMM8:
  10919. case BeMCInstForm_RM16_IMM16:
  10920. case BeMCInstForm_RM64_R16_ADDR:
  10921. case BeMCInstForm_RM16:
  10922. case BeMCInstForm_R16:
  10923. Emit(0x66);
  10924. }
  10925. switch (instForm)
  10926. {
  10927. case BeMCInstForm_R64_RM64:
  10928. EmitREX(inst->mArg0, inst->mArg1, true);
  10929. break;
  10930. case BeMCInstForm_R8_RM8:
  10931. case BeMCInstForm_R16_RM16:
  10932. case BeMCInstForm_R32_RM32:
  10933. case BeMCInstForm_R8_RM64_ADDR:
  10934. case BeMCInstForm_R16_RM64_ADDR:
  10935. case BeMCInstForm_R32_RM64_ADDR:
  10936. EmitREX(inst->mArg0, inst->mArg1, false);
  10937. break;
  10938. case BeMCInstForm_R64_RM64_ADDR:
  10939. EmitREX(inst->mArg0, inst->mArg1, true);
  10940. break;
  10941. case BeMCInstForm_RM8_R8:
  10942. case BeMCInstForm_RM16_R16:
  10943. case BeMCInstForm_RM32_R32:
  10944. case BeMCInstForm_RM64_R8_ADDR:
  10945. case BeMCInstForm_RM64_R16_ADDR:
  10946. case BeMCInstForm_RM64_R32_ADDR:
  10947. EmitREX(inst->mArg1, inst->mArg0, false);
  10948. break;
  10949. case BeMCInstForm_RM64_R64:
  10950. case BeMCInstForm_RM64_R64_ADDR:
  10951. EmitREX(inst->mArg1, inst->mArg0, true);
  10952. break;
  10953. case BeMCInstForm_RM8_IMM8:
  10954. case BeMCInstForm_RM16_IMM8:
  10955. case BeMCInstForm_RM16_IMM16:
  10956. case BeMCInstForm_RM32_IMM8:
  10957. case BeMCInstForm_RM32_IMM16:
  10958. case BeMCInstForm_RM32_IMM32:
  10959. EmitREX(inst->mArg1, inst->mArg0, false);
  10960. break;
  10961. case BeMCInstForm_RM64_IMM8:
  10962. case BeMCInstForm_RM64_IMM16:
  10963. case BeMCInstForm_RM64_IMM32:
  10964. case BeMCInstForm_RM64_IMM64:
  10965. EmitREX(inst->mArg1, inst->mArg0, true);
  10966. break;
  10967. }
  10968. }
  10969. void BeMCContext::EmitInst(BeMCInstForm instForm, uint16 codeBytes, BeMCInst* inst)
  10970. {
  10971. EmitInstPrefix(instForm, inst);
  10972. if (codeBytes <= 0xFF)
  10973. mOut.Write((uint8)codeBytes);
  10974. else
  10975. mOut.Write((int16)codeBytes);
  10976. switch (instForm)
  10977. {
  10978. case BeMCInstForm_R8_RM8:
  10979. case BeMCInstForm_R16_RM16:
  10980. case BeMCInstForm_R32_RM32:
  10981. case BeMCInstForm_R64_RM64:
  10982. EmitModRM(inst->mArg0, inst->mArg1);
  10983. break;
  10984. case BeMCInstForm_R8_RM64_ADDR:
  10985. case BeMCInstForm_R16_RM64_ADDR:
  10986. case BeMCInstForm_R32_RM64_ADDR:
  10987. case BeMCInstForm_R64_RM64_ADDR:
  10988. EmitModRM_Addr(inst->mArg0, inst->mArg1);
  10989. break;
  10990. case BeMCInstForm_RM8_R8:
  10991. case BeMCInstForm_RM16_R16:
  10992. case BeMCInstForm_RM32_R32:
  10993. case BeMCInstForm_RM64_R64:
  10994. EmitModRM(inst->mArg1, inst->mArg0);
  10995. break;
  10996. case BeMCInstForm_RM64_R8_ADDR:
  10997. case BeMCInstForm_RM64_R16_ADDR:
  10998. case BeMCInstForm_RM64_R32_ADDR:
  10999. case BeMCInstForm_RM64_R64_ADDR:
  11000. EmitModRM_Addr(inst->mArg1, inst->mArg0);
  11001. break;
  11002. case BeMCInstForm_RM64_IMM32:
  11003. break;
  11004. default:
  11005. SoftFail("Notimpl EmitInst");
  11006. }
  11007. }
  11008. void BeMCContext::EmitInst(BeMCInstForm instForm, uint16 codeBytes, uint8 rx, BeMCInst* inst)
  11009. {
  11010. EmitInstPrefix(instForm, inst);
  11011. BF_ASSERT(codeBytes != 0);
  11012. if (codeBytes <= 0xFF)
  11013. mOut.Write((uint8)codeBytes);
  11014. else
  11015. mOut.Write((int16)codeBytes);
  11016. switch (instForm)
  11017. {
  11018. case BeMCInstForm_RM8_IMM8:
  11019. case BeMCInstForm_RM16_IMM8:
  11020. case BeMCInstForm_RM32_IMM8:
  11021. case BeMCInstForm_RM64_IMM8:
  11022. EmitModRM(rx, inst->mArg0, -1);
  11023. mOut.Write((int8)inst->mArg1.GetImmediateInt());
  11024. break;
  11025. case BeMCInstForm_RM16_IMM16:
  11026. case BeMCInstForm_RM32_IMM16:
  11027. case BeMCInstForm_RM64_IMM16:
  11028. EmitModRM(rx, inst->mArg0, -2);
  11029. mOut.Write((int16)inst->mArg1.GetImmediateInt());
  11030. break;
  11031. case BeMCInstForm_RM32_IMM32:
  11032. case BeMCInstForm_RM64_IMM32:
  11033. EmitModRM(rx, inst->mArg0, -4);
  11034. mOut.Write((int32)inst->mArg1.GetImmediateInt());
  11035. break;
  11036. case BeMCInstForm_RM64_IMM64:
  11037. EmitModRM(rx, inst->mArg0, -8);
  11038. mOut.Write((int64)inst->mArg1.GetImmediateInt());
  11039. break;
  11040. default:
  11041. SoftFail("Notimpl EmitInst");
  11042. }
  11043. }
  11044. void BeMCContext::EmitStdInst(BeMCInstForm instForm, BeMCInst* inst, uint8 opcode_rm_r, uint8 opcode_r_rm, uint8 opcode_rm_imm, uint8 opcode_rm_imm_rx)
  11045. {
  11046. switch (instForm)
  11047. {
  11048. case BeMCInstForm_R8_RM8: EmitInst(instForm, opcode_r_rm - 1, inst); break;
  11049. case BeMCInstForm_R16_RM16:
  11050. case BeMCInstForm_R32_RM32:
  11051. case BeMCInstForm_R64_RM64: EmitInst(instForm, opcode_r_rm, inst); break;
  11052. case BeMCInstForm_RM8_R8: EmitInst(instForm, opcode_rm_r - 1, inst); break;
  11053. case BeMCInstForm_RM16_R16:
  11054. case BeMCInstForm_RM32_R32:
  11055. case BeMCInstForm_RM64_R64: EmitInst(instForm, opcode_rm_r, inst); break;
  11056. // These immediate forms assume an expansion to imm32, OR the register size for 8 and 16 bit registers
  11057. case BeMCInstForm_RM8_IMM8: EmitInst(BeMCInstForm_RM8_IMM8, opcode_rm_imm - 1, opcode_rm_imm_rx, inst); break;
  11058. case BeMCInstForm_RM16_IMM8: EmitInst(BeMCInstForm_RM16_IMM16, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11059. case BeMCInstForm_RM32_IMM8: EmitInst(BeMCInstForm_RM32_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11060. case BeMCInstForm_RM64_IMM8: EmitInst(BeMCInstForm_RM64_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11061. case BeMCInstForm_RM32_IMM16: EmitInst(BeMCInstForm_RM32_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11062. case BeMCInstForm_RM16_IMM16: EmitInst(BeMCInstForm_RM16_IMM16, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11063. case BeMCInstForm_RM64_IMM16: EmitInst(BeMCInstForm_RM64_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11064. case BeMCInstForm_RM32_IMM32: EmitInst(BeMCInstForm_RM32_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11065. case BeMCInstForm_RM64_IMM32: EmitInst(BeMCInstForm_RM64_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11066. default:
  11067. Print();
  11068. NotImpl();
  11069. }
  11070. }
  11071. void BeMCContext::EmitStdInst(BeMCInstForm instForm, BeMCInst * inst, uint8 opcode_rm_r, uint8 opcode_r_rm, uint8 opcode_rm_imm, uint8 opcode_rm_imm_rx, uint8 opcode_rm_imm8, uint8 opcode_rm_imm8_rx)
  11072. {
  11073. switch (instForm)
  11074. {
  11075. case BeMCInstForm_R8_RM8: EmitInst(instForm, opcode_r_rm - 1, inst); break;
  11076. case BeMCInstForm_R16_RM16:
  11077. case BeMCInstForm_R32_RM32:
  11078. case BeMCInstForm_R64_RM64: EmitInst(instForm, opcode_r_rm, inst); break;
  11079. case BeMCInstForm_RM8_R8: EmitInst(instForm, opcode_rm_r - 1, inst); break;
  11080. case BeMCInstForm_RM16_R16:
  11081. case BeMCInstForm_RM32_R32:
  11082. case BeMCInstForm_RM64_R64: EmitInst(instForm, opcode_rm_r, inst); break;
  11083. case BeMCInstForm_RM8_IMM8: EmitInst(BeMCInstForm_RM8_IMM8, opcode_rm_imm - 1, opcode_rm_imm_rx, inst); break;
  11084. case BeMCInstForm_RM16_IMM8: EmitInst(BeMCInstForm_RM16_IMM8, opcode_rm_imm8, opcode_rm_imm8_rx, inst); break;
  11085. case BeMCInstForm_RM32_IMM8: EmitInst(BeMCInstForm_RM32_IMM8, opcode_rm_imm8, opcode_rm_imm8_rx, inst); break;
  11086. case BeMCInstForm_RM64_IMM8: EmitInst(BeMCInstForm_RM64_IMM8, opcode_rm_imm8, opcode_rm_imm8_rx, inst); break;
  11087. // These immediate forms assume an expansion to imm32, OR the register size for 8 and 16 bit registers
  11088. case BeMCInstForm_RM32_IMM16: EmitInst(BeMCInstForm_RM32_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11089. case BeMCInstForm_RM16_IMM16: EmitInst(BeMCInstForm_RM16_IMM16, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11090. case BeMCInstForm_RM64_IMM16: EmitInst(BeMCInstForm_RM64_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11091. case BeMCInstForm_RM32_IMM32: EmitInst(BeMCInstForm_RM32_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11092. case BeMCInstForm_RM64_IMM32: EmitInst(BeMCInstForm_RM64_IMM32, opcode_rm_imm, opcode_rm_imm_rx, inst); break;
  11093. default:
  11094. NotImpl();
  11095. }
  11096. }
  11097. bool BeMCContext::EmitStdXMMInst(BeMCInstForm instForm, BeMCInst* inst, uint8 opcode)
  11098. {
  11099. bool is64Bit = false;
  11100. switch (instForm)
  11101. {
  11102. case BeMCInstForm_R64_F32:
  11103. case BeMCInstForm_XMM32_RM64:
  11104. is64Bit = true;
  11105. // Fallthrough
  11106. case BeMCInstForm_R32_F32:
  11107. case BeMCInstForm_XMM32_RM32:
  11108. case BeMCInstForm_XMM32_IMM:
  11109. case BeMCInstForm_XMM32_FRM32:
  11110. case BeMCInstForm_XMM64_FRM32:
  11111. Emit(0xF3); EmitREX(inst->mArg0, inst->mArg1, is64Bit);
  11112. Emit(0x0F); Emit(opcode);
  11113. EmitModRM(inst->mArg0, inst->mArg1);
  11114. return true;
  11115. case BeMCInstForm_R64_F64:
  11116. case BeMCInstForm_XMM64_RM64:
  11117. is64Bit = true;
  11118. // Fallthrough
  11119. case BeMCInstForm_R32_F64:
  11120. case BeMCInstForm_XMM64_RM32:
  11121. case BeMCInstForm_XMM64_IMM:
  11122. case BeMCInstForm_XMM64_FRM64:
  11123. case BeMCInstForm_XMM32_FRM64:
  11124. {
  11125. auto arg0 = GetFixedOperand(inst->mArg0);
  11126. auto arg1 = GetFixedOperand(inst->mArg1);
  11127. Emit(0xF2); EmitREX(arg0, arg1, is64Bit);
  11128. Emit(0x0F); Emit(opcode);
  11129. EmitModRM(arg0, arg1);
  11130. }
  11131. return true;
  11132. case BeMCInstForm_XMM128_RM128:
  11133. {
  11134. BeTypeCode elemType = BeTypeCode_Float;
  11135. auto arg0 = GetFixedOperand(inst->mArg0);
  11136. auto arg1 = GetFixedOperand(inst->mArg1);
  11137. auto arg0Type = GetType(inst->mArg0);
  11138. auto arg1Type = GetType(inst->mArg1);
  11139. if (arg0Type->IsExplicitVectorType())
  11140. {
  11141. auto vecType = (BeVectorType*)arg0Type;
  11142. elemType = vecType->mElementType->mTypeCode;
  11143. }
  11144. if (arg1Type->IsFloat())
  11145. {
  11146. if (elemType == BeTypeCode_Double)
  11147. Emit(0x66);
  11148. EmitREX(arg1, arg1, false);
  11149. Emit(0x0F); Emit(0xC6); // SHUFPS / SHUFPD
  11150. EmitModRM(arg1, arg1);
  11151. Emit(0);
  11152. }
  11153. if (elemType == BeTypeCode_Double)
  11154. Emit(0x66);
  11155. EmitREX(arg0, arg1, is64Bit);
  11156. Emit(0x0F); Emit(opcode);
  11157. EmitModRM(arg0, arg1);
  11158. return true;
  11159. }
  11160. break;
  11161. case BeMCInstForm_FRM128_XMM128:
  11162. // {
  11163. // Emit(0xF2); EmitREX(inst->mArg0, inst->mArg1, is64Bit);
  11164. // Emit(0x0F); Emit(opcode);
  11165. // EmitModRM(inst->mArg0, inst->mArg1);
  11166. // }
  11167. NOP;
  11168. break;
  11169. }
  11170. return false;
  11171. }
  11172. bool BeMCContext::EmitStdXMMInst(BeMCInstForm instForm, BeMCInst* inst, uint8 opcode, uint8 opcode_dest_frm)
  11173. {
  11174. bool is64Bit = false;
  11175. switch (instForm)
  11176. {
  11177. case BeMCInstForm_FRM32_XMM32:
  11178. {
  11179. auto arg0 = GetFixedOperand(inst->mArg0);
  11180. auto arg1 = GetFixedOperand(inst->mArg1);
  11181. Emit(0xF3); EmitREX(arg1, arg0, is64Bit);
  11182. Emit(0x0F); Emit(opcode_dest_frm);
  11183. EmitModRM(arg1, arg0);
  11184. }
  11185. return true;
  11186. case BeMCInstForm_FRM64_XMM64:
  11187. {
  11188. auto arg0 = GetFixedOperand(inst->mArg0);
  11189. auto arg1 = GetFixedOperand(inst->mArg1);
  11190. Emit(0xF2); EmitREX(arg1, arg0, is64Bit);
  11191. Emit(0x0F); Emit(opcode_dest_frm);
  11192. EmitModRM(arg1, arg0);
  11193. }
  11194. return true;
  11195. default:
  11196. return EmitStdXMMInst(instForm, inst, opcode);
  11197. }
  11198. return false;
  11199. }
  11200. bool BeMCContext::EmitPackedXMMInst(BeMCInstForm instForm, BeMCInst* inst, uint8 opcode)
  11201. {
  11202. bool is64Bit = false;
  11203. switch (instForm)
  11204. {
  11205. case BeMCInstForm_R64_F32:
  11206. case BeMCInstForm_XMM32_RM64:
  11207. is64Bit = true;
  11208. // Fallthrough
  11209. case BeMCInstForm_R32_F32:
  11210. case BeMCInstForm_XMM32_RM32:
  11211. case BeMCInstForm_XMM32_IMM:
  11212. case BeMCInstForm_XMM32_FRM32:
  11213. case BeMCInstForm_XMM64_FRM32: // CVTSS2SD
  11214. {
  11215. auto arg0 = GetFixedOperand(inst->mArg0);
  11216. auto arg1 = GetFixedOperand(inst->mArg1);
  11217. EmitREX(arg0, arg1, is64Bit);
  11218. Emit(0x0F); Emit(opcode);
  11219. EmitModRM(arg0, arg1);
  11220. }
  11221. return true;
  11222. case BeMCInstForm_R64_F64:
  11223. case BeMCInstForm_XMM64_RM64:
  11224. is64Bit = true;
  11225. // Fallthrough
  11226. case BeMCInstForm_R32_F64:
  11227. case BeMCInstForm_XMM64_RM32:
  11228. case BeMCInstForm_XMM64_IMM:
  11229. case BeMCInstForm_XMM64_FRM64:
  11230. case BeMCInstForm_XMM32_FRM64: // CVTSD2SS
  11231. {
  11232. auto arg0 = GetFixedOperand(inst->mArg0);
  11233. auto arg1 = GetFixedOperand(inst->mArg1);
  11234. EmitREX(arg0, arg1, is64Bit);
  11235. Emit(0x0F); Emit(opcode);
  11236. EmitModRM(arg0, arg1);
  11237. }
  11238. return true;
  11239. }
  11240. return false;
  11241. }
  11242. BeMCOperand BeMCContext::IntXMMGetPacked(BeMCOperand arg, BeVectorType* vecType)
  11243. {
  11244. auto argType = GetType(arg);
  11245. if (!argType->IsVector())
  11246. {
  11247. BeMCOperand xmm15;
  11248. xmm15.mReg = X64Reg_M128_XMM15;
  11249. xmm15.mKind = BeMCOperandKind_NativeReg;
  11250. if (arg.IsImmediate())
  11251. {
  11252. BeMCOperand immOperand;
  11253. immOperand.mKind = BeMCOperandKind_Immediate_int32x4;
  11254. immOperand.mImmediate = arg.mImmediate;
  11255. Emit(0xF3);
  11256. EmitREX(xmm15, immOperand, false);
  11257. Emit(0x0F); Emit(0x6F); // MOVDQU
  11258. EmitModRM(xmm15, immOperand);
  11259. }
  11260. else
  11261. {
  11262. Emit(0x66);
  11263. EmitREX(xmm15, arg, false);
  11264. Emit(0x0F); Emit(0x6E); // MOVD
  11265. EmitModRM(xmm15, arg);
  11266. if (vecType->mElementType->mTypeCode == BeTypeCode_Int16)
  11267. Emit(0xF2);
  11268. else
  11269. Emit(0x66);
  11270. EmitREX(xmm15, xmm15, false);
  11271. Emit(0x0F);
  11272. switch (vecType->mElementType->mTypeCode)
  11273. {
  11274. case BeTypeCode_Int8:
  11275. Emit(0x38); Emit(0x00); // PSHUFB
  11276. break;
  11277. case BeTypeCode_Int16:
  11278. Emit(0x70); // PSHUFW
  11279. break;
  11280. case BeTypeCode_Int32:
  11281. Emit(0x70); // PSHUFD
  11282. break;
  11283. }
  11284. EmitModRM(xmm15, xmm15);
  11285. Emit(0);
  11286. }
  11287. arg = xmm15;
  11288. }
  11289. return arg;
  11290. }
  11291. bool BeMCContext::EmitIntXMMInst(BeMCInstForm instForm, BeMCInst* inst, uint8 opcode)
  11292. {
  11293. if (instForm != BeMCInstForm_XMM128_RM128)
  11294. return false;
  11295. auto arg0 = GetFixedOperand(inst->mArg0);
  11296. auto arg1 = GetFixedOperand(inst->mArg1);
  11297. auto arg0Type = GetType(inst->mArg0);
  11298. auto arg1Type = GetType(inst->mArg1);
  11299. if (arg0Type->IsExplicitVectorType())
  11300. {
  11301. auto vecType = (BeVectorType*)arg0Type;
  11302. if ((vecType->mElementType->mTypeCode == BeTypeCode_Int8) ||
  11303. (vecType->mElementType->mTypeCode == BeTypeCode_Int16) ||
  11304. (vecType->mElementType->mTypeCode == BeTypeCode_Int32))
  11305. {
  11306. arg1 = IntXMMGetPacked(arg1, vecType);
  11307. Emit(0x66);
  11308. EmitREX(arg0, arg1, false);
  11309. Emit(0x0F);
  11310. switch (vecType->mElementType->mTypeCode)
  11311. {
  11312. case BeTypeCode_Int8:
  11313. Emit(opcode);
  11314. break;
  11315. case BeTypeCode_Int16:
  11316. Emit(opcode + 1);
  11317. break;
  11318. case BeTypeCode_Int32:
  11319. Emit(opcode + 2);
  11320. break;
  11321. }
  11322. EmitModRM(arg0, arg1);
  11323. return true;
  11324. }
  11325. }
  11326. return false;
  11327. }
  11328. bool BeMCContext::EmitIntBitwiseXMMInst(BeMCInstForm instForm, BeMCInst* inst, uint8 opcode)
  11329. {
  11330. if (instForm != BeMCInstForm_XMM128_RM128)
  11331. return false;
  11332. auto arg0 = GetFixedOperand(inst->mArg0);
  11333. auto arg1 = GetFixedOperand(inst->mArg1);
  11334. auto arg0Type = GetType(inst->mArg0);
  11335. if (arg0Type->IsExplicitVectorType())
  11336. {
  11337. auto vecType = (BeVectorType*)arg0Type;
  11338. if ((vecType->mElementType->mTypeCode == BeTypeCode_Int8) ||
  11339. (vecType->mElementType->mTypeCode == BeTypeCode_Int16) ||
  11340. (vecType->mElementType->mTypeCode == BeTypeCode_Int32))
  11341. {
  11342. arg1 = IntXMMGetPacked(arg1, vecType);
  11343. Emit(0x66);
  11344. EmitREX(arg0, arg1, false);
  11345. Emit(0x0F);
  11346. Emit(opcode); // PXOR
  11347. EmitModRM(arg0, arg1);
  11348. return true;
  11349. }
  11350. }
  11351. return false;
  11352. }
  11353. void BeMCContext::EmitAggMov(const BeMCOperand& dest, const BeMCOperand& src)
  11354. {
  11355. BeRMParamsInfo rmInfo;
  11356. GetRMParams(dest, rmInfo);
  11357. BF_ASSERT(src.mKind == BeMCOperandKind_ConstAgg);
  11358. auto aggConstant = src.mConstant;
  11359. BeConstData constData;
  11360. aggConstant->GetData(constData);
  11361. Array<uint8>& dataVec = constData.mData;
  11362. int memSize = dataVec.size();
  11363. int curOfs = 0;
  11364. bool allowRep = dataVec.size() >= BF_REP_MOV_LIMIT;
  11365. union IntUnion
  11366. {
  11367. int64 mInt64;
  11368. int32 mInt32;
  11369. int16 mInt16;
  11370. int8 mInt8;
  11371. };
  11372. bool isValRegSet = false;
  11373. int64 curVal = 0;
  11374. auto _EmitMovR11 = [&](int64 val)
  11375. {
  11376. int64 prevVal = curVal;
  11377. bool wasValRegSet = isValRegSet;
  11378. isValRegSet = true;
  11379. curVal = val;
  11380. if ((wasValRegSet) && (val == prevVal))
  11381. return;
  11382. if (val == 0)
  11383. {
  11384. // xor r11, r11
  11385. Emit(0x4D); Emit(0x31); Emit(0xDB);
  11386. return;
  11387. }
  11388. if (wasValRegSet)
  11389. {
  11390. if ((val & ~0xFF) == (prevVal & ~0xFF))
  11391. {
  11392. // mov r11b, val8
  11393. Emit(0x41); Emit(0xB3);
  11394. Emit((int8)val);
  11395. return;
  11396. }
  11397. if ((val & ~0xFFFF) == (prevVal & ~0xFFFF))
  11398. {
  11399. // mov r11w, val16
  11400. Emit(0x66); Emit(0x41); Emit(0xBB);
  11401. mOut.Write((int16)val);
  11402. return;
  11403. }
  11404. }
  11405. if ((val >= 0) && (val <= 0x7FFFFFFF))
  11406. {
  11407. // mov r11d, val32
  11408. Emit(0x41); Emit(0xBB);
  11409. mOut.Write((int32)val);
  11410. }
  11411. else if ((val >= -0x80000000LL) && (val <= 0x7FFFFFFF))
  11412. {
  11413. // mov r11, val32 (sign extended)
  11414. Emit(0x49); Emit(0xC7); Emit(0xC3);
  11415. mOut.Write((int32)val);
  11416. }
  11417. else
  11418. {
  11419. // movabs r11, val64
  11420. Emit(0x49); Emit(0xBB);
  11421. mOut.Write((int64)val);
  11422. }
  11423. };
  11424. for (; curOfs <= memSize - 8; )
  11425. {
  11426. if (allowRep)
  11427. {
  11428. uint8 val = dataVec[curOfs];
  11429. int repSize = 0;
  11430. int checkOfs = curOfs;
  11431. while (checkOfs < (int)dataVec.size())
  11432. {
  11433. if (dataVec[checkOfs] != val)
  11434. break;
  11435. checkOfs++;
  11436. repSize++;
  11437. }
  11438. if (repSize >= 16)
  11439. {
  11440. bool regSaved = false;
  11441. if ((rmInfo.mRegA == X64Reg_RAX) ||
  11442. (rmInfo.mRegA == X64Reg_RCX) ||
  11443. (rmInfo.mRegA == X64Reg_RDI))
  11444. {
  11445. BF_ASSERT(rmInfo.mRegB == X64Reg_None);
  11446. // mov R11, regA
  11447. Emit(0x49); Emit(0x89);
  11448. EmitModRM(BeMCOperand::FromReg(rmInfo.mRegA), BeMCOperand::FromReg(X64Reg_R11));
  11449. regSaved = true;
  11450. }
  11451. // lea rdi, <dest+curOfs>
  11452. EmitREX(BeMCOperand::FromReg(X64Reg_RDI), dest, true);
  11453. Emit(0x8D);
  11454. EmitModRMRel(EncodeRegNum(X64Reg_RDI), rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp + curOfs);
  11455. // mov al, <val>
  11456. Emit(0xB0); Emit(val);
  11457. // mov edx, <repSize>
  11458. Emit(0xB9);
  11459. mOut.Write((int32)repSize);
  11460. // rep stosb
  11461. Emit(0xF3); Emit(0xAA);
  11462. if (regSaved)
  11463. {
  11464. // mov regA, R11
  11465. Emit(0x4C); Emit(0x89);
  11466. EmitModRM(BeMCOperand::FromReg(X64Reg_R11), BeMCOperand::FromReg(rmInfo.mRegA));
  11467. }
  11468. curOfs += repSize;
  11469. continue;
  11470. }
  11471. }
  11472. int64 newValReg = *((int64*)&dataVec[curOfs]);
  11473. _EmitMovR11(newValReg);
  11474. // mov <dest+curOfs>, R11
  11475. EmitREX(BeMCOperand::FromReg(X64Reg_R11), dest, true);
  11476. Emit(0x89);
  11477. EmitModRMRel(EncodeRegNum(X64Reg_R11), rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp + curOfs);
  11478. curOfs += 8;
  11479. }
  11480. for (; curOfs <= memSize - 4; curOfs += 4)
  11481. {
  11482. int64 newValReg = curVal;
  11483. *((int32*)&newValReg) = *((int32*)&dataVec[curOfs]);
  11484. _EmitMovR11(newValReg);
  11485. // mov <dest+curOfs>, R11d
  11486. EmitREX(BeMCOperand::FromReg(X64Reg_R11D), dest, false);
  11487. Emit(0x89);
  11488. EmitModRMRel(EncodeRegNum(X64Reg_R11D), rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp + curOfs);
  11489. }
  11490. for (; curOfs <= memSize - 2; curOfs += 2)
  11491. {
  11492. int64 newValReg = curVal;
  11493. *((int16*)&newValReg) = *((int16*)&dataVec[curOfs]);
  11494. _EmitMovR11(newValReg);
  11495. // mov <dest+curOfs>, R11w
  11496. Emit(0x66); EmitREX(BeMCOperand::FromReg(X64Reg_R11W), dest, false);
  11497. Emit(0x89);
  11498. EmitModRMRel(EncodeRegNum(X64Reg_R11W), rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp + curOfs);
  11499. }
  11500. for (; curOfs <= memSize - 1; curOfs += 1)
  11501. {
  11502. int64 newValReg = curVal;
  11503. *((int8*)&newValReg) = *((int8*)&dataVec[curOfs]);
  11504. _EmitMovR11(newValReg);
  11505. // mov <dest+curOfs>, R11b
  11506. EmitREX(BeMCOperand::FromReg(X64Reg_R11B), dest, false);
  11507. Emit(0x89 - 1);
  11508. EmitModRMRel(EncodeRegNum(X64Reg_R11B), rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp + curOfs);
  11509. }
  11510. for (auto constVal : constData.mConsts)
  11511. {
  11512. BeMCRelocation reloc;
  11513. // movabs r11, val64
  11514. Emit(0x49); Emit(0xBB);
  11515. reloc.mKind = BeMCRelocationKind_ADDR64;
  11516. reloc.mOffset = mOut.GetPos();
  11517. auto operand = GetOperand(constVal.mConstant);
  11518. reloc.mSymTableIdx = operand.mSymbolIdx;
  11519. mCOFFObject->mTextSect.mRelocs.push_back(reloc);
  11520. mTextRelocs.push_back((int)mCOFFObject->mTextSect.mRelocs.size() - 1);
  11521. mOut.Write((int64)0);
  11522. // mov <dest+curOfs>, R11
  11523. EmitREX(BeMCOperand::FromReg(X64Reg_R11), dest, true);
  11524. Emit(0x89);
  11525. EmitModRMRel(EncodeRegNum(X64Reg_R11), rmInfo.mRegA, rmInfo.mRegB, rmInfo.mBScale, rmInfo.mDisp + constVal.mIdx);
  11526. }
  11527. }
  11528. void BeMCContext::DoCodeEmission()
  11529. {
  11530. BP_ZONE("BeMCContext::DoCodeEmission");
  11531. struct BeMCJump
  11532. {
  11533. public:
  11534. int mCodeOffset;
  11535. int mLabelIdx;
  11536. int mJumpKind; // 0 = rel8, 1 = rel16 (not valid in x64), 2 = rel32
  11537. BeCmpKind mCmpKind;
  11538. public:
  11539. BeMCJump()
  11540. {
  11541. mCmpKind = BeCmpKind_None;
  11542. mCodeOffset = -1;
  11543. mLabelIdx = -1;
  11544. mJumpKind = -1;
  11545. }
  11546. };
  11547. SizedArray<int, 64> labelPositions;
  11548. labelPositions.resize(mCurLabelIdx);
  11549. for (int labelIdx = 0; labelIdx < mCurLabelIdx; labelIdx++)
  11550. labelPositions[labelIdx] = -1;
  11551. SizedArray<BeMCJump, 64> deferredJumps;
  11552. auto& xdata = mCOFFObject->mXDataSect.mData;
  11553. int xdataStartPos = xdata.GetPos();
  11554. int textSectStartPos = mOut.GetPos();
  11555. int lastLabelIdx = -1;
  11556. bool hasPData = mStackSize != 0;
  11557. if (hasPData)
  11558. {
  11559. // xdata must be DWORD aligned
  11560. while ((xdataStartPos & 3) != 0)
  11561. {
  11562. xdata.Write((uint8)0);
  11563. xdataStartPos++;
  11564. }
  11565. // pdata starting loc
  11566. BeMCRelocation reloc;
  11567. reloc.mKind = BeMCRelocationKind_ADDR32NB;
  11568. reloc.mOffset = mCOFFObject->mPDataSect.mData.GetPos();
  11569. reloc.mSymTableIdx = mCOFFObject->GetSymbol(mBeFunction)->mIdx;
  11570. mCOFFObject->mPDataSect.mRelocs.push_back(reloc);
  11571. mCOFFObject->mPDataSect.mData.Write((int32)0);
  11572. xdata.Write((uint8)1); // version
  11573. xdata.Write((uint8)0); // prolog size (placeholder)
  11574. xdata.Write((uint8)0); // num unwind codes (placeholder)
  11575. if (mUseBP)
  11576. {
  11577. int frameReg = 5;
  11578. int offset = 0; // scaled by 16
  11579. xdata.Write((uint8)(frameReg + (offset << 4))); // frame register
  11580. }
  11581. else
  11582. xdata.Write((uint8)0); // frame register
  11583. }
  11584. struct BeMCDeferredUnwind
  11585. {
  11586. BeMCInst* mUnwindInst;
  11587. int mCodePos;
  11588. };
  11589. struct BeDbgInstPos
  11590. {
  11591. int mPos;
  11592. int mOrigPos;
  11593. BeMCInst* mInst;
  11594. };
  11595. SizedArray<BeDbgInstPos, 64> dbgInstPositions;
  11596. String dbgStr;
  11597. if (mDebugging)
  11598. {
  11599. dbgStr += "DoCodeEmission:\n";
  11600. }
  11601. SizedArray<BeMCDeferredUnwind, 64> deferredUnwinds;
  11602. mCurDbgLoc = NULL;
  11603. bool needsInstructionAtDbgLoc = false;
  11604. bool dbgExtendLifetime = false;
  11605. int hotJumpLen = 5;
  11606. int funcCodePos = 0;
  11607. Array<BeDbgVariable*> deferredGapEnds;
  11608. BeVTrackingList* vregsLive = mLivenessContext.AllocEmptyList();
  11609. BeVTrackingList* vregsInitialized = mVRegInitializedContext.AllocEmptyList();
  11610. for (auto mcBlock : mBlocks)
  11611. {
  11612. for (auto inst : mcBlock->mInstructions)
  11613. {
  11614. mActiveInst = inst;
  11615. if (mDebugging)
  11616. {
  11617. ToString(inst, dbgStr, true, true);
  11618. }
  11619. auto checkInst = inst;
  11620. int newFuncCodePos = mOut.GetPos() - textSectStartPos;
  11621. if (newFuncCodePos != funcCodePos)
  11622. {
  11623. funcCodePos = newFuncCodePos;
  11624. if (dbgExtendLifetime)
  11625. {
  11626. dbgExtendLifetime = false;
  11627. if (mDebugging)
  11628. {
  11629. //
  11630. }
  11631. }
  11632. }
  11633. while (!deferredGapEnds.IsEmpty())
  11634. {
  11635. BeDbgVariable* dbgVar = deferredGapEnds.back();
  11636. deferredGapEnds.pop_back();
  11637. auto& range = dbgVar->mGaps.back();
  11638. range.mLength = funcCodePos - range.mOffset;
  11639. if (range.mLength <= 0)
  11640. {
  11641. dbgVar->mGaps.pop_back();
  11642. }
  11643. }
  11644. if ((inst->mKind == BeMCInstKind_LifetimeEnd) || (inst->mKind == BeMCInstKind_LifetimeExtend))
  11645. {
  11646. if (needsInstructionAtDbgLoc)
  11647. {
  11648. // This 'dbgExtendLifetime' is useful in cases like
  11649. // repeat { int a = 123; } while (false);
  11650. // Otherwise 'a' wouldn't be viewable
  11651. dbgExtendLifetime = true;
  11652. }
  11653. }
  11654. switch (inst->mKind)
  11655. {
  11656. case BeMCInstKind_Label:
  11657. //Why did we set this FALSE here? Just because we have a label doesn't mean we've emitted any actual instructions....
  11658. // Leaving this out broke a test where we couldn't view a 'this' at the end of a particular method
  11659. //dbgExtendLifetime = false;
  11660. break;
  11661. }
  11662. if ((inst->mDbgLoc != NULL) && (mDbgFunction != NULL))
  11663. {
  11664. if (inst->mDbgLoc != mCurDbgLoc)
  11665. {
  11666. bool allowEmission = true;
  11667. if (inst->mKind == BeMCInstKind_DbgDecl)
  11668. {
  11669. // We need to separate out dbgDecls if we are attempting to extend a previous one,
  11670. // otherwise wait for a real instruction
  11671. if (dbgExtendLifetime)
  11672. {
  11673. mOut.Write((uint8)0x90);
  11674. funcCodePos++;
  11675. needsInstructionAtDbgLoc = false;
  11676. dbgExtendLifetime = false;
  11677. }
  11678. else
  11679. allowEmission = false;
  11680. }
  11681. else
  11682. {
  11683. mCurDbgLoc = inst->mDbgLoc;
  11684. if ((!mDbgFunction->mEmissions.empty()) && (mDbgFunction->mEmissions.back().mPos == funcCodePos))
  11685. {
  11686. bool hadEmission = false;
  11687. if (needsInstructionAtDbgLoc)
  11688. {
  11689. // We only need an instruction on this LINE, not necessarily the same column or lexical scope
  11690. auto prevDbgLoc = mDbgFunction->mEmissions.back().mDbgLoc;
  11691. if ((prevDbgLoc->mLine != mCurDbgLoc->mLine) ||
  11692. (prevDbgLoc->GetDbgFile() != mCurDbgLoc->GetDbgFile()) ||
  11693. ((prevDbgLoc->mColumn != -1) != (mCurDbgLoc->mColumn != -1)))
  11694. {
  11695. // We didn't emit anything so we need to put a NOP in
  11696. mOut.Write((uint8)0x90);
  11697. funcCodePos++;
  11698. needsInstructionAtDbgLoc = false;
  11699. hadEmission = true;
  11700. dbgExtendLifetime = false;
  11701. }
  11702. }
  11703. if (!hadEmission)
  11704. {
  11705. // No code emitted, take back the last codeEmission
  11706. mDbgFunction->mEmissions.pop_back();
  11707. }
  11708. }
  11709. }
  11710. if (allowEmission)
  11711. {
  11712. BeDbgCodeEmission codeEmission;
  11713. codeEmission.mDbgLoc = mCurDbgLoc;
  11714. codeEmission.mPos = funcCodePos;
  11715. mDbgFunction->mEmissions.push_back(codeEmission);
  11716. }
  11717. }
  11718. }
  11719. BeMCJump jump;
  11720. auto instForm = GetInstForm(inst);
  11721. auto arg0 = GetFixedOperand(inst->mArg0);
  11722. auto arg1 = GetFixedOperand(inst->mArg1);
  11723. auto arg0Type = GetType(inst->mArg0);
  11724. auto arg1Type = GetType(inst->mArg1);
  11725. // Generate gaps
  11726. if ((inst->mVRegsInitialized != NULL) && (vregsInitialized != inst->mVRegsInitialized))
  11727. {
  11728. for (int changeIdx = 0; changeIdx < inst->mVRegsInitialized->mNumChanges; changeIdx++)
  11729. {
  11730. int vregInitializedIdx = inst->mVRegsInitialized->GetChange(changeIdx);
  11731. if (vregInitializedIdx >= 0) // Added
  11732. {
  11733. if (vregInitializedIdx >= mVRegInitializedContext.mNumItems)
  11734. continue;
  11735. auto dbgVar = mVRegInfo[vregInitializedIdx]->mDbgVariable;
  11736. if ((dbgVar != NULL) && (!dbgVar->mDbgLifeEnded))
  11737. {
  11738. // Wasn't initialized, now it is
  11739. if (!dbgVar->mGaps.empty())
  11740. {
  11741. auto& range = dbgVar->mGaps.back();
  11742. if (dbgVar->mDeclLifetimeExtend)
  11743. {
  11744. if (mDebugging)
  11745. {
  11746. dbgStr += StrFormat("#### %d Dbg Applied LifetimeExtend %s from %d to %d\n", funcCodePos, dbgVar->mName.c_str(), dbgVar->mDeclEnd, range.mOffset);
  11747. }
  11748. range.mOffset++;
  11749. if (range.mOffset > dbgVar->mDeclEnd)
  11750. dbgVar->mDeclEnd = range.mOffset;
  11751. dbgVar->mDeclLifetimeExtend = false;
  11752. }
  11753. // We have to check for this because it's possible we get multiple adds
  11754. if (range.mLength == -1)
  11755. {
  11756. deferredGapEnds.Add(dbgVar);
  11757. }
  11758. if (mDebugging)
  11759. {
  11760. dbgStr += StrFormat("#### %d Dbg End Gap %s\n", funcCodePos, dbgVar->mName.c_str());
  11761. }
  11762. }
  11763. }
  11764. }
  11765. else // Removed
  11766. {
  11767. vregInitializedIdx = -vregInitializedIdx - 1;
  11768. if (vregInitializedIdx >= mVRegInitializedContext.mNumItems)
  11769. continue;
  11770. auto dbgVar = mVRegInfo[vregInitializedIdx]->mDbgVariable;
  11771. if ((dbgVar != NULL) && (!dbgVar->mDbgLifeEnded))
  11772. {
  11773. bool inGap = false;
  11774. if (!dbgVar->mGaps.empty())
  11775. inGap = dbgVar->mGaps.back().mLength == -1;
  11776. if (!inGap)
  11777. {
  11778. if (mDebugging)
  11779. {
  11780. dbgStr += StrFormat("#### %d Dbg Start Gap %s\n", funcCodePos, dbgVar->mName.c_str());
  11781. }
  11782. if (dbgVar->mDeclStart == -1)
  11783. {
  11784. // Ignore
  11785. }
  11786. else
  11787. {
  11788. // Starting a new gap
  11789. BeDbgVariableRange range;
  11790. range.mOffset = funcCodePos;
  11791. range.mLength = -1;
  11792. dbgVar->mGaps.push_back(range);
  11793. }
  11794. }
  11795. }
  11796. }
  11797. }
  11798. vregsInitialized = inst->mVRegsInitialized;
  11799. }
  11800. // Finish range for variables exiting liveness. Since liveness is built in end-to-start order,
  11801. // an "add" means that the NEXT instruction won't have this entry (if the next entry doesn't share
  11802. // the exact liveness value)
  11803. if ((inst->mLiveness != NULL) && (inst->mLiveness != vregsLive))
  11804. {
  11805. for (int changeIdx = 0; changeIdx < vregsLive->mNumChanges; changeIdx++)
  11806. {
  11807. int vregLiveIdx = vregsLive->GetChange(changeIdx);
  11808. if (vregLiveIdx < 0)
  11809. continue;
  11810. // Check for both the 'exists' flag changing and the 'in scope' flag changing
  11811. int vregIdx = vregLiveIdx % mLivenessContext.mNumItems;
  11812. auto dbgVar = mVRegInfo[vregIdx]->mDbgVariable;
  11813. if ((dbgVar != NULL) && (!dbgVar->mDbgLifeEnded))
  11814. {
  11815. //if (!mLivenessContext.IsSet(inst->mLiveness, vregIdx))
  11816. {
  11817. if (!dbgVar->mGaps.empty())
  11818. {
  11819. auto& lastGap = dbgVar->mGaps.back();
  11820. if (lastGap.mLength == -1)
  11821. {
  11822. // Unused gap
  11823. //dbgVar->mGaps.pop_back();
  11824. // Don't remove this 'unused gap' because the variable may come back alive
  11825. // This can happen in blocks after NoReturn calls
  11826. }
  11827. }
  11828. if (dbgVar->mDeclStart != -1)
  11829. {
  11830. dbgVar->mDeclEnd = funcCodePos;
  11831. if (mDebugging)
  11832. {
  11833. dbgStr += StrFormat("#### %d Dbg Setting DeclEnd %s\n", funcCodePos, dbgVar->mName.c_str());
  11834. }
  11835. }
  11836. if ((mDebugging) && (dbgExtendLifetime))
  11837. {
  11838. dbgStr += StrFormat("#### %d Dbg Setting LifetimeExtend %s\n", funcCodePos, dbgVar->mName.c_str());
  11839. }
  11840. dbgVar->mDeclLifetimeExtend = dbgExtendLifetime;
  11841. //if (dbgExtendLifetime)
  11842. //dbgVar->mDeclEnd++;
  11843. }
  11844. }
  11845. }
  11846. vregsLive = inst->mLiveness;
  11847. }
  11848. if (mDebugging)
  11849. {
  11850. BeDbgInstPos dbgInstPos = { funcCodePos, funcCodePos, inst };
  11851. dbgInstPositions.push_back(dbgInstPos);
  11852. }
  11853. switch (inst->mKind)
  11854. {
  11855. case BeMCInstKind_DbgDecl:
  11856. {
  11857. auto vregInfo = GetVRegInfo(inst->mArg0);
  11858. auto dbgVar = vregInfo->mDbgVariable;
  11859. dbgVar->mDeclStart = funcCodePos;
  11860. while (vregInfo->IsDirectRelTo())
  11861. {
  11862. if (vregInfo->mRelTo.IsNativeReg())
  11863. {
  11864. break;
  11865. }
  11866. else
  11867. vregInfo = GetVRegInfo(vregInfo->mRelTo);
  11868. }
  11869. BeMCOperand defArg = inst->mArg0;
  11870. if ((vregInfo->mIsRetVal) && (mCompositeRetVRegIdx != -1) && (mCompositeRetVRegIdx != inst->mArg0.mVRegIdx))
  11871. {
  11872. defArg = BeMCOperand::FromVReg(mCompositeRetVRegIdx);
  11873. if ((inst->mArg0.mKind == BeMCOperandKind_VReg) && (vregInfo->mType->IsNonVectorComposite()))
  11874. defArg.mKind = BeMCOperandKind_VRegLoad;
  11875. vregInfo = mVRegInfo[mCompositeRetVRegIdx];
  11876. }
  11877. if (vregInfo->mReg != X64Reg_None)
  11878. {
  11879. if ((defArg.mKind == BeMCOperandKind_VRegAddr) || (dbgVar->mIsValue))
  11880. dbgVar->mPrimaryLoc.mKind = BeDbgVariableLoc::Kind_Reg;
  11881. else
  11882. dbgVar->mPrimaryLoc.mKind = BeDbgVariableLoc::Kind_Indexed;
  11883. dbgVar->mPrimaryLoc.mReg = vregInfo->mReg;
  11884. dbgVar->mPrimaryLoc.mOfs = 0;
  11885. }
  11886. else if (vregInfo->mRelTo.IsNativeReg())
  11887. {
  11888. if ((defArg.mKind == BeMCOperandKind_VRegAddr) || (dbgVar->mIsValue))
  11889. dbgVar->mPrimaryLoc.mKind = BeDbgVariableLoc::Kind_Reg;
  11890. else
  11891. dbgVar->mPrimaryLoc.mKind = BeDbgVariableLoc::Kind_Indexed;
  11892. dbgVar->mPrimaryLoc.mReg = vregInfo->mRelTo.mReg;
  11893. if (vregInfo->mRelOffset.IsImmediateInt())
  11894. {
  11895. dbgVar->mPrimaryLoc.mOfs = vregInfo->mRelOffset.mImmediate;
  11896. }
  11897. else
  11898. BF_ASSERT(!vregInfo->mRelOffset);
  11899. }
  11900. else if (vregInfo->mRelTo.mKind == BeMCOperandKind_SymbolAddr)
  11901. {
  11902. SoftFail("Not supported SymbolAddr");
  11903. //dbgVar->mPrimaryLoc.mKind = BeDbgVariableLoc::Kind_SymbolAddr;
  11904. //dbgVar->mPrimaryLoc.mOfs = vregInfo->mRelTo.mSymbolIdx;
  11905. }
  11906. if ((dbgVar->mPrimaryLoc.mKind == BeDbgVariableLoc::Kind_None) && (vregInfo->mType->mSize != 0))
  11907. {
  11908. dbgVar->mPrimaryLoc.mKind = BeDbgVariableLoc::Kind_Indexed;
  11909. GetValAddr(BeMCOperand::FromVRegAddr(inst->mArg0.mVRegIdx), dbgVar->mPrimaryLoc.mReg, dbgVar->mPrimaryLoc.mOfs);
  11910. }
  11911. if (!mVRegInitializedContext.IsSet(vregsInitialized, inst->mArg0.mVRegIdx))
  11912. {
  11913. bool inGap = false;
  11914. if (!dbgVar->mGaps.empty())
  11915. inGap = dbgVar->mGaps.back().mLength == -1;
  11916. if (!inGap)
  11917. {
  11918. if (mDebugging)
  11919. {
  11920. dbgStr += StrFormat(" Dbg Starting Gap %s\n", dbgVar->mName.c_str());
  11921. }
  11922. BeDbgVariableRange range;
  11923. range.mOffset = funcCodePos;
  11924. range.mLength = -1;
  11925. dbgVar->mGaps.push_back(range);
  11926. }
  11927. }
  11928. if (vregInfo->mVolatileVRegSave != -1)
  11929. {
  11930. auto savedVRegInfo = mVRegInfo[vregInfo->mVolatileVRegSave];
  11931. dbgVar->mSavedLoc.mKind = BeDbgVariableLoc::Kind_Indexed;
  11932. GetValAddr(BeMCOperand::FromVRegAddr(vregInfo->mVolatileVRegSave), dbgVar->mSavedLoc.mReg, dbgVar->mSavedLoc.mOfs);
  11933. }
  11934. }
  11935. break;
  11936. case BeMCInstKind_LifetimeExtend:
  11937. break;
  11938. case BeMCInstKind_LifetimeStart:
  11939. break;
  11940. case BeMCInstKind_LifetimeEnd:
  11941. break;
  11942. case BeMCInstKind_LifetimeSoftEnd:
  11943. {
  11944. auto vregInfo = GetVRegInfo(inst->mArg0);
  11945. if ((vregInfo != NULL) && (vregInfo->mDbgVariable != NULL))
  11946. {
  11947. auto dbgVar = vregInfo->mDbgVariable;
  11948. if (dbgVar->mDeclStart != -1)
  11949. {
  11950. dbgVar->mDeclEnd = funcCodePos;
  11951. dbgVar->mDeclLifetimeExtend = false;
  11952. dbgVar->mDbgLifeEnded = true;
  11953. BF_ASSERT((uint)dbgVar->mDeclEnd >= (uint)dbgVar->mDeclStart);
  11954. }
  11955. }
  11956. }
  11957. break;
  11958. case BeMCInstKind_ValueScopeSoftEnd:
  11959. break;
  11960. case BeMCInstKind_ValueScopeHardEnd:
  11961. break;
  11962. case BeMCInstKind_MemSet:
  11963. {
  11964. // This doesn't currently attempt to align properly
  11965. int memSize = inst->mArg0.mMemSetInfo.mSize;
  11966. uint8 memValue = inst->mArg0.mMemSetInfo.mValue;
  11967. int curOfs = 0;
  11968. X64CPURegister destReg = X64Reg_R10;
  11969. int destOfs = 0;
  11970. if (inst->mArg1.IsNativeReg())
  11971. {
  11972. destReg = inst->mArg1.mReg;
  11973. }
  11974. else if (inst->mArg1)
  11975. {
  11976. //BF_ASSERT(inst->mArg1.mKind == BeMCOperandKind_VReg);
  11977. GetValAddr(inst->mArg1, destReg, destOfs);
  11978. }
  11979. if (memValue == 0)
  11980. {
  11981. // xor r11, r11
  11982. Emit(0x4D); Emit(0x31); Emit(0xDB);
  11983. }
  11984. else
  11985. {
  11986. // mov r11, memValue*8
  11987. Emit(0x49); Emit(0xBB);
  11988. for (int i = 0; i < 8; i++)
  11989. Emit(memValue);
  11990. }
  11991. for (; curOfs <= memSize - 8; curOfs += 8)
  11992. {
  11993. EmitREX(BeMCOperand::FromReg(X64Reg_R11), BeMCOperand::FromReg(destReg), true);
  11994. Emit(0x89);
  11995. EmitModRMRel(EncodeRegNum(X64Reg_R11), destReg, X64Reg_None, 1, curOfs + destOfs);
  11996. }
  11997. // If there's more that one 'small' write required then we just do another 64-bit move
  11998. // which overlaps partially into the previous 64-bit move
  11999. if ((memSize > 8) && (curOfs < memSize) && (memSize - curOfs != 4) &&
  12000. (memSize - curOfs != 2) && (memSize - curOfs != 1))
  12001. {
  12002. EmitREX(BeMCOperand::FromReg(X64Reg_R11), BeMCOperand::FromReg(destReg), true);
  12003. Emit(0x89);
  12004. EmitModRMRel(EncodeRegNum(X64Reg_R11), destReg, X64Reg_None, 1, (memSize - 8) + destOfs);
  12005. curOfs = memSize;
  12006. }
  12007. for (; curOfs <= memSize - 4; curOfs += 4)
  12008. {
  12009. EmitREX(BeMCOperand::FromReg(X64Reg_R11D), BeMCOperand::FromReg(destReg), false);
  12010. Emit(0x89);
  12011. EmitModRMRel(EncodeRegNum(X64Reg_R11D), destReg, X64Reg_None, 1, curOfs + destOfs);
  12012. }
  12013. for (; curOfs <= memSize - 2; curOfs += 2)
  12014. {
  12015. Emit(0x66); EmitREX(BeMCOperand::FromReg(X64Reg_R11W), BeMCOperand::FromReg(destReg), false);
  12016. Emit(0x89);
  12017. EmitModRMRel(EncodeRegNum(X64Reg_R11W), destReg, X64Reg_None, 1, curOfs + destOfs);
  12018. }
  12019. for (; curOfs <= memSize - 1; curOfs += 1)
  12020. {
  12021. EmitREX(BeMCOperand::FromReg(X64Reg_R11B), BeMCOperand::FromReg(destReg), false); //-V530
  12022. Emit(0x89 - 1);
  12023. EmitModRMRel(EncodeRegNum(X64Reg_R11B), destReg, X64Reg_None, 1, curOfs + destOfs);
  12024. }
  12025. }
  12026. break;
  12027. case BeMCInstKind_MemCpy:
  12028. {
  12029. // This doesn't currently attempt to align properly
  12030. int memSize = inst->mArg0.mMemCpyInfo.mSize;
  12031. int curOfs = 0;
  12032. X64CPURegister destReg = X64Reg_R10;
  12033. X64CPURegister srcReg = X64Reg_R11;
  12034. int destOfs = 0;
  12035. int srcOfs = 0;
  12036. if (inst->mArg1)
  12037. {
  12038. BF_ASSERT(inst->mArg1.mKind == BeMCOperandKind_VRegPair);
  12039. GetValAddr(BeMCOperand::FromEncoded(inst->mArg1.mVRegPair.mVRegIdx0), destReg, destOfs);
  12040. GetValAddr(BeMCOperand::FromEncoded(inst->mArg1.mVRegPair.mVRegIdx1), srcReg, srcOfs);
  12041. }
  12042. if ((srcReg == destReg) && (srcOfs == destOfs))
  12043. break;
  12044. for (; curOfs <= memSize - 8; curOfs += 8)
  12045. {
  12046. EmitREX(BeMCOperand::FromReg(X64Reg_R11), BeMCOperand::FromReg(srcReg), true);
  12047. Emit(0x8B);
  12048. EmitModRMRel(EncodeRegNum(X64Reg_R11), srcReg, X64Reg_None, 1, curOfs + srcOfs);
  12049. EmitREX(BeMCOperand::FromReg(X64Reg_R11), BeMCOperand::FromReg(destReg), true);
  12050. Emit(0x89);
  12051. EmitModRMRel(EncodeRegNum(X64Reg_R11), destReg, X64Reg_None, 1, curOfs + destOfs);
  12052. }
  12053. // If there's more that one 'small' write required then we just do another 64-bit move
  12054. // which overlaps partially into the previous 64-bit move
  12055. if ((memSize > 8) && (curOfs < memSize) && (memSize - curOfs != 4) &&
  12056. (memSize - curOfs != 2) && (memSize - curOfs != 1))
  12057. {
  12058. EmitREX(BeMCOperand::FromReg(X64Reg_R11), BeMCOperand::FromReg(srcReg), true);
  12059. Emit(0x8B);
  12060. EmitModRMRel(EncodeRegNum(X64Reg_R11), srcReg, X64Reg_None, 1, (memSize - 8) + srcOfs);
  12061. EmitREX(BeMCOperand::FromReg(X64Reg_R11), BeMCOperand::FromReg(destReg), true);
  12062. Emit(0x89);
  12063. EmitModRMRel(EncodeRegNum(X64Reg_R11), destReg, X64Reg_None, 1, (memSize - 8) + destOfs);
  12064. curOfs = memSize;
  12065. }
  12066. for (; curOfs <= memSize - 4; curOfs += 4)
  12067. {
  12068. EmitREX(BeMCOperand::FromReg(X64Reg_R11D), BeMCOperand::FromReg(srcReg), false);
  12069. Emit(0x8B);
  12070. EmitModRMRel(EncodeRegNum(X64Reg_R11D), srcReg, X64Reg_None, 1, curOfs + srcOfs);
  12071. EmitREX(BeMCOperand::FromReg(X64Reg_R11D), BeMCOperand::FromReg(destReg), false);
  12072. Emit(0x89);
  12073. EmitModRMRel(EncodeRegNum(X64Reg_R11D), destReg, X64Reg_None, 1, curOfs + destOfs);
  12074. }
  12075. for (; curOfs <= memSize - 2; curOfs += 2)
  12076. {
  12077. Emit(0x66); EmitREX(BeMCOperand::FromReg(X64Reg_R11W), BeMCOperand::FromReg(srcReg), false);
  12078. Emit(0x8B);
  12079. EmitModRMRel(EncodeRegNum(X64Reg_R11W), srcReg, X64Reg_None, 1, curOfs + srcOfs);
  12080. Emit(0x66); EmitREX(BeMCOperand::FromReg(X64Reg_R11W), BeMCOperand::FromReg(destReg), false);
  12081. Emit(0x89);
  12082. EmitModRMRel(EncodeRegNum(X64Reg_R11W), destReg, X64Reg_None, 1, curOfs + destOfs);
  12083. }
  12084. for (; curOfs <= memSize - 1; curOfs += 1)
  12085. {
  12086. EmitREX(BeMCOperand::FromReg(X64Reg_R11B), BeMCOperand::FromReg(srcReg), false);
  12087. Emit(0x8B - 1);
  12088. EmitModRMRel(EncodeRegNum(X64Reg_R11B), srcReg, X64Reg_None, 1, curOfs + srcOfs);
  12089. EmitREX(BeMCOperand::FromReg(X64Reg_R11B), BeMCOperand::FromReg(destReg), false);
  12090. Emit(0x89 - 1);
  12091. EmitModRMRel(EncodeRegNum(X64Reg_R11B), destReg, X64Reg_None, 1, curOfs + destOfs);
  12092. }
  12093. }
  12094. break;
  12095. case BeMCInstKind_FastCheckStack:
  12096. {
  12097. // CMP RSP, [RSP]
  12098. BF_ASSERT(inst->mArg0 == BeMCOperand::FromReg(X64Reg_RSP));
  12099. Emit(0x48);
  12100. Emit(0x3B);
  12101. Emit(0x24);
  12102. Emit(0x24);
  12103. }
  12104. break;
  12105. case BeMCInstKind_TLSSetup:
  12106. {
  12107. if (inst->mArg1) // Alt
  12108. {
  12109. BF_ASSERT(inst->mArg0.IsNativeReg());
  12110. // mov arg0, qword ptr gs:[0x58]
  12111. Emit(0x65);
  12112. EmitREX(inst->mArg0, BeMCOperand(), true);
  12113. Emit(0x8B); Emit(0x04 | (EncodeRegNum(inst->mArg0.mReg) << 3)); Emit(0x25);
  12114. mOut.Write((int32)0x58);
  12115. }
  12116. else
  12117. {
  12118. // mov eax, _tls_index
  12119. Emit(0x8B); Emit(0x05);
  12120. auto sym = mCOFFObject->GetSymbolRef("_tls_index");
  12121. BeMCRelocation reloc;
  12122. reloc.mKind = BeMCRelocationKind_REL32;
  12123. reloc.mOffset = mOut.GetPos();
  12124. reloc.mSymTableIdx = sym->mIdx;
  12125. mCOFFObject->mTextSect.mRelocs.push_back(reloc);
  12126. mTextRelocs.push_back((int)mCOFFObject->mTextSect.mRelocs.size() - 1);
  12127. mOut.Write((int32)0);
  12128. auto vregInfo = mVRegInfo[mTLSVRegIdx];
  12129. BF_ASSERT(vregInfo->mReg != X64Reg_None);
  12130. auto tlsOperand = BeMCOperand::FromReg(vregInfo->mReg);
  12131. // mov tlsReg, qword ptr gs:[0x58]
  12132. Emit(0x65);
  12133. EmitREX(tlsOperand, BeMCOperand(), true);
  12134. Emit(0x8B); Emit(0x04 | (EncodeRegNum(vregInfo->mReg) << 3)); Emit(0x25);
  12135. mOut.Write((int32)0x58);
  12136. // mov tlsReg, qword ptr [tlsReg + 8*rax]
  12137. EmitREX(tlsOperand, tlsOperand, true);
  12138. Emit(0x8B);
  12139. EmitModRMRel(EncodeRegNum(vregInfo->mReg), vregInfo->mReg, X64Reg_RAX, 8, 0);
  12140. }
  12141. }
  12142. break;
  12143. case BeMCInstKind_PreserveVolatiles:
  12144. {
  12145. for (int vregIdx : *inst->mLiveness)
  12146. {
  12147. if (vregIdx >= mLivenessContext.mNumItems)
  12148. continue;
  12149. auto vregInfo = mVRegInfo[vregIdx];
  12150. if ((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg != vregInfo->mReg))
  12151. continue;
  12152. if (vregInfo->mVolatileVRegSave != -1)
  12153. {
  12154. if (vregInfo->mDbgVariable != NULL)
  12155. {
  12156. BeDbgVariableRange range;
  12157. range.mOffset = funcCodePos;
  12158. range.mLength = -1;
  12159. vregInfo->mDbgVariable->mSavedRanges.push_back(range);
  12160. }
  12161. }
  12162. }
  12163. }
  12164. break;
  12165. case BeMCInstKind_RestoreVolatiles:
  12166. {
  12167. for (int vregIdx : *inst->mLiveness)
  12168. {
  12169. if (vregIdx >= mLivenessContext.mNumItems)
  12170. continue;
  12171. auto vregInfo = mVRegInfo[vregIdx];
  12172. if ((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg != vregInfo->mReg))
  12173. continue;
  12174. if (vregInfo->mVolatileVRegSave != -1)
  12175. {
  12176. if ((vregInfo->mDbgVariable != NULL) && (!vregInfo->mDbgVariable->mSavedRanges.empty()))
  12177. {
  12178. BeDbgVariableRange& range = vregInfo->mDbgVariable->mSavedRanges.back();
  12179. range.mLength = funcCodePos - range.mOffset;
  12180. bool handled = false;
  12181. if (!vregInfo->mDbgVariable->mGaps.empty())
  12182. {
  12183. auto& lastGap = vregInfo->mDbgVariable->mGaps.back();
  12184. if (lastGap.mLength == -1)
  12185. {
  12186. lastGap.mLength = funcCodePos - lastGap.mOffset;
  12187. handled = true;
  12188. }
  12189. }
  12190. if (!handled)
  12191. {
  12192. BeDbgVariableRange gapRange = range;
  12193. // gapRange.mOffset++;
  12194. // gapRange.mLength--;
  12195. vregInfo->mDbgVariable->mGaps.push_back(gapRange);
  12196. }
  12197. }
  12198. }
  12199. }
  12200. }
  12201. break;
  12202. case BeMCInstKind_None:
  12203. break;
  12204. case BeMCInstKind_Unwind_Alloc:
  12205. case BeMCInstKind_Unwind_PushReg:
  12206. case BeMCInstKind_Unwind_SaveXMM:
  12207. case BeMCInstKind_Unwind_SetBP:
  12208. {
  12209. BeMCDeferredUnwind deferredUnwind = { inst, funcCodePos };
  12210. deferredUnwinds.push_back(deferredUnwind);
  12211. }
  12212. break;
  12213. case BeMCInstKind_Label:
  12214. while (funcCodePos < hotJumpLen)
  12215. {
  12216. Emit(0x90);
  12217. funcCodePos++;
  12218. }
  12219. lastLabelIdx = inst->mArg0.mLabelIdx;
  12220. labelPositions[inst->mArg0.mLabelIdx] = funcCodePos;
  12221. // This ensures we can't jump back into the hot jump area
  12222. break;
  12223. case BeMCInstKind_Nop:
  12224. Emit(0x90);
  12225. break;
  12226. case BeMCInstKind_Unreachable:
  12227. // ud2
  12228. Emit(0x0F); Emit(0x0B);
  12229. break;
  12230. case BeMCInstKind_EnsureInstructionAt:
  12231. needsInstructionAtDbgLoc = true;
  12232. /*if (mDbgFunction != NULL)
  12233. {
  12234. // If this is true, this means that our instruction was the first one encountered at this mDbgPos,
  12235. // so we emit a NOP so we can step onto this line
  12236. if (mDbgFunction->mEmissions.back().mPos == funcCodePos)
  12237. {
  12238. mOut.Write((uint8)0x90);
  12239. }
  12240. }*/
  12241. break;
  12242. case BeMCInstKind_DbgBreak:
  12243. Emit(0xCC);
  12244. break;
  12245. case BeMCInstKind_MFence:
  12246. Emit(0x0F); Emit(0xAE); Emit(0xF0);
  12247. break;
  12248. case BeMCInstKind_Mov:
  12249. {
  12250. if (inst->mArg1.mKind == BeMCOperandKind_ConstAgg)
  12251. {
  12252. EmitAggMov(inst->mArg0, inst->mArg1);
  12253. break;
  12254. }
  12255. auto vregInfo = GetVRegInfo(inst->mArg1);
  12256. auto arg0Type = GetType(inst->mArg0);
  12257. auto arg1Type = GetType(inst->mArg1);
  12258. // auto arg1 = inst->mArg1;
  12259. // while (arg1.IsVReg())
  12260. // {
  12261. // auto vregInfo = GetVRegInfo(arg1);
  12262. // if (vregInfo->IsDirectRelTo())
  12263. // arg1 = vregInfo->mRelTo;
  12264. // }
  12265. if (instForm == BeMCInstForm_Unknown)
  12266. {
  12267. if ((inst->mArg0.IsNativeReg()) && (inst->mArg1.IsNativeReg()))
  12268. {
  12269. if ((arg0Type->IsInt()) && (arg0Type->mSize < arg1Type->mSize))
  12270. {
  12271. // Reduce arg1 to arg0 size
  12272. BeMCOperand mcDest;
  12273. BeMCOperand mcSrc = BeMCOperand::FromReg(ResizeRegister(inst->mArg1.mReg, arg0Type));
  12274. BeTypeCode useTypeCode = BeTypeCode_None;
  12275. if (mcSrc.mReg != X64Reg_None)
  12276. {
  12277. // Reduction worked
  12278. mcDest = inst->mArg0;
  12279. useTypeCode = arg0Type->mTypeCode;
  12280. }
  12281. else
  12282. {
  12283. SoftFail("Error in Mov");
  12284. }
  12285. switch (useTypeCode)
  12286. {
  12287. case BeTypeCode_Int8:
  12288. EmitREX(mcDest, mcSrc, false);
  12289. Emit(0x8A); EmitModRM(mcDest, mcSrc);
  12290. break;
  12291. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  12292. case BeTypeCode_Int32:
  12293. case BeTypeCode_Int64:
  12294. EmitREX(mcDest, mcSrc, arg0Type->mTypeCode == BeTypeCode_Int64);
  12295. Emit(0x8B); EmitModRM(mcDest, mcSrc);
  12296. break;
  12297. default: NotImpl();
  12298. }
  12299. break;
  12300. }
  12301. }
  12302. if (inst->mArg0.IsNativeReg())
  12303. {
  12304. BF_ASSERT(arg0Type->IsIntable());
  12305. BF_ASSERT(arg1Type->IsIntable());
  12306. BF_ASSERT(arg0Type->mSize > arg1Type->mSize);
  12307. // Zero extend 'dest' to 'src', emit movzx
  12308. switch (arg0Type->mTypeCode)
  12309. {
  12310. case BeTypeCode_Int16:
  12311. BF_ASSERT(arg1Type->mTypeCode == BeTypeCode_Int8);
  12312. Emit(0x66);
  12313. EmitREX(inst->mArg0, inst->mArg1, false);
  12314. Emit(0x0F); Emit(0xB6); EmitModRM(inst->mArg0, inst->mArg1);
  12315. break;
  12316. case BeTypeCode_Int32:
  12317. switch (arg1Type->mTypeCode)
  12318. {
  12319. case BeTypeCode_Int8:
  12320. case BeTypeCode_Boolean:
  12321. EmitREX(inst->mArg0, inst->mArg1, false);
  12322. Emit(0x0F); Emit(0xB6); EmitModRM(inst->mArg0, inst->mArg1);
  12323. break;
  12324. case BeTypeCode_Int16:
  12325. EmitREX(inst->mArg0, inst->mArg1, false);
  12326. Emit(0x0F); Emit(0xB7); EmitModRM(inst->mArg0, inst->mArg1);
  12327. break;
  12328. default: NotImpl();
  12329. }
  12330. break;
  12331. case BeTypeCode_Int64:
  12332. switch (arg1Type->mTypeCode)
  12333. {
  12334. case BeTypeCode_Int8:
  12335. case BeTypeCode_Boolean:
  12336. EmitREX(inst->mArg0, inst->mArg1, true);
  12337. Emit(0x0F); Emit(0xB6); EmitModRM(inst->mArg0, inst->mArg1); break;
  12338. case BeTypeCode_Int16:
  12339. EmitREX(inst->mArg0, inst->mArg1, true);
  12340. Emit(0x0F); Emit(0xB7); EmitModRM(inst->mArg0, inst->mArg1); break;
  12341. case BeTypeCode_Int32:
  12342. {
  12343. // We convert from a "mov r64, r32" to a "mov r32, r32", which forces high 32 bits of original r64 to zero
  12344. BeMCOperand mcDest = BeMCOperand::FromReg(ResizeRegister(inst->mArg0.mReg, arg1Type));
  12345. EmitREX(mcDest, inst->mArg1, false);
  12346. Emit(0x8B); EmitModRM(mcDest, inst->mArg1);
  12347. }
  12348. break;
  12349. }
  12350. break;
  12351. default:
  12352. NotImpl();
  12353. }
  12354. break; // from inst switch
  12355. }
  12356. if (arg0Type->mTypeCode == BeTypeCode_Float)
  12357. {
  12358. if (inst->mArg1.IsImmediate())
  12359. {
  12360. // Emit as a int32 mov
  12361. EmitREX(BeMCOperand(), inst->mArg0, false);
  12362. Emit(0xC7);
  12363. EmitModRM(0, inst->mArg0, -4);
  12364. float val = inst->mArg1.mImmF32;
  12365. mOut.Write(*(int32*)&val);
  12366. break;
  12367. }
  12368. }
  12369. }
  12370. if (inst->mArg1.mKind == BeMCOperandKind_CmpResult)
  12371. {
  12372. BF_ASSERT(instForm == BeMCInstForm_R8_RM8);
  12373. // SETcc
  12374. auto& cmpResult = mCmpResults[inst->mArg1.mCmpResultIdx];
  12375. uint8 opCode = GetJumpOpCode(cmpResult.mCmpKind, false) + 0x20;
  12376. EmitREX(BeMCOperand(), inst->mArg0, false);
  12377. Emit(0x0F);
  12378. Emit(opCode);
  12379. EmitModRM(0, inst->mArg0);
  12380. break;
  12381. }
  12382. bool isLEA = inst->mArg1.mKind == BeMCOperandKind_SymbolAddr;
  12383. if ((instForm == BeMCInstForm_R64_RM64) && (inst->mArg1.IsVRegAny()))
  12384. {
  12385. bool isMulti = false;
  12386. if (GetRMForm(inst->mArg1, isMulti) == BeMCRMMode_Direct)
  12387. {
  12388. // LEA
  12389. auto modInst = *inst;
  12390. BF_ASSERT(modInst.mArg1.IsVRegAny());
  12391. modInst.mArg1 = BeMCOperand::ToLoad(modInst.mArg1);
  12392. EmitInst(BeMCInstForm_R64_RM64, 0x8D, &modInst);
  12393. break;
  12394. }
  12395. else
  12396. {
  12397. auto vregInfo1 = GetVRegInfo(inst->mArg1);
  12398. if (vregInfo1->mRelTo.mKind == BeMCOperandKind_SymbolAddr)
  12399. isLEA = true;
  12400. }
  12401. }
  12402. if (inst->mArg1.mKind == BeMCOperandKind_VRegAddr)
  12403. {
  12404. NotImpl();
  12405. /*// LEA
  12406. switch (instForm)
  12407. {
  12408. case BeMCInstForm_R64_RM64: EmitInst(BeMCInstForm_R64_RM64, 0x8D, inst); break;
  12409. default:
  12410. NotImpl();
  12411. }*/
  12412. }
  12413. else if (isLEA)
  12414. {
  12415. // LEA
  12416. switch (instForm)
  12417. {
  12418. case BeMCInstForm_R64_RM64: EmitInst(BeMCInstForm_R64_RM64, 0x8D, inst); break;
  12419. default:
  12420. NotImpl();
  12421. }
  12422. }
  12423. // ??
  12424. /*else if ((vregInfo != NULL) && (vregInfo->mIsExpr))
  12425. {
  12426. // LEA
  12427. switch (instForm)
  12428. {
  12429. case BeMCInstForm_R64_RM64: EmitInst(BeMCInstForm_R64_RM64_ADDR, 0x8D, inst); break;
  12430. default:
  12431. NotImpl();
  12432. }
  12433. }*/
  12434. else if ((inst->mArg0.IsNativeReg()) && (inst->mArg1.IsZero()))
  12435. {
  12436. auto type = GetType(inst->mArg0);
  12437. if (type->mSize == 1)
  12438. {
  12439. // Emit as XOR <arg0>, <arg0>
  12440. EmitREX(inst->mArg0, inst->mArg0, false);
  12441. mOut.Write((uint8)0x30); // r, r/m64
  12442. EmitModRM(inst->mArg0, inst->mArg0);
  12443. }
  12444. else
  12445. {
  12446. // Emit as XOR <arg0>, <arg0>
  12447. EmitREX(inst->mArg0, inst->mArg0, type->mSize == 8);
  12448. mOut.Write((uint8)0x33); // r, r/m64
  12449. EmitModRM(inst->mArg0, inst->mArg0);
  12450. }
  12451. }
  12452. else
  12453. {
  12454. switch (instForm)
  12455. {
  12456. case BeMCInstForm_XMM32_RM32:
  12457. {
  12458. BF_ASSERT(inst->mArg1.IsNativeReg());
  12459. // CVTSI2SS - use zero-extended 64-bit register
  12460. EmitStdXMMInst(BeMCInstForm_XMM32_RM64, inst, 0x2A);
  12461. }
  12462. break;
  12463. case BeMCInstForm_XMM64_RM32:
  12464. {
  12465. BF_ASSERT(inst->mArg1.IsNativeReg());
  12466. // CVTSI2SD - use zero-extended 64-bit register
  12467. EmitStdXMMInst(BeMCInstForm_XMM64_RM64, inst, 0x2A);
  12468. }
  12469. break;
  12470. case BeMCInstForm_XMM64_FRM32:
  12471. case BeMCInstForm_XMM32_FRM64:
  12472. {
  12473. // CVTSS2SD / CVTSD2SS
  12474. EmitStdXMMInst(instForm, inst, 0x5A);
  12475. }
  12476. break;
  12477. case BeMCInstForm_R32_F32:
  12478. case BeMCInstForm_R32_F64:
  12479. case BeMCInstForm_R64_F32:
  12480. case BeMCInstForm_R64_F64:
  12481. {
  12482. // CVTTSS2SI
  12483. EmitStdXMMInst(instForm, inst, 0x2C);
  12484. }
  12485. break;
  12486. case BeMCInstForm_XMM128_RM128:
  12487. {
  12488. // MOVUPS
  12489. EmitREX(arg0, arg1, true);
  12490. Emit(0x0F); Emit(0x10);
  12491. if (arg1.IsImmediateInt())
  12492. arg1.mKind = BeMCOperandKind_Immediate_int32x4;
  12493. EmitModRM(arg0, arg1);
  12494. }
  12495. break;
  12496. case BeMCInstForm_FRM128_XMM128:
  12497. {
  12498. if (arg0Type->mSize == 4)
  12499. {
  12500. BeMCOperand imm1;
  12501. imm1.mImmediate = 1;
  12502. imm1.mKind = BeMCOperandKind_Immediate_int32x4;
  12503. BeMCOperand xmm15;
  12504. xmm15.mReg = X64Reg_M128_XMM15;
  12505. xmm15.mKind = BeMCOperandKind_NativeReg;
  12506. // MOVUPS
  12507. EmitREX(xmm15, arg1, true);
  12508. Emit(0x0F); Emit(0x10);
  12509. EmitModRM(xmm15, arg1);
  12510. // ANDPS xmm15, <1, 1, 1, 1>
  12511. EmitREX(xmm15, imm1, true);
  12512. Emit(0x0F); Emit(0x54);
  12513. EmitModRM(xmm15, imm1);
  12514. // PACKUSWB xmm15, xmm15
  12515. Emit(0x66); EmitREX(xmm15, xmm15, true);
  12516. Emit(0x0F); Emit(0x67);
  12517. EmitModRM(xmm15, xmm15);
  12518. // PACKUSWB xmm15, xmm15
  12519. Emit(0x66); EmitREX(xmm15, xmm15, true);
  12520. Emit(0x0F); Emit(0x67);
  12521. EmitModRM(xmm15, xmm15);
  12522. // MOVD <arg0>, xmm15
  12523. Emit(0x66); EmitREX(xmm15, arg0, false);
  12524. Emit(0x0F); Emit(0x7E);
  12525. EmitModRM(xmm15, arg0);
  12526. }
  12527. else
  12528. {
  12529. // MOVUPS
  12530. EmitREX(arg1, arg0, true);
  12531. Emit(0x0F); Emit(0x11);
  12532. EmitModRM(arg1, arg0);
  12533. }
  12534. }
  12535. break;
  12536. case BeMCInstForm_XMM64_RM64:
  12537. case BeMCInstForm_XMM32_RM64:
  12538. {
  12539. if ((arg0Type->IsVector()) && (arg1Type->IsVector()))
  12540. {
  12541. // Fall through
  12542. }
  12543. else
  12544. {
  12545. // uint64->xmm Not implemented
  12546. // We do a signed transform instead
  12547. // CVTSI2SS / CVTSI2SD
  12548. EmitStdXMMInst(instForm, inst, 0x2A);
  12549. break;
  12550. }
  12551. }
  12552. // Fall through
  12553. default:
  12554. {
  12555. // MOVSS/MOVSD
  12556. if (EmitStdXMMInst(instForm, inst, 0x10, 0x11))
  12557. break;
  12558. if ((inst->mArg0.IsNativeReg()) && (arg1.IsImmediateInt()))
  12559. {
  12560. auto arg0Type = GetType(inst->mArg0);
  12561. if (arg0Type->mTypeCode == BeTypeCode_Int16)
  12562. {
  12563. Emit(0x66);
  12564. EmitREX(BeMCOperand(), inst->mArg0, false);
  12565. Emit(0xB8 + EncodeRegNum(inst->mArg0.mReg));
  12566. mOut.Write((int16)arg1.mImmediate);
  12567. break;
  12568. }
  12569. else if ((arg0Type->mTypeCode == BeTypeCode_Int32) || (arg0Type->mTypeCode == BeTypeCode_Int64))
  12570. {
  12571. // For 64-bit writes, we would like to use 32-bit zero extension but we resort to the full
  12572. // 64-bits if necessary
  12573. auto arg0 = inst->mArg0;
  12574. if (arg0Type->mTypeCode == BeTypeCode_Int64)
  12575. {
  12576. auto resizedReg = ResizeRegister(arg0.mReg, 4);
  12577. if ((resizedReg != X64Reg_None) && (inst->mArg1.mImmediate < 0) && (inst->mArg1.mImmediate >= -0x80000000LL))
  12578. {
  12579. // Do sign-extend R32 mov
  12580. EmitREX(BeMCOperand(), inst->mArg0, true);
  12581. Emit(0xC7);
  12582. EmitModRM(0, inst->mArg0);
  12583. mOut.Write((int32)inst->mArg1.mImmediate);
  12584. break;
  12585. }
  12586. if ((resizedReg == X64Reg_None) || (inst->mArg1.mImmediate < 0) || (inst->mArg1.mImmediate >= 0x100000000LL))
  12587. {
  12588. EmitREX(BeMCOperand(), inst->mArg0, true);
  12589. Emit(0xB8 + EncodeRegNum(inst->mArg0.mReg));
  12590. mOut.Write((int64)inst->mArg1.mImmediate);
  12591. break;
  12592. }
  12593. else
  12594. arg0.mReg = resizedReg;
  12595. }
  12596. EmitREX(BeMCOperand(), arg0, false);
  12597. Emit(0xB8 + EncodeRegNum(arg0.mReg));
  12598. mOut.Write((int32)inst->mArg1.mImmediate);
  12599. break;
  12600. }
  12601. }
  12602. EmitStdInst(instForm, inst, 0x89, 0x8B, 0xC7, 0x0);
  12603. }
  12604. }
  12605. }
  12606. }
  12607. break;
  12608. case BeMCInstKind_MovRaw:
  12609. {
  12610. auto arg0Type = GetType(inst->mArg0);
  12611. auto arg1Type = GetType(inst->mArg1);
  12612. if ((arg0Type->mTypeCode == BeTypeCode_Int64) && (arg1Type->mTypeCode == BeTypeCode_Double))
  12613. {
  12614. Emit(0x66); EmitREX(inst->mArg1, inst->mArg0, true);
  12615. Emit(0x0F); Emit(0x7E);
  12616. EmitModRM(inst->mArg1, inst->mArg0);
  12617. }
  12618. else
  12619. {
  12620. NotImpl();
  12621. }
  12622. }
  12623. break;
  12624. case BeMCInstKind_MovSX:
  12625. {
  12626. auto arg0Type = GetType(inst->mArg0);
  12627. auto arg1Type = GetType(inst->mArg1);
  12628. switch (arg0Type->mTypeCode)
  12629. {
  12630. case BeTypeCode_Int16:
  12631. BF_ASSERT(arg1Type->mTypeCode == BeTypeCode_Int8);
  12632. Emit(0x66);
  12633. EmitREX(inst->mArg0, inst->mArg1, false);
  12634. Emit(0x0F); Emit(0xBE); EmitModRM(inst->mArg0, inst->mArg1);
  12635. break;
  12636. case BeTypeCode_Int32:
  12637. switch (arg1Type->mTypeCode)
  12638. {
  12639. case BeTypeCode_Int8:
  12640. EmitREX(inst->mArg0, inst->mArg1, false);
  12641. Emit(0x0F); Emit(0xBE); EmitModRM(inst->mArg0, inst->mArg1);
  12642. break;
  12643. case BeTypeCode_Int16:
  12644. EmitREX(inst->mArg0, inst->mArg1, false);
  12645. Emit(0x0F); Emit(0xBF); EmitModRM(inst->mArg0, inst->mArg1);
  12646. break;
  12647. case BeTypeCode_Float:
  12648. // CVTSS2SI
  12649. Emit(0xF3); EmitREX(inst->mArg0, inst->mArg1, false);
  12650. Emit(0x0F); Emit(0x2C);
  12651. EmitModRM(inst->mArg0, inst->mArg1);
  12652. break;
  12653. default: NotImpl();
  12654. }
  12655. break;
  12656. case BeTypeCode_Int64:
  12657. switch (arg1Type->mTypeCode)
  12658. {
  12659. case BeTypeCode_Int8:
  12660. EmitREX(inst->mArg0, inst->mArg1, true);
  12661. Emit(0x0F); Emit(0xBE); EmitModRM(inst->mArg0, inst->mArg1); break;
  12662. case BeTypeCode_Int16:
  12663. EmitREX(inst->mArg0, inst->mArg1, true);
  12664. Emit(0x0F); Emit(0xBF); EmitModRM(inst->mArg0, inst->mArg1); break;
  12665. case BeTypeCode_Int32:
  12666. EmitREX(inst->mArg0, inst->mArg1, true);
  12667. Emit(0x63); EmitModRM(inst->mArg0, inst->mArg1); break;
  12668. case BeTypeCode_Float:
  12669. // CVTSS2SI
  12670. Emit(0xF3); EmitREX(inst->mArg0, inst->mArg1, true);
  12671. Emit(0x0F); Emit(0x2C);
  12672. EmitModRM(inst->mArg0, inst->mArg1);
  12673. break;
  12674. default: NotImpl();
  12675. }
  12676. break;
  12677. case BeTypeCode_Float:
  12678. case BeTypeCode_Double:
  12679. {
  12680. if (arg1Type->IsInt())
  12681. {
  12682. // CVTSI2SS / CVTSI2SD
  12683. EmitStdXMMInst(instForm, inst, 0x2A);
  12684. }
  12685. else
  12686. NotImpl();
  12687. /*switch (arg1Type->mTypeCode)
  12688. {
  12689. case BeTypeCode_Double:
  12690. EmitStdXMMInst(BeMCInstForm_XMM32_RM64, inst, 0x5A);
  12691. break;
  12692. default:
  12693. NotImpl();
  12694. break;
  12695. }*/
  12696. }
  12697. break;
  12698. default: NotImpl();
  12699. }
  12700. }
  12701. break;
  12702. case BeMCInstKind_CmpXChg:
  12703. {
  12704. auto arg0Type = GetType(inst->mArg0);
  12705. if (inst->mArg1.IsNativeReg())
  12706. {
  12707. switch (instForm)
  12708. {
  12709. case BeMCInstForm_R8_RM8:
  12710. instForm = BeMCInstForm_RM8_R8;
  12711. break;
  12712. case BeMCInstForm_R16_RM16:
  12713. instForm = BeMCInstForm_RM16_R16;
  12714. break;
  12715. case BeMCInstForm_R32_RM32:
  12716. instForm = BeMCInstForm_RM32_R32;
  12717. break;
  12718. case BeMCInstForm_R64_RM64:
  12719. instForm = BeMCInstForm_RM64_R64;
  12720. break;
  12721. }
  12722. }
  12723. if (!inst->mArg0.IsNativeReg())
  12724. Emit(0xF0); // LOCK
  12725. switch (instForm)
  12726. {
  12727. case BeMCInstForm_RM8_R8:
  12728. EmitREX(inst->mArg1, inst->mArg0, false);
  12729. Emit(0x0F); Emit(0xB0); EmitModRM(inst->mArg1, inst->mArg0);
  12730. break;
  12731. case BeMCInstForm_RM16_R16:
  12732. EmitREX(inst->mArg1, inst->mArg0, false);
  12733. Emit(0x66); Emit(0x0F); Emit(0xB0); EmitModRM(inst->mArg1, inst->mArg0);
  12734. break;
  12735. case BeMCInstForm_RM32_R32:
  12736. EmitREX(inst->mArg1, inst->mArg0, false);
  12737. Emit(0x0F); Emit(0xB1); EmitModRM(inst->mArg1, inst->mArg0);
  12738. break;
  12739. case BeMCInstForm_RM64_R64:
  12740. EmitREX(inst->mArg1, inst->mArg0, true);
  12741. Emit(0x0F); Emit(0xB1); EmitModRM(inst->mArg1, inst->mArg0);
  12742. break;
  12743. default:
  12744. SoftFail("Invalid CmpXChg args");
  12745. }
  12746. }
  12747. break;
  12748. case BeMCInstKind_XAdd:
  12749. {
  12750. auto arg0Type = GetType(inst->mArg0);
  12751. switch (instForm)
  12752. {
  12753. case BeMCInstForm_R8_RM8:
  12754. if (inst->mArg1.IsNativeReg())
  12755. instForm = BeMCInstForm_RM8_R8;
  12756. break;
  12757. case BeMCInstForm_R16_RM16:
  12758. if (inst->mArg1.IsNativeReg())
  12759. instForm = BeMCInstForm_RM16_R16;
  12760. break;
  12761. case BeMCInstForm_R32_RM32:
  12762. if (inst->mArg1.IsNativeReg())
  12763. instForm = BeMCInstForm_RM32_R32;
  12764. break;
  12765. case BeMCInstForm_R64_RM64:
  12766. if (inst->mArg1.IsNativeReg())
  12767. instForm = BeMCInstForm_RM64_R64;
  12768. break;
  12769. }
  12770. if (!inst->mArg0.IsNativeReg())
  12771. Emit(0xF0); // LOCK
  12772. switch (instForm)
  12773. {
  12774. case BeMCInstForm_RM8_R8:
  12775. EmitREX(inst->mArg1, inst->mArg0, false);
  12776. Emit(0x0F); Emit(0xC0); EmitModRM(inst->mArg1, inst->mArg0);
  12777. break;
  12778. case BeMCInstForm_RM16_R16:
  12779. EmitREX(inst->mArg1, inst->mArg0, false);
  12780. Emit(0x66); Emit(0x0F); Emit(0xC0); EmitModRM(inst->mArg1, inst->mArg0);
  12781. break;
  12782. case BeMCInstForm_RM32_R32:
  12783. EmitREX(inst->mArg1, inst->mArg0, false);
  12784. Emit(0x0F); Emit(0xC1); EmitModRM(inst->mArg1, inst->mArg0);
  12785. break;
  12786. case BeMCInstForm_RM64_R64:
  12787. EmitREX(inst->mArg1, inst->mArg0, true);
  12788. Emit(0x0F); Emit(0xC1); EmitModRM(inst->mArg1, inst->mArg0);
  12789. break;
  12790. default:
  12791. SoftFail("Invalid XAdd args");
  12792. }
  12793. }
  12794. break;
  12795. case BeMCInstKind_XChg:
  12796. {
  12797. EmitStdInst(instForm, inst, 0x87, 0x87, 0x00, 0x0);
  12798. }
  12799. break;
  12800. case BeMCInstKind_Load:
  12801. {
  12802. if (inst->mArg1.IsSymbol())
  12803. {
  12804. BF_ASSERT(inst->mArg0.IsNativeReg());
  12805. }
  12806. switch (instForm)
  12807. {
  12808. case BeMCInstForm_XMM64_FRM64:
  12809. case BeMCInstForm_XMM32_FRM32:
  12810. // MOVSS/MOVSD
  12811. EmitStdXMMInst(instForm, inst, 0x10);
  12812. break;
  12813. case BeMCInstForm_R32_RM32:
  12814. EmitInst(BeMCInstForm_R32_RM64_ADDR, 0x8B, inst);
  12815. break;
  12816. case BeMCInstForm_R64_RM64:
  12817. EmitInst(BeMCInstForm_R64_RM64_ADDR, 0x8B, inst);
  12818. break;
  12819. case BeMCInstForm_XMM32_RM64:
  12820. case BeMCInstForm_XMM64_RM64:
  12821. // MOVSS / MOVSD
  12822. BF_ASSERT(inst->mArg0.IsNativeReg());
  12823. if (instForm == BeMCInstForm_XMM64_RM64)
  12824. Emit(0xF2);
  12825. else
  12826. Emit(0xF3);
  12827. EmitREX(inst->mArg0, inst->mArg1, false);
  12828. Emit(0x0F); Emit(0x10);
  12829. EmitModRM_Addr(inst->mArg0, inst->mArg1);
  12830. break;
  12831. default:
  12832. {
  12833. BF_ASSERT(inst->mArg0.IsNativeReg());
  12834. auto arg0Type = GetType(inst->mArg0);
  12835. auto arg1Type = GetType(inst->mArg1);
  12836. BF_ASSERT(arg1Type->mSize == 8);
  12837. switch (arg0Type->mSize)
  12838. {
  12839. case 1: EmitInst(BeMCInstForm_R8_RM64_ADDR, 0x8B, inst); break;
  12840. case 2: EmitInst(BeMCInstForm_R16_RM64_ADDR, 0x8B, inst); break;
  12841. case 4: EmitInst(BeMCInstForm_R32_RM64_ADDR, 0x8B, inst); break;
  12842. case 8: EmitInst(BeMCInstForm_R64_RM64_ADDR, 0x8B, inst); break;
  12843. }
  12844. }
  12845. }
  12846. /*BF_ASSERT(inst->mArg0.mKind == BeMCOperandKind_NativeReg);
  12847. BF_ASSERT(inst->mArg1.mKind == BeMCOperandKind_NativeReg);
  12848. EmitREX(inst->mArg0, inst->mArg1);
  12849. mOut.Write((uint8)0x8B);
  12850. uint8 modRM = (0x0 << 6) | (EncodeRegNum(inst->mArg1.mReg)) | (EncodeRegNum(inst->mArg0.mReg)); // <arg0>, [<arg1>]
  12851. mOut.Write(modRM);*/
  12852. }
  12853. break;
  12854. case BeMCInstKind_Store:
  12855. {
  12856. switch (instForm)
  12857. {
  12858. /*case BeMCInstForm_R64_RM64:
  12859. {
  12860. // Make sure its really R64, R64
  12861. BF_ASSERT(inst->mArg1.IsNativeReg());
  12862. EmitInst(BeMCInstForm_RM64_R64_ADDR, 0x89, inst);
  12863. }
  12864. break;*/
  12865. case BeMCInstForm_RM64_R64: EmitInst(BeMCInstForm_RM64_R64_ADDR, 0x89, inst); break;
  12866. case BeMCInstForm_FRM32_XMM32:
  12867. case BeMCInstForm_FRM64_XMM64:
  12868. // MOVSS/MOVSD
  12869. EmitStdXMMInst(instForm, inst, 0x10, 0x11);
  12870. break;
  12871. case BeMCInstForm_R64_F64:
  12872. case BeMCInstForm_R64_F32:
  12873. // MOVSS / MOVSD
  12874. BF_ASSERT(inst->mArg1.IsNativeReg());
  12875. if (instForm == BeMCInstForm_R64_F64)
  12876. Emit(0xF2);
  12877. else
  12878. Emit(0xF3);
  12879. EmitREX(inst->mArg1, inst->mArg0, false);
  12880. Emit(0x0F); Emit(0x11);
  12881. EmitModRM_Addr(inst->mArg1, inst->mArg0);
  12882. break;
  12883. default:
  12884. {
  12885. BF_ASSERT(inst->mArg0.IsNativeReg());
  12886. BF_ASSERT(inst->mArg1.IsNativeReg());
  12887. auto arg0Type = GetType(inst->mArg0);
  12888. auto arg1Type = GetType(inst->mArg1);
  12889. BF_ASSERT(arg0Type->mSize == 8);
  12890. switch (arg1Type->mSize)
  12891. {
  12892. case 1: EmitInst(BeMCInstForm_RM64_R8_ADDR, 0x89, inst); break;
  12893. case 2: EmitInst(BeMCInstForm_RM64_R16_ADDR, 0x89, inst); break;
  12894. case 4: EmitInst(BeMCInstForm_RM64_R32_ADDR, 0x89, inst); break;
  12895. case 8: EmitInst(BeMCInstForm_RM64_R64_ADDR, 0x89, inst); break;
  12896. }
  12897. }
  12898. }
  12899. }
  12900. break;
  12901. case BeMCInstKind_Push:
  12902. {
  12903. auto arg0Type = GetType(inst->mArg0);
  12904. if (inst->mArg1)
  12905. {
  12906. BF_ASSERT(IsXMMReg(inst->mArg0.mReg));
  12907. BF_ASSERT(inst->mArg1.IsImmediate());
  12908. // Is there a performance benefit to properly using MOVAPD vs MOVAPS if we only
  12909. // use this register for double storage?
  12910. // MOVAPS
  12911. EmitREX(inst->mArg0, BeMCOperand(), false);
  12912. Emit(0x0F); Emit(0x29);
  12913. EmitModRMRel(EncodeRegNum(inst->mArg0.mReg), X64Reg_RSP, X64Reg_None, 1, (int)inst->mArg1.mImmediate);
  12914. break;
  12915. }
  12916. if (arg0Type->IsFloat())
  12917. {
  12918. BF_ASSERT(IsXMMReg(inst->mArg0.mReg));
  12919. // sub rsp, sizeof(arg0)
  12920. Emit(0x48); Emit(0x83); Emit(0xEC); Emit(arg0Type->mSize);
  12921. Emit((arg0Type->mSize == 4) ? 0xF3 : 0xF2);
  12922. EmitREX(inst->mArg0, BeMCOperand(), false);
  12923. Emit(0x0F); Emit(0x11);
  12924. EmitModRMRel(EncodeRegNum(inst->mArg0.mReg), X64Reg_RSP, X64Reg_None, 1, 0);
  12925. break;
  12926. }
  12927. // This is used in combination with register pops, and we can't pop 32-bit registers
  12928. // so we need to make sure we're always pushing a full 64 bits
  12929. switch (instForm)
  12930. {
  12931. case BeMCInstForm_R8: // Always use full reg (always push 64 bits)
  12932. case BeMCInstForm_R16:
  12933. case BeMCInstForm_R32:
  12934. case BeMCInstForm_R64:
  12935. EmitREX(BeMCOperand(), inst->mArg0, true);
  12936. mOut.Write((uint8)(0x50 + EncodeRegNum(inst->mArg0.mReg)));
  12937. break;
  12938. case BeMCInstForm_RM64:
  12939. EmitREX(BeMCOperand(), inst->mArg0, true);
  12940. Emit(0xFF); EmitModRM(6, inst->mArg0);
  12941. break;
  12942. case BeMCInstForm_IMM32: // Always pad to 64 bits
  12943. {
  12944. Emit(0x68); mOut.Write((int32)inst->mArg0.mImmediate);
  12945. // Emit padded 0
  12946. Emit(0x68); mOut.Write((int32)0);
  12947. }
  12948. break;
  12949. default:
  12950. NotImpl();
  12951. }
  12952. }
  12953. break;
  12954. case BeMCInstKind_Pop:
  12955. {
  12956. auto arg0Type = GetType(inst->mArg0);
  12957. if (inst->mArg1)
  12958. {
  12959. BF_ASSERT(IsXMMReg(inst->mArg0.mReg));
  12960. BF_ASSERT(inst->mArg1.IsImmediate());
  12961. // Is there a performance benefit to properly using MOVAPD vs MOVAPS if we only
  12962. // use this register for double storage?
  12963. // MOVAPS
  12964. EmitREX(inst->mArg0, BeMCOperand(), false);
  12965. Emit(0x0F); Emit(0x28);
  12966. // Push always uses RSP (required for stack unwinding), but Pop uses RBP when applicable
  12967. // because it is still set up at that point
  12968. EmitModRMRelStack(EncodeRegNum(inst->mArg0.mReg), (int)inst->mArg1.mImmediate, 1);
  12969. break;
  12970. }
  12971. if (arg0Type->IsFloat())
  12972. {
  12973. BF_ASSERT(IsXMMReg(inst->mArg0.mReg));
  12974. Emit((arg0Type->mSize == 4) ? 0xF3 : 0xF2);
  12975. EmitREX(inst->mArg0, BeMCOperand(), false);
  12976. Emit(0x0F); Emit(0x10);
  12977. EmitModRMRel(EncodeRegNum(inst->mArg0.mReg), X64Reg_RSP, X64Reg_None, 1, 0);
  12978. // add rsp, sizeof(arg0)
  12979. Emit(0x48); Emit(0x83); Emit(0xC4); Emit(arg0Type->mSize);
  12980. break;
  12981. }
  12982. switch (instForm)
  12983. {
  12984. case BeMCInstForm_R64:
  12985. EmitREX(BeMCOperand(), inst->mArg0, true);
  12986. mOut.Write((uint8)(0x58 + EncodeRegNum(inst->mArg0.mReg)));
  12987. break;
  12988. case BeMCInstForm_RM64:
  12989. EmitREX(BeMCOperand(), inst->mArg0, true);
  12990. Emit(0x8F); EmitModRM(0, inst->mArg0);
  12991. default:
  12992. NotImpl();
  12993. }
  12994. }
  12995. break;
  12996. case BeMCInstKind_Neg:
  12997. {
  12998. auto typeCode = GetType(inst->mArg0)->mTypeCode;
  12999. switch (typeCode)
  13000. {
  13001. case BeTypeCode_Int8:
  13002. EmitREX(BeMCOperand(), inst->mArg0, false);
  13003. Emit(0xF6);
  13004. EmitModRM(3, inst->mArg0);
  13005. break;
  13006. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  13007. case BeTypeCode_Int32:
  13008. case BeTypeCode_Int64:
  13009. EmitREX(BeMCOperand(), inst->mArg0, typeCode == BeTypeCode_Int64);
  13010. Emit(0xF7); EmitModRM(3, inst->mArg0);
  13011. break;
  13012. default:
  13013. NotImpl();
  13014. }
  13015. }
  13016. break;
  13017. case BeMCInstKind_Not:
  13018. {
  13019. auto typeCode = GetType(inst->mArg0)->mTypeCode;
  13020. switch (typeCode)
  13021. {
  13022. case BeTypeCode_Int8:
  13023. EmitREX(BeMCOperand(), inst->mArg0, false);
  13024. Emit(0xF6);
  13025. EmitModRM(2, inst->mArg0);
  13026. break;
  13027. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  13028. case BeTypeCode_Int32:
  13029. case BeTypeCode_Int64:
  13030. EmitREX(BeMCOperand(), inst->mArg0, typeCode == BeTypeCode_Int64);
  13031. Emit(0xF7); EmitModRM(2, inst->mArg0);
  13032. break;
  13033. default:
  13034. NotImpl();
  13035. }
  13036. }
  13037. break;
  13038. case BeMCInstKind_Add:
  13039. {
  13040. if ((inst->mResult) && (inst->mResult != inst->mArg0) && (!inst->mDisableShortForm))
  13041. {
  13042. BF_ASSERT(!inst->mDisableShortForm);
  13043. BF_ASSERT(inst->mResult.IsNativeReg());
  13044. BF_ASSERT(inst->mArg0.IsNativeReg());
  13045. BF_ASSERT(inst->mArg1.IsImmediate());
  13046. // LEA form
  13047. auto resultType = GetType(inst->mArg0);
  13048. switch (resultType->mTypeCode)
  13049. {
  13050. case BeTypeCode_Int16:
  13051. Emit(0x66);
  13052. case BeTypeCode_Int32:
  13053. case BeTypeCode_Int64:
  13054. EmitREX(inst->mResult, inst->mArg0, resultType->mTypeCode == BeTypeCode_Int64);
  13055. Emit(0x8D); EmitModRMRel(EncodeRegNum(inst->mResult.mReg), ResizeRegister(inst->mArg0.mReg, 8), X64Reg_None, 1, (int)inst->mArg1.mImmediate);
  13056. break;
  13057. default:
  13058. NotImpl();
  13059. }
  13060. }
  13061. else if ((inst->mArg1.IsImmediateInt()) && (inst->mArg1.mImmediate == 1) && (!inst->mDisableShortForm))
  13062. {
  13063. // Emit as INC <arg0>
  13064. auto typeCode = GetType(inst->mArg0)->mTypeCode;
  13065. switch (typeCode)
  13066. {
  13067. case BeTypeCode_Int8:
  13068. EmitREX(BeMCOperand(), inst->mArg0, false);
  13069. Emit(0xFE); EmitModRM(0, inst->mArg0);
  13070. break;
  13071. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  13072. case BeTypeCode_Int32:
  13073. case BeTypeCode_Int64:
  13074. EmitREX(BeMCOperand(), inst->mArg0, typeCode == BeTypeCode_Int64);
  13075. Emit(0xFF); EmitModRM(0, inst->mArg0);
  13076. break;
  13077. default:
  13078. NotImpl();
  13079. }
  13080. }
  13081. else if (((inst->mArg1.IsImmediateInt()) && (inst->mArg1.mImmediate == -1)) && (!inst->mDisableShortForm))
  13082. {
  13083. // Emit as DEC <arg0>
  13084. auto typeCode = GetType(inst->mArg0)->mTypeCode;
  13085. switch (typeCode)
  13086. {
  13087. case BeTypeCode_Int8:
  13088. EmitREX(BeMCOperand(), inst->mArg0, false);
  13089. Emit(0xFE); EmitModRM(1, inst->mArg0);
  13090. break;
  13091. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  13092. case BeTypeCode_Int32:
  13093. case BeTypeCode_Int64:
  13094. EmitREX(BeMCOperand(), inst->mArg0, typeCode == BeTypeCode_Int64);
  13095. Emit(0xFF); EmitModRM(1, inst->mArg0);
  13096. break;
  13097. default:
  13098. NotImpl();
  13099. }
  13100. }
  13101. else
  13102. {
  13103. if (((instForm == BeMCInstForm_RM64_IMM16) || (instForm == BeMCInstForm_RM64_IMM32)) &&
  13104. (inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_RAX))
  13105. {
  13106. // Emit as ADD RAX, <imm32>
  13107. EmitREX(inst->mArg0, BeMCOperand(), true);
  13108. mOut.Write((uint8)0x05);
  13109. mOut.Write((int32)inst->mArg1.mImmediate);
  13110. break;
  13111. }
  13112. if (EmitIntXMMInst(instForm, inst, 0xFC)) // PADD?
  13113. break;
  13114. if (EmitStdXMMInst(instForm, inst, 0x58))
  13115. break;
  13116. EmitStdInst(instForm, inst, 0x01, 0x03, 0x81, 0x0, 0x83, 0x0);
  13117. }
  13118. }
  13119. break;
  13120. case BeMCInstKind_Sub:
  13121. {
  13122. if ((inst->mArg1.IsImmediateInt()) && (inst->mArg1.mImmediate == 1) && (!inst->mDisableShortForm))
  13123. {
  13124. // Emit as DEC <arg0>
  13125. auto typeCode = GetType(inst->mArg0)->mTypeCode;
  13126. switch (typeCode)
  13127. {
  13128. case BeTypeCode_Int8:
  13129. EmitREX(BeMCOperand(), inst->mArg0, false);
  13130. Emit(0xFE); EmitModRM(1, inst->mArg0);
  13131. break;
  13132. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  13133. case BeTypeCode_Int32:
  13134. case BeTypeCode_Int64:
  13135. EmitREX(BeMCOperand(), inst->mArg0, typeCode == BeTypeCode_Int64);
  13136. Emit(0xFF); EmitModRM(1, inst->mArg0);
  13137. break;
  13138. default:
  13139. NotImpl();
  13140. }
  13141. }
  13142. else
  13143. {
  13144. if (((instForm == BeMCInstForm_RM64_IMM16) || (instForm == BeMCInstForm_RM64_IMM32)) &&
  13145. (inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_RAX))
  13146. {
  13147. // Emit as SUB RAX, <imm32>
  13148. EmitREX(inst->mArg0, BeMCOperand(), true);
  13149. mOut.Write((uint8)0x2D);
  13150. mOut.Write((int32)inst->mArg1.mImmediate);
  13151. break;
  13152. }
  13153. if (EmitIntXMMInst(instForm, inst, 0xF8)) // PSUB?
  13154. break;
  13155. if (EmitStdXMMInst(instForm, inst, 0x5C))
  13156. break;
  13157. EmitStdInst(instForm, inst, 0x29, 0x2B, 0x81, 0x5, 0x83, 0x5);
  13158. }
  13159. }
  13160. break;
  13161. case BeMCInstKind_Mul:
  13162. {
  13163. if (arg0Type->IsIntable())
  13164. {
  13165. bool isValid = true;
  13166. auto typeCode = GetType(inst->mArg1)->mTypeCode;
  13167. switch (typeCode)
  13168. {
  13169. case BeTypeCode_Int8:
  13170. isValid = inst->mArg0 == BeMCOperand::FromReg(X64Reg_AL);
  13171. break;
  13172. case BeTypeCode_Int16:
  13173. isValid = inst->mArg0 == BeMCOperand::FromReg(X64Reg_AX);
  13174. break;
  13175. case BeTypeCode_Int32:
  13176. isValid = inst->mArg0 == BeMCOperand::FromReg(X64Reg_EAX);
  13177. break;
  13178. case BeTypeCode_Int64:
  13179. isValid = inst->mArg0 == BeMCOperand::FromReg(X64Reg_RAX);
  13180. break;
  13181. default:
  13182. isValid = false;
  13183. }
  13184. if (!isValid)
  13185. SoftFail("Invalid mul arguments");
  13186. switch (typeCode)
  13187. {
  13188. case BeTypeCode_Int8:
  13189. EmitREX(BeMCOperand(), inst->mArg1, false);
  13190. Emit(0xF6);
  13191. EmitModRM(4, inst->mArg1);
  13192. break;
  13193. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  13194. case BeTypeCode_Int32:
  13195. case BeTypeCode_Int64:
  13196. EmitREX(BeMCOperand(), inst->mArg1, typeCode == BeTypeCode_Int64);
  13197. Emit(0xF7); EmitModRM(4, inst->mArg1);
  13198. break;
  13199. default:
  13200. NotImpl();
  13201. }
  13202. break;
  13203. }
  13204. }
  13205. //Fallthrough
  13206. case BeMCInstKind_IMul:
  13207. {
  13208. if (instForm == BeMCInstForm_XMM128_RM128)
  13209. {
  13210. if (arg0Type->IsExplicitVectorType())
  13211. {
  13212. auto vecType = (BeVectorType*)arg0Type;
  13213. if (vecType->mElementType->mTypeCode == BeTypeCode_Int32)
  13214. {
  13215. Emit(0x66);
  13216. EmitREX(arg0, arg1, false);
  13217. Emit(0x0F);
  13218. if (inst->mKind == BeMCInstKind_IMul)
  13219. Emit(0xD5); // PMULLW
  13220. else
  13221. {
  13222. Emit(0x38); Emit(0x40); // PMULLD
  13223. }
  13224. EmitModRM(arg0, arg1);
  13225. break;
  13226. }
  13227. }
  13228. }
  13229. if (EmitStdXMMInst(instForm, inst, 0x59))
  13230. break;
  13231. BeMCOperand result = inst->mResult;
  13232. if ((!result) && (inst->mArg1.IsImmediate()))
  13233. result = inst->mArg0; // Do as 3-form anyway
  13234. if (result)
  13235. {
  13236. BF_ASSERT(inst->mArg1.IsImmediate());
  13237. if ((inst->mArg0.IsNativeReg()) && (!inst->mDisableShortForm) &&
  13238. ((inst->mArg1.mImmediate == 2) || (inst->mArg1.mImmediate == 4) || (inst->mArg1.mImmediate == 8)))
  13239. {
  13240. // LEA form
  13241. auto resultType = GetType(inst->mArg0);
  13242. if (resultType->mTypeCode != BeTypeCode_Int8)
  13243. {
  13244. switch (resultType->mTypeCode)
  13245. {
  13246. case BeTypeCode_Int16:
  13247. Emit(0x66);
  13248. case BeTypeCode_Int32:
  13249. case BeTypeCode_Int64:
  13250. uint8 rex = GetREX(result, inst->mArg0, resultType->mTypeCode == BeTypeCode_Int64);
  13251. if (rex != 0)
  13252. {
  13253. if (rex & 1)
  13254. {
  13255. // Mov REX flag from RM to SiB
  13256. rex = (rex & ~1) | 2;
  13257. }
  13258. Emit(rex);
  13259. }
  13260. Emit(0x8D); EmitModRMRel(EncodeRegNum(result.mReg), X64Reg_None, inst->mArg0.mReg, (int)inst->mArg1.mImmediate, 0);
  13261. break;
  13262. }
  13263. break;
  13264. }
  13265. }
  13266. auto typeCode = GetType(inst->mArg0)->mTypeCode;
  13267. switch (typeCode)
  13268. {
  13269. case BeTypeCode_Int16:
  13270. Emit(0x66);
  13271. // Fall through
  13272. case BeTypeCode_Int32:
  13273. case BeTypeCode_Int64:
  13274. EmitREX(result, inst->mArg0, typeCode == BeTypeCode_Int64);
  13275. if ((inst->mArg1.mImmediate >= -0x80) && (inst->mArg1.mImmediate <= 0x7F))
  13276. {
  13277. Emit(0x6B);
  13278. EmitModRM(result, inst->mArg0, -1);
  13279. mOut.Write((int8)inst->mArg1.mImmediate);
  13280. }
  13281. else
  13282. {
  13283. Emit(0x69);
  13284. EmitModRM(result, inst->mArg0, (typeCode == BeTypeCode_Int16) ? -2 : -4);
  13285. if (typeCode == BeTypeCode_Int16)
  13286. mOut.Write((int16)inst->mArg1.mImmediate);
  13287. else
  13288. mOut.Write((int32)inst->mArg1.mImmediate);
  13289. }
  13290. break;
  13291. default:
  13292. NotImpl();
  13293. }
  13294. }
  13295. else
  13296. {
  13297. if (inst->mArg1.IsImmediate())
  13298. {
  13299. int64 multVal = inst->mArg1.mImmediate;
  13300. if (IsPowerOfTwo(multVal))
  13301. {
  13302. int shiftCount = 0;
  13303. while (multVal > 1)
  13304. {
  13305. shiftCount++;
  13306. multVal >>= 1;
  13307. }
  13308. auto arg0Type = GetType(inst->mArg0);
  13309. switch (arg0Type->mTypeCode)
  13310. {
  13311. case BeTypeCode_Int64:
  13312. EmitREX(inst->mArg0, BeMCOperand(), true);
  13313. // Fall through
  13314. case BeTypeCode_Int32:
  13315. Emit(0xC1); Emit((0x4 << 3) | (3 << 6) | (EncodeRegNum(inst->mArg0.mReg)));
  13316. Emit((uint8)shiftCount);
  13317. break;
  13318. default:
  13319. NotImpl();
  13320. }
  13321. break;
  13322. }
  13323. }
  13324. switch (instForm)
  13325. {
  13326. case BeMCInstForm_R8_RM8:
  13327. BF_ASSERT(inst->mArg0.mReg == X64Reg_AL);
  13328. EmitREX(BeMCOperand(), inst->mArg1, false);
  13329. Emit(0xF6);
  13330. EmitModRM(0x5, inst->mArg1);
  13331. break;
  13332. case BeMCInstForm_R16_RM16:
  13333. case BeMCInstForm_R32_RM32:
  13334. case BeMCInstForm_R64_RM64: EmitInst(instForm, 0xAF0F, inst); break;
  13335. default:
  13336. NotImpl();
  13337. }
  13338. }
  13339. }
  13340. break;
  13341. case BeMCInstKind_Div:
  13342. {
  13343. auto arg0Type = GetType(inst->mArg0);
  13344. switch (arg0Type->mTypeCode)
  13345. {
  13346. case BeTypeCode_Int8:
  13347. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_AL));
  13348. // XOR ah, ah
  13349. Emit(0x30); Emit(0xE4);
  13350. // DIV rm
  13351. EmitREX(BeMCOperand::FromReg(X64Reg_AX), inst->mArg1, false);
  13352. Emit(0xF6); EmitModRM(0x6, inst->mArg1);
  13353. break;
  13354. case BeTypeCode_Int16:
  13355. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_AX));
  13356. // XOR dx, dx
  13357. Emit(0x66); Emit(0x31); Emit(0xD2);
  13358. // DIV rm
  13359. Emit(0x66);
  13360. EmitREX(BeMCOperand::FromReg(X64Reg_AX), inst->mArg1, false);
  13361. Emit(0xF7); EmitModRM(0x6, inst->mArg1);
  13362. break;
  13363. case BeTypeCode_Int32:
  13364. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_EAX));
  13365. // XOR edx, edx
  13366. Emit(0x31); Emit(0xD2);
  13367. // DIV rm
  13368. EmitREX(BeMCOperand::FromReg(X64Reg_EAX), inst->mArg1, false);
  13369. Emit(0xF7); EmitModRM(0x6, inst->mArg1);
  13370. break;
  13371. case BeTypeCode_Int64:
  13372. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_RAX));
  13373. // XOR rdx, rdx
  13374. Emit(0x48); Emit(0x31); Emit(0xD2);
  13375. // DIV rm
  13376. EmitREX(BeMCOperand::FromReg(X64Reg_RAX), inst->mArg1, true);
  13377. Emit(0xF7); EmitModRM(0x6, inst->mArg1);
  13378. break;
  13379. }
  13380. }
  13381. break;
  13382. case BeMCInstKind_IDiv:
  13383. {
  13384. if (EmitStdXMMInst(instForm, inst, 0x5E))
  13385. break;
  13386. auto arg0Type = GetType(inst->mArg0);
  13387. switch (arg0Type->mTypeCode)
  13388. {
  13389. case BeTypeCode_Int8:
  13390. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_AL));
  13391. // CBW
  13392. Emit(0x66); Emit(0x98);
  13393. // IDIV rm
  13394. EmitREX(BeMCOperand::FromReg(X64Reg_AX), inst->mArg1, false);
  13395. Emit(0xF6); EmitModRM(0x7, inst->mArg1);
  13396. break;
  13397. case BeTypeCode_Int16:
  13398. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_AX));
  13399. // CWD
  13400. Emit(0x66); Emit(0x99);
  13401. // IDIV rm
  13402. Emit(0x66);
  13403. EmitREX(BeMCOperand::FromReg(X64Reg_AX), inst->mArg1, false);
  13404. Emit(0xF7); EmitModRM(0x7, inst->mArg1);
  13405. break;
  13406. case BeTypeCode_Int32:
  13407. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_EAX));
  13408. // CDQ
  13409. Emit(0x99);
  13410. // IDIV rm
  13411. EmitREX(BeMCOperand::FromReg(X64Reg_EAX), inst->mArg1, false);
  13412. Emit(0xF7); EmitModRM(0x7, inst->mArg1);
  13413. break;
  13414. case BeTypeCode_Int64:
  13415. BF_ASSERT((inst->mArg0.IsNativeReg()) && (inst->mArg0.mReg == X64Reg_RAX));
  13416. // CQO
  13417. Emit(0x48); Emit(0x99);
  13418. // IDIV rm
  13419. EmitREX(BeMCOperand::FromReg(X64Reg_RAX), inst->mArg1, true);
  13420. Emit(0xF7); EmitModRM(0x7, inst->mArg1);
  13421. break;
  13422. }
  13423. }
  13424. break;
  13425. case BeMCInstKind_IRem:
  13426. {
  13427. NotImpl();
  13428. //if (EmitStdXMMInst(instForm, inst, 0x5E))
  13429. //break;
  13430. }
  13431. break;
  13432. case BeMCInstKind_Cmp:
  13433. {
  13434. switch (instForm)
  13435. {
  13436. case BeMCInstForm_XMM32_FRM32:
  13437. case BeMCInstForm_XMM32_IMM:
  13438. // UCOMISS
  13439. EmitREX(inst->mArg0, inst->mArg1, false);
  13440. Emit(0x0F); Emit(0x2E);
  13441. EmitModRM(inst->mArg0, inst->mArg1);
  13442. break;
  13443. case BeMCInstForm_XMM64_FRM64:
  13444. case BeMCInstForm_XMM64_IMM:
  13445. // UCOMISD
  13446. Emit(0x66);
  13447. EmitREX(inst->mArg0, inst->mArg1, false);
  13448. Emit(0x0F); Emit(0x2E);
  13449. EmitModRM(inst->mArg0, inst->mArg1);
  13450. break;
  13451. default:
  13452. {
  13453. if (((inst->mArg0.IsNativeReg()) && (inst->mArg1.IsImmediateInt())) && (inst->mArg1.GetImmediateInt() == 0))
  13454. {
  13455. // Test
  13456. auto typeCode = GetType(inst->mArg0)->mTypeCode;
  13457. switch (typeCode)
  13458. {
  13459. case BeTypeCode_Int8:
  13460. EmitREX(inst->mArg0, inst->mArg0, false);
  13461. Emit(0x84);
  13462. EmitModRM(inst->mArg0, inst->mArg0);
  13463. break;
  13464. case BeTypeCode_Int16: Emit(0x66); // Fallthrough
  13465. case BeTypeCode_Int32:
  13466. case BeTypeCode_Int64:
  13467. EmitREX(inst->mArg0, inst->mArg0, typeCode == BeTypeCode_Int64);
  13468. Emit(0x85); EmitModRM(inst->mArg0, inst->mArg0);
  13469. break;
  13470. default:
  13471. NotImpl();
  13472. }
  13473. /*if (arg0Type->mSize == 2)
  13474. Emit(0x66);
  13475. // Emit as TEST <arg0>, <arg0>
  13476. EmitREX(inst->mArg0, inst->mArg0, GetType(inst->mArg0)->mSize == 8);
  13477. mOut.Write((uint8)0x85); // r/m64, r
  13478. EmitModRM(inst->mArg0, inst->mArg0);*/
  13479. }
  13480. else
  13481. {
  13482. EmitStdInst(instForm, inst, 0x39, 0x3B, 0x81, 0x7, 0x83, 0x7);
  13483. }
  13484. }
  13485. }
  13486. }
  13487. break;
  13488. case BeMCInstKind_And:
  13489. {
  13490. if (EmitIntBitwiseXMMInst(instForm, inst, 0xDB)) //PAND
  13491. break;
  13492. BeMCInst modInst = *inst;
  13493. bool isZeroing = false;
  13494. // Use a shorter form if the upper bytes are masked in
  13495. if ((instForm == BeMCInstForm_RM32_IMM32) || (instForm == BeMCInstForm_RM64_IMM32))
  13496. {
  13497. if ((modInst.mArg1.mImmediate & 0xFFFF0000) == 0xFFFF0000)
  13498. {
  13499. if (modInst.mArg0.IsNativeReg())
  13500. modInst.mArg0.mReg = ResizeRegister(modInst.mArg0.mReg, 2);
  13501. instForm = BeMCInstForm_RM16_IMM16;
  13502. if ((modInst.mArg1.mImmediate & 0xFFFF) == 0)
  13503. isZeroing = true;
  13504. }
  13505. else if (instForm == BeMCInstForm_RM32_IMM32)
  13506. {
  13507. if ((modInst.mArg1.mImmediate & 0xFFFFFFFF) == 0)
  13508. {
  13509. isZeroing = true;
  13510. }
  13511. }
  13512. }
  13513. if ((instForm == BeMCInstForm_RM16_IMM16) || (instForm == BeMCInstForm_RM32_IMM16) || (instForm == BeMCInstForm_RM64_IMM16))
  13514. {
  13515. if ((modInst.mArg1.mImmediate & 0xFFFFFF00) == 0xFFFFFF00)
  13516. {
  13517. if (modInst.mArg0.IsNativeReg())
  13518. modInst.mArg0.mReg = ResizeRegister(modInst.mArg0.mReg, 1);
  13519. instForm = BeMCInstForm_RM8_IMM8;
  13520. }
  13521. else if (instForm == BeMCInstForm_RM16_IMM16)
  13522. {
  13523. if ((modInst.mArg1.mImmediate & 0xFFFF) == 0)
  13524. {
  13525. isZeroing = true;
  13526. }
  13527. }
  13528. }
  13529. if (instForm == BeMCInstForm_RM8_IMM8)
  13530. {
  13531. // Are we really just zeroing out the lower byte?
  13532. if (((modInst.mArg1.mImmediate & 0xFF) == 0) && (modInst.mArg0.IsNativeReg()))
  13533. {
  13534. isZeroing = true;
  13535. }
  13536. }
  13537. if (isZeroing)
  13538. {
  13539. int size = 0;
  13540. switch (instForm)
  13541. {
  13542. case BeMCInstForm_RM32_IMM32:
  13543. instForm = BeMCInstForm_RM32_R32;
  13544. break;
  13545. case BeMCInstForm_RM16_IMM16:
  13546. instForm = BeMCInstForm_RM16_R16;
  13547. break;
  13548. case BeMCInstForm_RM8_IMM8:
  13549. instForm = BeMCInstForm_RM8_R8;
  13550. break;
  13551. default:
  13552. NotImpl();
  13553. }
  13554. //XOR arg0, arg0
  13555. modInst.mKind = BeMCInstKind_Xor;
  13556. modInst.mArg1 = modInst.mArg0;
  13557. EmitStdInst(instForm, &modInst, 0x31, 0x33, 0x81, 0x6, 0x83, 0x6);
  13558. break;
  13559. }
  13560. if (modInst.mArg1.IsImmediateFloat())
  13561. { //ANDPS
  13562. bool is64Bit = (modInst.mArg1.mKind == BeMCOperandKind_Immediate_f64_Packed128);
  13563. EmitREX(inst->mArg0, inst->mArg1, is64Bit);
  13564. Emit(0x0F); Emit(0x54);
  13565. EmitModRM(inst->mArg0, inst->mArg1);
  13566. break;
  13567. }
  13568. EmitStdInst(instForm, &modInst, 0x21, 0x23, 0x81, 0x4, 0x83, 0x4);
  13569. }
  13570. break;
  13571. case BeMCInstKind_Or:
  13572. {
  13573. if (EmitIntBitwiseXMMInst(instForm, inst, 0xEB)) //POR
  13574. break;
  13575. EmitStdInst(instForm, inst, 0x09, 0x0B, 0x81, 0x1, 0x83, 0x1);
  13576. }
  13577. break;
  13578. case BeMCInstKind_Xor:
  13579. {
  13580. if (EmitIntBitwiseXMMInst(instForm, inst, 0xEF)) //PXOR
  13581. break;
  13582. if (EmitPackedXMMInst(instForm, inst, 0x57))
  13583. break;
  13584. EmitStdInst(instForm, inst, 0x31, 0x33, 0x81, 0x6, 0x83, 0x6);
  13585. }
  13586. break;
  13587. case BeMCInstKind_Shl:
  13588. case BeMCInstKind_Shr:
  13589. case BeMCInstKind_Sar:
  13590. {
  13591. if (instForm == Beefy::BeMCInstForm_XMM128_RM128)
  13592. {
  13593. if (arg1.IsImmediate())
  13594. {
  13595. Emit(0x66);
  13596. EmitREX(arg1, arg0, false);
  13597. Emit(0x0F);
  13598. int rx = 0;
  13599. switch (inst->mKind)
  13600. {
  13601. case BeMCInstKind_Shl:
  13602. rx = 6;
  13603. break;
  13604. case BeMCInstKind_Shr:
  13605. rx = 2;
  13606. break;
  13607. case BeMCInstKind_Sar:
  13608. rx = 4;
  13609. break;
  13610. }
  13611. Emit(0x71); // PSLLW / PSRAW / PSRLW
  13612. EmitModRM(rx, arg0);
  13613. Emit((uint8)arg1.mImmediate);
  13614. }
  13615. else
  13616. {
  13617. Emit(0x66);
  13618. EmitREX(arg0, arg1, false);
  13619. Emit(0x0F);
  13620. switch (inst->mKind)
  13621. {
  13622. case BeMCInstKind_Shl:
  13623. Emit(0xF1); // PSLLW
  13624. break;
  13625. case BeMCInstKind_Shr:
  13626. Emit(0xD1); // PSRLW
  13627. break;
  13628. case BeMCInstKind_Sar:
  13629. Emit(0xE1); // PSRAW
  13630. break;
  13631. }
  13632. EmitModRM(arg0, arg1);
  13633. }
  13634. break;
  13635. }
  13636. int rx = 0;
  13637. switch (inst->mKind)
  13638. {
  13639. case BeMCInstKind_Shl:
  13640. rx = 4;
  13641. break;
  13642. case BeMCInstKind_Shr:
  13643. rx = 5;
  13644. break;
  13645. case BeMCInstKind_Sar:
  13646. rx = 7;
  13647. break;
  13648. }
  13649. bool handled = false;
  13650. switch (instForm)
  13651. {
  13652. case BeMCInstForm_RM8_IMM8:
  13653. case BeMCInstForm_RM16_IMM8:
  13654. case BeMCInstForm_RM32_IMM8:
  13655. case BeMCInstForm_RM64_IMM8:
  13656. if (inst->mArg1.mImmediate == 1)
  13657. {
  13658. // Shift by one has a short form
  13659. if (instForm == BeMCInstForm_RM16_IMM8) Emit(0x66);
  13660. EmitREX(inst->mArg1, inst->mArg0, instForm == BeMCInstForm_RM64_IMM8);
  13661. if (instForm == BeMCInstForm_RM8_IMM8)
  13662. Emit(0xD0);
  13663. else
  13664. Emit(0xD1);
  13665. EmitModRM(rx, inst->mArg0);
  13666. handled = true;
  13667. }
  13668. else
  13669. {
  13670. if (instForm == BeMCInstForm_RM16_IMM8) Emit(0x66);
  13671. EmitREX(inst->mArg1, inst->mArg0, instForm == BeMCInstForm_RM64_IMM8);
  13672. if (instForm == BeMCInstForm_RM8_IMM8)
  13673. Emit(0xC0);
  13674. else
  13675. Emit(0xC1);
  13676. EmitModRM(rx, inst->mArg0);
  13677. Emit((uint8)inst->mArg1.mImmediate);
  13678. handled = true;
  13679. }
  13680. break;
  13681. default:
  13682. {
  13683. BF_ASSERT(inst->mArg1.IsNativeReg());
  13684. BF_ASSERT(inst->mArg1.mReg == X64Reg_RCX);
  13685. int destSize = GetType(inst->mArg0)->mSize;
  13686. if (destSize == 2) Emit(0x66);
  13687. EmitREX(BeMCOperand(), inst->mArg0, destSize == 8);
  13688. if (destSize == 1)
  13689. Emit(0xD2);
  13690. else
  13691. Emit(0xD3);
  13692. EmitModRM(rx, inst->mArg0);
  13693. handled = true;
  13694. }
  13695. break;
  13696. }
  13697. }
  13698. break;
  13699. case BeMCInstKind_Test:
  13700. {
  13701. if (instForm == BeMCInstForm_R64_RM64)
  13702. {
  13703. BF_SWAP(inst->mArg0, inst->mArg1);
  13704. instForm = BeMCInstForm_RM64_R64;
  13705. }
  13706. EmitStdInst(instForm, inst, 0x85, 0x00, 0xF7, 0x0);
  13707. }
  13708. break;
  13709. case BeMCInstKind_CondBr:
  13710. {
  13711. for (int pass = 0; pass < 2; pass++)
  13712. {
  13713. if (inst->mArg0.mKind == BeMCOperandKind_Immediate_i64)
  13714. {
  13715. if (pass == 1)
  13716. break;
  13717. mOut.Write(GetJumpOpCode(inst->mArg1.mCmpKind, false));
  13718. mOut.Write((uint8)inst->mArg0.mImmediate);
  13719. }
  13720. else
  13721. {
  13722. BeCmpKind cmpKind = inst->mArg1.mCmpKind;
  13723. if (pass == 1)
  13724. {
  13725. switch (cmpKind)
  13726. {
  13727. case BeCmpKind_OEQ:
  13728. cmpKind = BeCmpKind_EQ;
  13729. break;
  13730. case BeCmpKind_UNE:
  13731. cmpKind = BeCmpKind_NE;
  13732. break;
  13733. default:
  13734. cmpKind = BeCmpKind_None;
  13735. }
  13736. if (cmpKind == BeCmpKind_None)
  13737. break;
  13738. }
  13739. BF_ASSERT(inst->mArg0.mKind == BeMCOperandKind_Label);
  13740. BeMCJump jump;
  13741. jump.mCodeOffset = funcCodePos;
  13742. jump.mLabelIdx = inst->mArg0.mLabelIdx;
  13743. // Speculative make it a short jump
  13744. jump.mJumpKind = 0;
  13745. jump.mCmpKind = cmpKind;;
  13746. deferredJumps.push_back(jump);
  13747. mOut.Write(GetJumpOpCode(jump.mCmpKind, false));
  13748. mOut.Write((uint8)0);
  13749. funcCodePos += 2;
  13750. }
  13751. }
  13752. }
  13753. break;
  13754. case BeMCInstKind_Br:
  13755. {
  13756. if (inst->mArg1.mKind == BeMCOperandKind_Immediate_i8)
  13757. {
  13758. if (inst->mArg1.mImmediate == 2) // Fake?
  13759. break;
  13760. }
  13761. if (inst->mArg0.mKind == BeMCOperandKind_Label)
  13762. {
  13763. BeMCJump jump;
  13764. jump.mCodeOffset = funcCodePos;
  13765. jump.mLabelIdx = inst->mArg0.mLabelIdx;
  13766. // Speculatively make it a short jump
  13767. jump.mJumpKind = 0;
  13768. jump.mCmpKind = BeCmpKind_None;
  13769. deferredJumps.push_back(jump);
  13770. mOut.Write((uint8)0xEB);
  13771. mOut.Write((uint8)0);
  13772. }
  13773. else
  13774. {
  13775. auto arg0Type = GetType(inst->mArg0);
  13776. BF_ASSERT(arg0Type->mTypeCode == BeTypeCode_Int64);
  13777. uint8 rex = GetREX(BeMCOperand(), inst->mArg0, true);
  13778. if (rex != 0x40)
  13779. Emit(rex);
  13780. Emit(0xFF);
  13781. EmitModRM(4, inst->mArg0);
  13782. }
  13783. }
  13784. break;
  13785. case BeMCInstKind_Ret:
  13786. mOut.Write((uint8)0xC3);
  13787. break;
  13788. case BeMCInstKind_Call:
  13789. {
  13790. switch (instForm)
  13791. {
  13792. case BeMCInstForm_Symbol:
  13793. {
  13794. // Call [rip+<X>]
  13795. mOut.Write((uint8)0xFF);
  13796. mOut.Write((uint8)0x15);
  13797. BeMCRelocation reloc;
  13798. reloc.mKind = BeMCRelocationKind_REL32;
  13799. reloc.mOffset = mOut.GetPos();
  13800. reloc.mSymTableIdx = inst->mArg0.mSymbolIdx;
  13801. mCOFFObject->mTextSect.mRelocs.push_back(reloc);
  13802. mTextRelocs.push_back((int)mCOFFObject->mTextSect.mRelocs.size() - 1);
  13803. mOut.Write((int32)0);
  13804. }
  13805. break;
  13806. case BeMCInstForm_SymbolAddr:
  13807. {
  13808. // Call <X>
  13809. mOut.Write((uint8)0xE8);
  13810. BeMCRelocation reloc;
  13811. reloc.mKind = BeMCRelocationKind_REL32;
  13812. reloc.mOffset = mOut.GetPos();
  13813. reloc.mSymTableIdx = inst->mArg0.mSymbolIdx;
  13814. mCOFFObject->mTextSect.mRelocs.push_back(reloc);
  13815. mTextRelocs.push_back((int)mCOFFObject->mTextSect.mRelocs.size() - 1);
  13816. mOut.Write((int32)0);
  13817. }
  13818. break;
  13819. default:
  13820. {
  13821. EmitREX(BeMCOperand(), inst->mArg0, true);
  13822. mOut.Write((uint8)0xFF);
  13823. EmitModRM(0x2, inst->mArg0);
  13824. }
  13825. break;
  13826. }
  13827. }
  13828. //mOut.Write((uint8)0xC3);
  13829. break;
  13830. default:
  13831. SoftFail("Unhandled instruction in DoCodeEmission", inst->mDbgLoc);
  13832. break;
  13833. }
  13834. }
  13835. }
  13836. mActiveInst = NULL;
  13837. if (mDebugging)
  13838. {
  13839. dbgStr += "\nEMISSIONS:\n";
  13840. }
  13841. // Finish range for all outstanding variables
  13842. for (int vregLiveIdx : *vregsLive)
  13843. {
  13844. if (vregLiveIdx >= mLivenessContext.mNumItems)
  13845. continue;
  13846. int vregIdx = vregLiveIdx % mLivenessContext.mNumItems;
  13847. auto dbgVar = mVRegInfo[vregIdx]->mDbgVariable;
  13848. if (dbgVar != NULL)
  13849. {
  13850. dbgVar->mDeclEnd = mOut.GetPos() - textSectStartPos;
  13851. dbgVar->mDeclLifetimeExtend = false;
  13852. BF_ASSERT(dbgVar->mDeclEnd >= dbgVar->mDeclStart);
  13853. if (!dbgVar->mSavedRanges.empty())
  13854. {
  13855. auto& savedRange = dbgVar->mSavedRanges.back();
  13856. if (savedRange.mLength == -1)
  13857. {
  13858. savedRange.mLength = dbgVar->mDeclEnd - savedRange.mOffset;
  13859. dbgVar->mGaps.push_back(savedRange);
  13860. }
  13861. }
  13862. }
  13863. }
  13864. auto& codeVec = mOut.mData;
  13865. for (int pass = 0; true; pass++)
  13866. {
  13867. bool didWidening = false;
  13868. for (auto& jump : deferredJumps)
  13869. {
  13870. int labelPos = labelPositions[jump.mLabelIdx];
  13871. int offsetRel = jump.mCodeOffset;
  13872. if (jump.mJumpKind == 0)
  13873. offsetRel += 2;
  13874. else if (jump.mJumpKind == 1)
  13875. offsetRel += 4;
  13876. else if (jump.mJumpKind == 2)
  13877. {
  13878. if (jump.mCmpKind == BeCmpKind_None)
  13879. offsetRel += 5;
  13880. else
  13881. offsetRel += 6;
  13882. }
  13883. int offset = labelPos - offsetRel;
  13884. //BF_ASSERT((offset >= -128) && (offset <= 127));
  13885. if ((jump.mJumpKind == 0) &&
  13886. ((offset < -0x80) || (offset > 0x7F)))
  13887. {
  13888. // Extend this guy into a rel32
  13889. int adjustFrom = jump.mCodeOffset + 2;
  13890. int adjustBytes = 3;
  13891. if (jump.mCmpKind != BeCmpKind_None)
  13892. adjustBytes++;
  13893. codeVec.Insert(jump.mCodeOffset + 1 + textSectStartPos, (uint8)0xCC, adjustBytes);
  13894. mOut.mPos += adjustBytes;
  13895. if (jump.mCmpKind == BeCmpKind_None)
  13896. {
  13897. codeVec[jump.mCodeOffset + textSectStartPos] = 0xE9;
  13898. }
  13899. else
  13900. {
  13901. codeVec[jump.mCodeOffset + textSectStartPos] = 0x0F;
  13902. codeVec[jump.mCodeOffset + 1 + textSectStartPos] = GetJumpOpCode(jump.mCmpKind, true);
  13903. }
  13904. #define CODE_OFFSET_ADJUST(val) if (val >= adjustFrom) val += adjustBytes
  13905. for (auto& labelPosition : labelPositions)
  13906. CODE_OFFSET_ADJUST(labelPosition);
  13907. for (auto& checkJump : deferredJumps)
  13908. CODE_OFFSET_ADJUST(checkJump.mCodeOffset);
  13909. if (mDbgFunction != NULL)
  13910. {
  13911. for (auto& codeEmission : mDbgFunction->mEmissions)
  13912. CODE_OFFSET_ADJUST(codeEmission.mPos);
  13913. for (auto dbgVar : mDbgFunction->mVariables)
  13914. {
  13915. if (dbgVar == NULL)
  13916. continue;
  13917. CODE_OFFSET_ADJUST(dbgVar->mDeclStart);
  13918. CODE_OFFSET_ADJUST(dbgVar->mDeclEnd);
  13919. for (auto& range : dbgVar->mSavedRanges)
  13920. CODE_OFFSET_ADJUST(range.mOffset);
  13921. for (auto& range : dbgVar->mGaps)
  13922. CODE_OFFSET_ADJUST(range.mOffset);
  13923. }
  13924. }
  13925. for (auto& deferredUnwind : deferredUnwinds)
  13926. CODE_OFFSET_ADJUST(deferredUnwind.mCodePos);
  13927. for (int textRelocIdx : mTextRelocs)
  13928. {
  13929. auto& reloc = mCOFFObject->mTextSect.mRelocs[textRelocIdx];
  13930. if (reloc.mOffset - textSectStartPos >= adjustFrom)
  13931. reloc.mOffset += adjustBytes;
  13932. }
  13933. for (auto& dbgInstPos : dbgInstPositions)
  13934. CODE_OFFSET_ADJUST(dbgInstPos.mPos);
  13935. #undef CODE_OFFSET_ADJUST
  13936. jump.mJumpKind = 2;
  13937. didWidening = true;
  13938. }
  13939. //TODO: Test extending into a long jump
  13940. if (jump.mJumpKind == 0)
  13941. codeVec[jump.mCodeOffset + 1 + textSectStartPos] = (uint8)offset;
  13942. else if (jump.mCmpKind == BeCmpKind_None)
  13943. *(int32*)(&codeVec[jump.mCodeOffset + 1 + textSectStartPos]) = (uint32)offset;
  13944. else
  13945. *(int32*)(&codeVec[jump.mCodeOffset + 2 + textSectStartPos]) = (uint32)offset;
  13946. }
  13947. if (!didWidening)
  13948. break;
  13949. }
  13950. if (!mSwitchEntries.empty())
  13951. {
  13952. auto thisFuncSym = mCOFFObject->GetSymbol(mBeFunction);
  13953. for (auto& switchEntry : mSwitchEntries)
  13954. {
  13955. auto& sect = mCOFFObject->mRDataSect;
  13956. int32* ofsPtr = (int32*)((uint8*)sect.mData.GetPtr() + switchEntry.mOfs);
  13957. *ofsPtr = labelPositions[switchEntry.mBlock->mLabelIdx];
  13958. BeMCRelocation reloc;
  13959. reloc.mKind = BeMCRelocationKind_ADDR32NB;
  13960. reloc.mOffset = switchEntry.mOfs;
  13961. reloc.mSymTableIdx = thisFuncSym->mIdx;
  13962. sect.mRelocs.push_back(reloc);
  13963. }
  13964. }
  13965. //for (auto& deferredUnwind : deferredUnwinds)
  13966. for (int deferredUnwindIdx = (int)deferredUnwinds.size() - 1; deferredUnwindIdx >= 0; deferredUnwindIdx--)
  13967. {
  13968. auto inst = deferredUnwinds[deferredUnwindIdx].mUnwindInst;
  13969. int codePos = deferredUnwinds[deferredUnwindIdx].mCodePos;
  13970. switch (inst->mKind)
  13971. {
  13972. case BeMCInstKind_Unwind_Alloc:
  13973. {
  13974. xdata.Write((uint8)(codePos));
  13975. int allocSize = (int)inst->mArg0.mImmediate;
  13976. if (allocSize <= 128)
  13977. {
  13978. // UWOP_ALLOC_SMALL
  13979. xdata.Write((uint8)((2) | ((allocSize / 8 - 1) << 4)));
  13980. }
  13981. else if ((allocSize <= 0x7FFF8) && ((allocSize & 7) == 0)) // up to 512k-8 bytes
  13982. {
  13983. // UWOP_ALLOC_LARGE
  13984. xdata.Write((uint8)((1) | ((0) << 4)));
  13985. xdata.Write((uint8)((allocSize / 8) & 0xFF));
  13986. xdata.Write((uint8)((allocSize / 8) >> 8));
  13987. }
  13988. else
  13989. {
  13990. // UWOP_ALLOC_LARGE+
  13991. xdata.Write((uint8)((1) | ((1) << 4)));
  13992. xdata.Write((int32)allocSize);
  13993. }
  13994. }
  13995. break;
  13996. case BeMCInstKind_Unwind_PushReg:
  13997. {
  13998. xdata.Write((uint8)(codePos));
  13999. int regNum = 0;
  14000. switch (inst->mArg0.mReg)
  14001. {
  14002. case X64Reg_RAX: regNum = 0; break;
  14003. case X64Reg_RCX: regNum = 1; break;
  14004. case X64Reg_RDX: regNum = 2; break;
  14005. case X64Reg_RBX: regNum = 3; break;
  14006. case X64Reg_RSP: regNum = 4; break;
  14007. case X64Reg_RBP: regNum = 5; break;
  14008. case X64Reg_RSI: regNum = 6; break;
  14009. case X64Reg_RDI: regNum = 7; break;
  14010. case X64Reg_R8: regNum = 8; break;
  14011. case X64Reg_R9: regNum = 9; break;
  14012. case X64Reg_R10: regNum = 10; break;
  14013. case X64Reg_R11: regNum = 11; break;
  14014. case X64Reg_R12: regNum = 12; break;
  14015. case X64Reg_R13: regNum = 13; break;
  14016. case X64Reg_R14: regNum = 14; break;
  14017. case X64Reg_R15: regNum = 15; break;
  14018. default: NotImpl();
  14019. }
  14020. // UWOP_PUSH_NONVOL
  14021. xdata.Write((uint8)((0) | (regNum << 4)));
  14022. }
  14023. break;
  14024. case BeMCInstKind_Unwind_SaveXMM:
  14025. {
  14026. xdata.Write((uint8)(codePos));
  14027. int regNum = 0;
  14028. switch (inst->mArg0.mReg)
  14029. {
  14030. case X64Reg_M128_XMM0: regNum = 0; break;
  14031. case X64Reg_M128_XMM1: regNum = 1; break;
  14032. case X64Reg_M128_XMM2: regNum = 2; break;
  14033. case X64Reg_M128_XMM3: regNum = 3; break;
  14034. case X64Reg_M128_XMM4: regNum = 4; break;
  14035. case X64Reg_M128_XMM5: regNum = 5; break;
  14036. case X64Reg_M128_XMM6: regNum = 6; break;
  14037. case X64Reg_M128_XMM7: regNum = 7; break;
  14038. case X64Reg_M128_XMM8: regNum = 8; break;
  14039. case X64Reg_M128_XMM9: regNum = 9; break;
  14040. case X64Reg_M128_XMM10: regNum = 10; break;
  14041. case X64Reg_M128_XMM11: regNum = 11; break;
  14042. case X64Reg_M128_XMM12: regNum = 12; break;
  14043. case X64Reg_M128_XMM13: regNum = 13; break;
  14044. case X64Reg_M128_XMM14: regNum = 14; break;
  14045. case X64Reg_M128_XMM15: regNum = 15; break;
  14046. default: NotImpl();
  14047. }
  14048. // UWOP_SAVE_XMM128
  14049. xdata.Write((uint8)((8) | (regNum << 4)));
  14050. xdata.Write((int16)(inst->mArg1.mImmediate / 16));
  14051. }
  14052. break;
  14053. case BeMCInstKind_Unwind_SetBP:
  14054. {
  14055. xdata.Write((uint8)(codePos));
  14056. // UWOP_SET_FPREG
  14057. xdata.Write((uint8)((3) | (0 << 4)));
  14058. }
  14059. break;
  14060. }
  14061. }
  14062. int codeLen = mOut.GetPos() - textSectStartPos;
  14063. int minCodeLen = hotJumpLen;
  14064. int addCodeLen = minCodeLen - codeLen;
  14065. for (int i = 0; i < addCodeLen; i++)
  14066. {
  14067. mOut.Write((uint8)0x90);
  14068. }
  14069. if (mDbgFunction != NULL)
  14070. {
  14071. mDbgFunction->mCodeLen = mOut.GetPos() - textSectStartPos;
  14072. }
  14073. if (hasPData)
  14074. {
  14075. int codeLen = mOut.GetPos() - textSectStartPos;
  14076. // PDATA end addr
  14077. BeMCRelocation reloc;
  14078. reloc.mKind = BeMCRelocationKind_ADDR32NB;
  14079. reloc.mOffset = mCOFFObject->mPDataSect.mData.GetPos();
  14080. reloc.mSymTableIdx = mCOFFObject->GetSymbol(mBeFunction)->mIdx;
  14081. mCOFFObject->mPDataSect.mRelocs.push_back(reloc);
  14082. mCOFFObject->mPDataSect.mData.Write((int32)codeLen);
  14083. // XDATA pos
  14084. reloc.mKind = BeMCRelocationKind_ADDR32NB;
  14085. reloc.mOffset = mCOFFObject->mPDataSect.mData.GetPos();
  14086. reloc.mSymTableIdx = mCOFFObject->mXDataSect.mSymbolIdx;
  14087. mCOFFObject->mPDataSect.mRelocs.push_back(reloc);
  14088. mCOFFObject->mPDataSect.mData.Write((int32)xdataStartPos);
  14089. int numCodes = (xdata.GetPos() - xdataStartPos - 4) / 2;
  14090. if (numCodes > 0)
  14091. {
  14092. xdata.mData[xdataStartPos + 1] = (uint8)(deferredUnwinds.back().mCodePos); // prolog size
  14093. xdata.mData[xdataStartPos + 2] = (uint8)numCodes;
  14094. }
  14095. }
  14096. for (auto& dbgInstPos : dbgInstPositions)
  14097. {
  14098. auto inst = dbgInstPos.mInst;
  14099. dbgStr += StrFormat("%d[%d]", dbgInstPos.mPos, dbgInstPos.mOrigPos);
  14100. if (inst->mDbgLoc != NULL)
  14101. dbgStr += StrFormat("@%d", inst->mDbgLoc->mIdx);
  14102. if (inst->mResult.mKind != BeMCOperandKind_None)
  14103. {
  14104. dbgStr += " ";
  14105. dbgStr += ToString(inst->mResult);
  14106. dbgStr += " = ";
  14107. }
  14108. dbgStr += " ";
  14109. dbgStr += gOpName[(int)inst->mKind];
  14110. if (inst->mArg0.mKind != BeMCOperandKind_None)
  14111. {
  14112. dbgStr += " ";
  14113. dbgStr += ToString(inst->mArg0);
  14114. }
  14115. if (inst->mArg1.mKind != BeMCOperandKind_None)
  14116. {
  14117. dbgStr += ", ";
  14118. dbgStr += ToString(inst->mArg1);
  14119. }
  14120. dbgStr += "\n";
  14121. }
  14122. if ((mDebugging) && (mDbgFunction != NULL))
  14123. {
  14124. dbgStr += "\nDebug Variables:\n";
  14125. for (auto dbgVar : mDbgFunction->mVariables)
  14126. {
  14127. if (dbgVar == NULL)
  14128. continue;
  14129. dbgStr += StrFormat("%s %d to %d", dbgVar->mName.c_str(), dbgVar->mDeclStart, dbgVar->mDeclEnd);
  14130. if (dbgVar->mDeclLifetimeExtend)
  14131. dbgStr += " LifetimeExtend";
  14132. dbgStr += "\n";
  14133. for (auto& gap : dbgVar->mGaps)
  14134. {
  14135. if (gap.mLength == -1)
  14136. dbgStr += StrFormat(" Gap %d to <unterminated>\n", gap.mOffset);
  14137. else
  14138. dbgStr += StrFormat(" Gap %d to %d\n", gap.mOffset, gap.mOffset + gap.mLength);
  14139. }
  14140. for (auto& gap : dbgVar->mSavedRanges)
  14141. {
  14142. if (gap.mLength == -1)
  14143. dbgStr += StrFormat(" SavedRange %d to <unterminated>\n", gap.mOffset);
  14144. else
  14145. dbgStr += StrFormat(" SavedRange %d to %d\n", gap.mOffset, gap.mOffset + gap.mLength);
  14146. }
  14147. }
  14148. }
  14149. if (!dbgStr.empty())
  14150. {
  14151. dbgStr += "\n";
  14152. OutputDebugStr(dbgStr);
  14153. }
  14154. }
  14155. void BeMCContext::HandleParams()
  14156. {
  14157. auto beModule = mBeFunction->mModule;
  14158. int regIdxOfs = 0;
  14159. int paramOfs = 0;
  14160. auto retType = mBeFunction->GetFuncType()->mReturnType;
  14161. X64CPURegister compositeRetReg = X64Reg_None;
  14162. bool flipFirstRegs = false;
  14163. if (mBeFunction->HasStructRet())
  14164. {
  14165. flipFirstRegs = mBeFunction->mCallingConv == BfIRCallingConv_ThisCall;
  14166. //paramOfs = 1;
  14167. /*auto ptrType = (BePointerType*)mBeFunction->mFuncType->mParams[0].mType;
  14168. BF_ASSERT(ptrType->mTypeCode == BeTypeCode_Pointer);
  14169. retType = ptrType->mElementType;*/
  14170. //retType = mBeFunction->mFuncType->mParams[0].mType;
  14171. //flipFirstRegs = mBeFunction->mCallingConv == BfIRCallingConv_ThisCall;
  14172. /*
  14173. auto beArg = beModule->GetArgument(0);
  14174. compositeRetReg = (mBeFunction->mCallingConv == BfIRCallingConv_ThisCall) ? X64Reg_RDX : X64Reg_RCX;
  14175. mParamsUsedRegs.push_back(compositeRetReg);
  14176. BeMCOperand mcOperand;
  14177. mcOperand.mReg = compositeRetReg;
  14178. mcOperand.mKind = BeMCOperandKind_NativeReg;
  14179. auto ptrType = (BePointerType*)mBeFunction->mFuncType->mParams[0].mType;
  14180. BF_ASSERT(ptrType->mTypeCode == BeTypeCode_Pointer);
  14181. auto paramVReg = AllocVirtualReg(ptrType->mElementType);
  14182. auto paramVRegInfo = GetVRegInfo(paramVReg);
  14183. CreateDefineVReg(paramVReg);
  14184. paramVRegInfo->mNaturalReg = compositeRetReg;
  14185. AllocInst(BeMCInstKind_Mov, paramVReg, mcOperand);
  14186. mValueToOperand[beArg] = paramVReg;*/
  14187. }
  14188. else if (retType->IsNonVectorComposite())
  14189. {
  14190. compositeRetReg = (mBeFunction->mCallingConv == BfIRCallingConv_ThisCall) ? X64Reg_RDX : X64Reg_RCX;
  14191. auto retVReg = AllocVirtualReg(mModule->mContext->GetPrimitiveType(BeTypeCode_Int64));
  14192. auto retVRegInfo = GetVRegInfo(retVReg);
  14193. retVRegInfo->mNaturalReg = compositeRetReg;
  14194. retVRegInfo->mForceReg = true;
  14195. retVRegInfo->mMustExist = true;
  14196. AllocInst(BeMCInstKind_Mov, retVReg, BeMCOperand::FromReg(compositeRetReg));
  14197. mCompositeRetVRegIdx = retVReg.mVRegIdx;
  14198. mParamsUsedRegs.push_back(compositeRetReg);
  14199. }
  14200. for (int paramIdx = 0; paramIdx < (int)mBeFunction->mParams.size() - paramOfs; paramIdx++)
  14201. {
  14202. if (((paramIdx == 0) && (compositeRetReg == X64Reg_RCX)) ||
  14203. ((paramIdx == 1) && (compositeRetReg == X64Reg_RDX)))
  14204. regIdxOfs = 1;
  14205. auto funcType = mBeFunction->GetFuncType();
  14206. auto& typeParam = funcType->mParams[paramIdx + paramOfs];
  14207. auto& param = mBeFunction->mParams[paramIdx + paramOfs];
  14208. auto beArg = beModule->GetArgument(paramIdx + paramOfs);
  14209. BeMCOperand mcOperand;
  14210. mcOperand.mReg = X64Reg_None;
  14211. mcOperand.mKind = BeMCOperandKind_NativeReg;
  14212. int regIdx = paramIdx + regIdxOfs;
  14213. if (typeParam.mType->IsFloat())
  14214. {
  14215. switch (regIdx)
  14216. {
  14217. case 0:
  14218. mcOperand.mReg = X64Reg_M128_XMM0; //X64Reg_XMM0_f64;
  14219. break;
  14220. case 1:
  14221. mcOperand.mReg = X64Reg_M128_XMM1; //X64Reg_XMM1_f64;
  14222. break;
  14223. case 2:
  14224. mcOperand.mReg = X64Reg_M128_XMM2; //X64Reg_XMM2_f64;
  14225. break;
  14226. case 3:
  14227. mcOperand.mReg = X64Reg_M128_XMM3; //X64Reg_XMM3_f64;
  14228. break;
  14229. }
  14230. }
  14231. else
  14232. {
  14233. switch (regIdx)
  14234. {
  14235. case 0:
  14236. mcOperand.mReg = !flipFirstRegs ? X64Reg_RCX : X64Reg_RDX;
  14237. break;
  14238. case 1:
  14239. mcOperand.mReg = !flipFirstRegs ? X64Reg_RDX : X64Reg_RCX;
  14240. break;
  14241. case 2:
  14242. mcOperand.mReg = X64Reg_R8;
  14243. break;
  14244. case 3:
  14245. mcOperand.mReg = X64Reg_R9;
  14246. break;
  14247. }
  14248. }
  14249. if (mcOperand.mReg != X64Reg_None)
  14250. {
  14251. mParamsUsedRegs.push_back(mcOperand.mReg);
  14252. mcOperand.mReg = ResizeRegister(mcOperand.mReg, typeParam.mType);
  14253. }
  14254. BeMCOperand paramVReg;
  14255. /*if ((paramIdx == 0) && (mBeFunction->mStructRet))
  14256. {
  14257. auto ptrType = (BePointerType*)typeParam.mType;
  14258. BF_ASSERT(ptrType->mTypeCode == BeTypeCode_Pointer);
  14259. paramVReg = AllocVirtualReg(ptrType->mElementType);
  14260. paramVReg.mKind = BeMCOperandKind_VRegAddr;
  14261. //paramVReg.mR
  14262. }
  14263. else*/
  14264. paramVReg = AllocVirtualReg(typeParam.mType);
  14265. auto paramVRegInfo = GetVRegInfo(paramVReg);
  14266. if ((mBeFunction->HasStructRet()) && (paramIdx == 0))
  14267. {
  14268. paramVRegInfo->SetRetVal();
  14269. }
  14270. else
  14271. paramVRegInfo->mForceMerge = true;
  14272. CreateDefineVReg(paramVReg);
  14273. if (mcOperand.mReg != X64Reg_None)
  14274. {
  14275. // This indirection allows us to NOT directly use a register for a parameter
  14276. // if the cost of saving/restoring this volatile reg is too great
  14277. paramVRegInfo->mNaturalReg = mcOperand.mReg;
  14278. AllocInst(BeMCInstKind_Mov, paramVReg, mcOperand);
  14279. }
  14280. else
  14281. {
  14282. paramVRegInfo->mMustExist = true;
  14283. paramVRegInfo->mForceMem = true;
  14284. paramVRegInfo->mFrameOffset = paramIdx * 8 + 8;
  14285. CreateDefineVReg(paramVReg);
  14286. }
  14287. //paramVRegInfo->mDbgVariable = mDbgFunction->mParams[paramIdx];
  14288. mValueToOperand[beArg] = paramVReg;
  14289. }
  14290. }
  14291. void BeMCContext::ToString(BeMCInst* inst, String& str, bool showVRegFlags, bool showVRegDetails)
  14292. {
  14293. if (inst == NULL)
  14294. {
  14295. str += "NULL\n";
  14296. return;
  14297. }
  14298. if (inst->mKind == BeMCInstKind_Label)
  14299. {
  14300. str += ToString(inst->mArg0);
  14301. str += ":";
  14302. }
  14303. else
  14304. {
  14305. str += " ";
  14306. if (inst->mResult)
  14307. {
  14308. str += ToString(inst->mResult);
  14309. str += " = ";
  14310. }
  14311. str += gOpName[(int)inst->mKind];
  14312. if (inst->mKind == BeMCInstKind_DbgDecl)
  14313. {
  14314. auto vregInfo = GetVRegInfo(inst->mArg0);
  14315. if ((vregInfo != NULL) && (vregInfo->mDbgVariable != NULL))
  14316. {
  14317. if (vregInfo->mDbgVariable->mIsValue)
  14318. str += " <value>";
  14319. else
  14320. str += " <addr>";
  14321. }
  14322. }
  14323. if (inst->mKind == BeMCInstKind_DefPhi)
  14324. {
  14325. str += " ";
  14326. str += ToString(inst->mArg0);
  14327. for (auto& val : inst->mArg0.mPhi->mValues)
  14328. {
  14329. str += ", [";
  14330. str += ToString(BeMCOperand::FromBlock(val.mBlockFrom));
  14331. str += ", ";
  14332. str += ToString(val.mValue);
  14333. str += "]";
  14334. }
  14335. }
  14336. else if (inst->mKind == BeMCInstKind_Load)
  14337. {
  14338. str += " ";
  14339. str += ToString(inst->mArg0);
  14340. str += ", [";
  14341. str += ToString(inst->mArg1);
  14342. str += "]";
  14343. }
  14344. else if (inst->mKind == BeMCInstKind_Store)
  14345. {
  14346. str += " [";
  14347. str += ToString(inst->mArg0);
  14348. str += "], ";
  14349. str += ToString(inst->mArg1);
  14350. }
  14351. else if (inst->mArg0.mKind != BeMCOperandKind_None)
  14352. {
  14353. str += " ";
  14354. str += ToString(inst->mArg0);
  14355. if (inst->mArg1.mKind != BeMCOperandKind_None)
  14356. {
  14357. str += ", ";
  14358. str += ToString(inst->mArg1);
  14359. }
  14360. }
  14361. }
  14362. if (inst->IsDef())
  14363. {
  14364. auto vregInfo = mVRegInfo[inst->mArg0.mVRegIdx];
  14365. if (vregInfo->mRefCount != -1)
  14366. str += StrFormat(" : %d refs", vregInfo->mRefCount);
  14367. }
  14368. /*if (inst->mKind == BeMCInstKind_Def)
  14369. {
  14370. str += " ";
  14371. str += mModule->ToString(GetType(inst->mArg0));
  14372. }*/
  14373. bool hadSemi = false;
  14374. if (inst->mDbgLoc != NULL)
  14375. {
  14376. str += StrFormat(" ; @%d[%d:%d]", inst->mDbgLoc->mIdx, inst->mDbgLoc->mLine + 1, inst->mDbgLoc->mColumn + 1);
  14377. hadSemi = true;
  14378. }
  14379. bool showLiveness = showVRegFlags;
  14380. if ((showLiveness) && (inst->mLiveness != NULL))
  14381. {
  14382. if (!hadSemi)
  14383. {
  14384. str += " ;";
  14385. hadSemi = true;
  14386. }
  14387. bool isFirstInScope = true;
  14388. str += " live: ";
  14389. int vregIdx = -1;
  14390. for (int nextVRegIdx : *inst->mLiveness)
  14391. {
  14392. if (nextVRegIdx >= mLivenessContext.mNumItems)
  14393. {
  14394. vregIdx = nextVRegIdx;
  14395. int showVRegIdx = vregIdx - mLivenessContext.mNumItems;
  14396. if (!mLivenessContext.IsSet(inst->mLiveness, showVRegIdx))
  14397. str += StrFormat(", %d*", showVRegIdx);
  14398. continue;
  14399. }
  14400. if (vregIdx != -1)
  14401. str += ", ";
  14402. vregIdx = nextVRegIdx;
  14403. str += StrFormat("%d", vregIdx);
  14404. if (showVRegDetails)
  14405. {
  14406. auto vregInfo = mVRegInfo[vregIdx];
  14407. if (vregInfo->mForceReg)
  14408. str += "r";
  14409. if (vregInfo->mForceMem)
  14410. str += "m";
  14411. if (vregInfo->mSpilled)
  14412. str += "s";
  14413. }
  14414. }
  14415. if (inst->mLiveness->mNumChanges > 0)
  14416. {
  14417. str += " | ";
  14418. for (int changeIdx = 0; changeIdx < inst->mLiveness->mNumChanges; changeIdx++)
  14419. {
  14420. if (changeIdx != 0)
  14421. str += ", ";
  14422. int vregIdx = inst->mLiveness->GetChange(changeIdx);
  14423. if (vregIdx >= 0)
  14424. str += StrFormat("+%d", vregIdx);
  14425. else
  14426. str += StrFormat("-%d", -vregIdx - 1);
  14427. }
  14428. }
  14429. }
  14430. bool showInitialized = showVRegFlags;
  14431. if ((showInitialized) && (inst->mVRegsInitialized != NULL))
  14432. {
  14433. if (!hadSemi)
  14434. {
  14435. str += " ;";
  14436. hadSemi = true;
  14437. }
  14438. bool isFirstUninit = true;
  14439. str += " init: ";
  14440. int vregIdx = -1;
  14441. for (int nextVRegIdx : *inst->mVRegsInitialized)
  14442. {
  14443. if (nextVRegIdx >= mVRegInitializedContext.mNumItems)
  14444. {
  14445. if (isFirstUninit)
  14446. str += " uninit: ";
  14447. else
  14448. str += ", ";
  14449. isFirstUninit = false;
  14450. vregIdx = nextVRegIdx;
  14451. str += StrFormat("%d", vregIdx - mVRegInitializedContext.mNumItems);
  14452. }
  14453. else
  14454. {
  14455. if (vregIdx != -1)
  14456. str += ", ";
  14457. vregIdx = nextVRegIdx;
  14458. str += StrFormat("%d", vregIdx);
  14459. auto vregInfo = mVRegInfo[vregIdx];
  14460. if (vregInfo->mValueScopeRetainedKind == BeMCValueScopeRetainKind_Soft)
  14461. str += "r";
  14462. else if (vregInfo->mValueScopeRetainedKind == BeMCValueScopeRetainKind_Hard)
  14463. str += "rh";
  14464. }
  14465. }
  14466. if (inst->mVRegsInitialized->mNumChanges > 0)
  14467. {
  14468. str += " | ";
  14469. for (int changeIdx = 0; changeIdx < inst->mVRegsInitialized->mNumChanges; changeIdx++)
  14470. {
  14471. if (changeIdx != 0)
  14472. str += ", ";
  14473. int vregIdx = inst->mVRegsInitialized->GetChange(changeIdx);
  14474. if (vregIdx >= 0)
  14475. str += StrFormat("+%d", vregIdx);
  14476. else
  14477. str += StrFormat("-%d", -vregIdx - 1);
  14478. }
  14479. }
  14480. }
  14481. if (inst->mVRegLastUseRecord != NULL)
  14482. {
  14483. str += " lastUse: ";
  14484. auto checkLastUse = inst->mVRegLastUseRecord;
  14485. while (checkLastUse != NULL)
  14486. {
  14487. if (checkLastUse != inst->mVRegLastUseRecord)
  14488. str += ", ";
  14489. str += StrFormat("%d", checkLastUse->mVRegIdx);
  14490. checkLastUse = checkLastUse->mNext;
  14491. }
  14492. }
  14493. str += "\n";
  14494. }
  14495. String BeMCContext::ToString(bool showVRegFlags, bool showVRegDetails)
  14496. {
  14497. String str;
  14498. str += mBeFunction->mName;
  14499. str += "\n";
  14500. str += StrFormat("Stack Size: 0x%X\n", mStackSize);
  14501. str += "Frame Objects:\n";
  14502. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  14503. {
  14504. int showVRegIdx = vregIdx;
  14505. auto vregInfo = mVRegInfo[showVRegIdx];
  14506. if ((vregInfo->mIsRetVal) && (mCompositeRetVRegIdx != -1))
  14507. {
  14508. showVRegIdx = mCompositeRetVRegIdx;
  14509. vregInfo = mVRegInfo[showVRegIdx];
  14510. }
  14511. if (vregInfo->mFrameOffset != INT_MIN)
  14512. {
  14513. str += " ";
  14514. str += ToString(BeMCOperand::FromVReg(vregIdx));
  14515. str += StrFormat(": size=%d, align=%d, at ", vregInfo->mType->mSize, vregInfo->mAlign);
  14516. X64CPURegister reg;
  14517. int offset;
  14518. GetValAddr(BeMCOperand::FromVRegAddr(showVRegIdx), reg, offset);
  14519. str += "[";
  14520. str += X64CPURegisters::GetRegisterName(reg);
  14521. if (offset != 0)
  14522. str += StrFormat(" + 0x%X", offset);
  14523. str += "]";
  14524. str += "\n";
  14525. }
  14526. }
  14527. str += "\n";
  14528. for (int blockIdx = 0; blockIdx < (int)mBlocks.size(); blockIdx++)
  14529. {
  14530. auto mcBlock = mBlocks[blockIdx];
  14531. if (blockIdx > 0)
  14532. str += "\n";
  14533. if (mBlocks.size() > 1)
  14534. {
  14535. str += mcBlock->mName;
  14536. str += ":";
  14537. if (mcBlock->mIsLooped)
  14538. str += " ; looped";
  14539. str += " ; preds = ";
  14540. for (int predIdx = 0; predIdx < (int)mcBlock->mPreds.size(); predIdx++)
  14541. {
  14542. if (predIdx != 0)
  14543. str += ", ";
  14544. str += "%";
  14545. str += mcBlock->mPreds[predIdx]->mName;
  14546. }
  14547. ///
  14548. str += " ; succs = ";
  14549. for (int succIdx = 0; succIdx < (int)mcBlock->mSuccs.size(); succIdx++)
  14550. {
  14551. if (succIdx != 0)
  14552. str += ", ";
  14553. str += "%";
  14554. str += mcBlock->mSuccs[succIdx]->mName;
  14555. }
  14556. //
  14557. str += "\n";
  14558. }
  14559. for (auto inst : mcBlock->mInstructions)
  14560. ToString(inst, str, showVRegFlags, showVRegDetails);
  14561. }
  14562. str += "\n";
  14563. return str;
  14564. }
  14565. void BeMCContext::Print()
  14566. {
  14567. OutputDebugStr(ToString(true, false));
  14568. }
  14569. void BeMCContext::Print(bool showVRegFlags, bool showVRegDetails)
  14570. {
  14571. OutputDebugStr(ToString(showVRegFlags, showVRegDetails));
  14572. }
  14573. BeMCOperand BeMCContext::AllocBinaryOp(BeMCInstKind instKind, const BeMCOperand& lhs, const BeMCOperand& rhs, BeMCBinIdentityKind identityKind, BeMCOverflowCheckKind overflowCheckKind)
  14574. {
  14575. if ((lhs.IsImmediate()) && (lhs.mKind == rhs.mKind))
  14576. {
  14577. if (instKind == BeMCInstKind_Add)
  14578. {
  14579. BeMCOperand result;
  14580. result.mKind = lhs.mKind;
  14581. switch (lhs.mKind)
  14582. {
  14583. case BeMCOperandKind_Immediate_i32:
  14584. result.mImmediate = lhs.mImmediate + rhs.mImmediate;
  14585. return result;
  14586. }
  14587. }
  14588. }
  14589. if (identityKind == BeMCBinIdentityKind_Any_IsOne)
  14590. {
  14591. if (((lhs.IsImmediateFloat()) && (lhs.GetImmediateDouble() == 1.0)) ||
  14592. ((lhs.IsImmediateInt()) && (lhs.mImmediate == 1)))
  14593. return rhs;
  14594. }
  14595. if (identityKind == BeMCBinIdentityKind_Right_IsOne_Result_Zero)
  14596. {
  14597. if (((rhs.IsImmediateFloat()) && (rhs.GetImmediateDouble() == 1.0)) ||
  14598. ((rhs.IsImmediateInt()) && (rhs.mImmediate == 1)))
  14599. {
  14600. BeMCOperand operand = rhs;
  14601. operand.mImmediate = 0;
  14602. return operand;
  14603. }
  14604. }
  14605. if ((identityKind == BeMCBinIdentityKind_Right_IsOne) || (identityKind == BeMCBinIdentityKind_Any_IsOne))
  14606. {
  14607. if (((rhs.IsImmediateFloat()) && (rhs.GetImmediateDouble() == 1.0)) ||
  14608. ((rhs.IsImmediateInt()) && (rhs.mImmediate == 1)))
  14609. return lhs;
  14610. }
  14611. if ((identityKind == BeMCBinIdentityKind_Right_IsZero) || (identityKind == BeMCBinIdentityKind_Any_IsZero))
  14612. {
  14613. if (((rhs.IsImmediateFloat()) && (rhs.GetImmediateDouble() == 0.0)) ||
  14614. ((rhs.IsImmediateInt()) && (rhs.mImmediate == 0)))
  14615. return lhs;
  14616. }
  14617. if (identityKind == BeMCBinIdentityKind_Any_IsZero)
  14618. {
  14619. if (((lhs.IsImmediateFloat()) && (lhs.GetImmediateDouble() == 0.0)) ||
  14620. ((lhs.IsImmediateInt()) && (lhs.mImmediate == 0)))
  14621. return rhs;
  14622. }
  14623. auto result = AllocVirtualReg(GetType(lhs));
  14624. AllocInst(BeMCInstKind_Def, result);
  14625. auto mcInst = AllocInst(instKind, lhs, rhs);
  14626. mcInst->mResult = result;
  14627. if (overflowCheckKind != BeMCOverflowCheckKind_None)
  14628. {
  14629. mcInst->mDisableShortForm = true;
  14630. AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromImmediate(1), BeMCOperand::FromCmpKind((overflowCheckKind == BeMCOverflowCheckKind_B) ? BeCmpKind_NB : BeCmpKind_NO));
  14631. AllocInst(BeMCInstKind_DbgBreak);
  14632. }
  14633. return result;
  14634. }
  14635. void BeMCContext::Generate(BeFunction* function)
  14636. {
  14637. BP_ZONE_F("BeMCContext::Generate %s", function->mName.c_str());
  14638. mBeFunction = function;
  14639. mDbgFunction = mBeFunction->mDbgFunction;
  14640. mModule = function->mModule;
  14641. if (!mModule->mTargetCPU.IsEmpty())
  14642. mModule->mBeIRCodeGen->Fail(StrFormat("Cannot set Target CPU to '%s' for +Og optimization. Considering compiling under a different optimization setting.", mModule->mTargetCPU.c_str()));
  14643. if ((!mModule->mTargetTriple.IsEmpty()) && (!mModule->mTargetTriple.StartsWith("x86_64-pc-windows")))
  14644. {
  14645. mModule->mBeIRCodeGen->Fail(StrFormat("Cannot set Target Triple to '%s' for +Og optimization. Considering compiling under a different optimization setting.", mModule->mTargetTriple.c_str()));
  14646. return;
  14647. }
  14648. //mDbgPreferredRegs[15] = X64Reg_RCX;
  14649. //mDbgPreferredRegs[7] = X64Reg_RCX;
  14650. /*mDbgPreferredRegs[14] = X64Reg_RAX;
  14651. mDbgPreferredRegs[15] = X64Reg_None;
  14652. mDbgPreferredRegs[19] = X64Reg_None;
  14653. mDbgPreferredRegs[31] = X64Reg_R8;
  14654. mDbgPreferredRegs[32] = X64Reg_R8;*/
  14655. //mDbgPreferredRegs[8] = X64Reg_RAX;
  14656. mDebugging = (function->mName == "?MyFunction@MyStruct@BeefTest5@bf@@QEAAXAEAI@Z");
  14657. // || (function->mName == "?MethodA@TestProgram@BeefTest@bf@@CAXXZ");
  14658. // || (function->mName == "?Hey@Blurg@bf@@SAXXZ")
  14659. // ;
  14660. //"?ColorizeCodeString@IDEUtils@IDE@bf@@SAXPEAVString@System@3@W4CodeKind@123@@Z";
  14661. //"?Main@Program@bf@@CAHPEAV?$Array1@PEAVString@System@bf@@@System@2@@Z";
  14662. //"?Hey@Blurg@bf@@SAXXZ";
  14663. //"?get__Value@?$Nullable@ULineAndColumn@EditWidgetContent@widgets@Beefy@bf@@@System@bf@@QEAAULineAndColumn@EditWidgetContent@widgets@Beefy@3@XZ";
  14664. //"?__BfCtor@StructA@bf@@QEAAXXZ";
  14665. if (mDebugging)
  14666. {
  14667. mModule->Print(mBeFunction);
  14668. }
  14669. for (auto beBlock : function->mBlocks)
  14670. {
  14671. int blockIdx = (int)mMCBlockAlloc.size();
  14672. auto mcBlock = mMCBlockAlloc.Alloc();
  14673. mcBlock->mName = beBlock->mName + StrFormat(":%d", blockIdx);
  14674. mcBlock->mBlockIdx = blockIdx;
  14675. mcBlock->mMaxDeclBlockId = blockIdx;
  14676. BeMCOperand mcOperand;
  14677. mcOperand.mKind = BeMCOperandKind_Block;
  14678. mcOperand.mBlock = mcBlock;
  14679. mValueToOperand[beBlock] = mcOperand;
  14680. mBlocks.push_back(mcBlock);
  14681. }
  14682. SizedArray<int, 64> dbgVarsAwaitingEnd;
  14683. BeMDNode* curDbgScope = NULL;
  14684. bool inHeadAlloca = true;
  14685. SizedArray<int, 64> stackSaveVRegs;
  14686. // Scan pass
  14687. mMaxCallParamCount = -1;
  14688. for (int blockIdx = 0; blockIdx < (int)function->mBlocks.size(); blockIdx++)
  14689. {
  14690. auto beBlock = function->mBlocks[blockIdx];
  14691. auto mcBlock = mBlocks[blockIdx];
  14692. for (int instIdx = 0; instIdx < (int)beBlock->mInstructions.size(); instIdx++)
  14693. {
  14694. auto inst = beBlock->mInstructions[instIdx];
  14695. int instType = inst->GetTypeId();
  14696. switch (instType)
  14697. {
  14698. case BeAllocaInst::TypeId:
  14699. {
  14700. auto castedInst = (BeAllocaInst*)inst;
  14701. if ((!inHeadAlloca) || (castedInst->mAlign > 16))
  14702. mUseBP = true;
  14703. }
  14704. break;
  14705. case BeNumericCastInst::TypeId:
  14706. case BeBitCastInst::TypeId:
  14707. break;
  14708. case BeStackSaveInst::TypeId:
  14709. {
  14710. auto stackVReg = AllocVirtualReg(mNativeIntType);
  14711. stackSaveVRegs.push_back(stackVReg.mVRegIdx);
  14712. }
  14713. break;
  14714. case BeCallInst::TypeId:
  14715. {
  14716. auto castedInst = (BeCallInst*)inst;
  14717. if (auto intrin = BeValueDynCast<BeIntrinsic>(castedInst->mFunc))
  14718. {
  14719. // Not a real call
  14720. switch (intrin->mKind)
  14721. {
  14722. case BfIRIntrinsic_VAStart:
  14723. mHasVAStart = true;
  14724. break;
  14725. }
  14726. }
  14727. }
  14728. break;
  14729. case BeMemSetInst::TypeId:
  14730. {
  14731. //mMaxCallParamCount = BF_MAX(mMaxCallParamCount, 4);
  14732. }
  14733. break;
  14734. default:
  14735. inHeadAlloca = false;
  14736. break;
  14737. }
  14738. }
  14739. }
  14740. // if (mMaxCallParamCount != -1)
  14741. // mMaxCallParamCount = BF_MAX(mMaxCallParamCount, 4);
  14742. int retCount = 0;
  14743. bool isFirstBlock = true;
  14744. inHeadAlloca = true;
  14745. SizedArray<int, 64> valueScopeStack;
  14746. // Generate pass
  14747. for (int blockIdx = 0; blockIdx < (int)function->mBlocks.size(); blockIdx++)
  14748. {
  14749. auto beBlock = function->mBlocks[blockIdx];
  14750. auto mcBlock = mBlocks[blockIdx];
  14751. mActiveBeBlock = beBlock;
  14752. mActiveBlock = mcBlock;
  14753. if (isFirstBlock)
  14754. HandleParams();
  14755. for (int instIdx = 0; instIdx < (int)beBlock->mInstructions.size(); instIdx++)
  14756. {
  14757. auto inst = beBlock->mInstructions[instIdx];
  14758. BeMCOperand result;
  14759. mCurDbgLoc = inst->mDbgLoc;
  14760. int instType = inst->GetTypeId();
  14761. switch (instType)
  14762. {
  14763. case BeAllocaInst::TypeId:
  14764. case BeNumericCastInst::TypeId:
  14765. case BeBitCastInst::TypeId:
  14766. break;
  14767. default:
  14768. inHeadAlloca = false;
  14769. break;
  14770. }
  14771. switch (instType)
  14772. {
  14773. case BeNopInst::TypeId:
  14774. {
  14775. auto mcInst = AllocInst();
  14776. mcInst->mKind = BeMCInstKind_Nop;
  14777. }
  14778. break;
  14779. case BeUnreachableInst::TypeId:
  14780. {
  14781. auto mcInst = AllocInst();
  14782. mcInst->mKind = BeMCInstKind_Unreachable;
  14783. // if (instIdx == beBlock->mInstructions.size() - 1)
  14784. // {
  14785. // // Fake branch to exit
  14786. // mcInst = AllocInst();
  14787. // mcInst->mKind = BeMCInstKind_Br;
  14788. // mcInst->mArg0 = BeMCOperand::FromBlock(mBlocks.back());
  14789. // mcInst->mArg0.mBlock->AddPred(mcBlock);
  14790. // }
  14791. }
  14792. break;
  14793. case BeEnsureInstructionAtInst::TypeId:
  14794. {
  14795. auto mcInst = AllocInst();
  14796. mcInst->mKind = BeMCInstKind_EnsureInstructionAt;
  14797. }
  14798. break;
  14799. case BeUndefValueInst::TypeId:
  14800. {
  14801. auto castedInst = (BeUndefValueInst*)inst;
  14802. result = AllocVirtualReg(castedInst->mType);
  14803. CreateDefineVReg(result);
  14804. }
  14805. break;
  14806. case BeExtractValueInst::TypeId:
  14807. {
  14808. auto castedInst = (BeExtractValueInst*)inst;
  14809. BeConstant* constant = BeValueDynCast<BeConstant>(castedInst->mAggVal);
  14810. BeMCOperand mcAgg;
  14811. if (constant == NULL)
  14812. {
  14813. mcAgg = GetOperand(castedInst->mAggVal);
  14814. if (mcAgg.mKind == BeMCOperandKind_ConstAgg)
  14815. {
  14816. constant = mcAgg.mConstant;
  14817. }
  14818. }
  14819. if (constant != NULL)
  14820. {
  14821. result.mImmediate = 0;
  14822. BeType* wantDefaultType = NULL;
  14823. if (constant->mType->IsStruct())
  14824. {
  14825. BeStructType* structType = (BeStructType*)constant->mType;
  14826. auto& member = structType->mMembers[castedInst->mIdx];
  14827. wantDefaultType = member.mType;
  14828. }
  14829. else if (constant->mType->IsSizedArray())
  14830. {
  14831. BeSizedArrayType* arrayType = (BeSizedArrayType*)constant->mType;
  14832. wantDefaultType = arrayType->mElementType;
  14833. }
  14834. if (wantDefaultType != NULL)
  14835. {
  14836. switch (wantDefaultType->mTypeCode)
  14837. {
  14838. case BeTypeCode_Boolean:
  14839. case BeTypeCode_Int8:
  14840. result.mKind = BeMCOperandKind_Immediate_i8;
  14841. break;
  14842. case BeTypeCode_Int16:
  14843. result.mKind = BeMCOperandKind_Immediate_i16;
  14844. break;
  14845. case BeTypeCode_Int32:
  14846. result.mKind = BeMCOperandKind_Immediate_i32;
  14847. break;
  14848. case BeTypeCode_Int64:
  14849. result.mKind = BeMCOperandKind_Immediate_i64;
  14850. break;
  14851. case BeTypeCode_Float:
  14852. result.mKind = BeMCOperandKind_Immediate_f32;
  14853. break;
  14854. case BeTypeCode_Double:
  14855. result.mKind = BeMCOperandKind_Immediate_f64;
  14856. break;
  14857. case BeTypeCode_Pointer:
  14858. result.mKind = BeMCOperandKind_Immediate_Null;
  14859. result.mType = wantDefaultType;
  14860. break;
  14861. case BeTypeCode_Struct:
  14862. case BeTypeCode_SizedArray:
  14863. {
  14864. auto subConst = mAlloc.Alloc<BeConstant>();
  14865. subConst->mType = wantDefaultType;
  14866. result.mConstant = subConst;
  14867. result.mKind = BeMCOperandKind_ConstAgg;
  14868. }
  14869. break;
  14870. default:
  14871. NotImpl();
  14872. }
  14873. }
  14874. break;
  14875. }
  14876. auto aggType = GetType(mcAgg);
  14877. int byteOffset = 0;
  14878. BeType* memberType = NULL;
  14879. if (aggType->IsSizedArray())
  14880. {
  14881. auto sizedArray = (BeSizedArrayType*)aggType;
  14882. memberType = sizedArray->mElementType;
  14883. byteOffset = BF_ALIGN(memberType->mSize, memberType->mAlign) * castedInst->mIdx;
  14884. }
  14885. else
  14886. {
  14887. BF_ASSERT(aggType->IsStruct());
  14888. BeStructType* structType = (BeStructType*)aggType;
  14889. auto& structMember = structType->mMembers[castedInst->mIdx];
  14890. byteOffset = structMember.mByteOffset;
  14891. memberType = structMember.mType;
  14892. }
  14893. if (mcAgg.mKind == BeMCOperandKind_VReg)
  14894. mcAgg.mKind = BeMCOperandKind_VRegAddr;
  14895. else if (mcAgg.mKind == BeMCOperandKind_VRegLoad)
  14896. mcAgg.mKind = BeMCOperandKind_VReg;
  14897. else
  14898. NotImpl();
  14899. auto memberPtrType = mModule->mContext->GetPointerTo(memberType);
  14900. result = AllocRelativeVirtualReg(memberPtrType, mcAgg, BeMCOperand::FromImmediate(byteOffset), 1);
  14901. result.mKind = BeMCOperandKind_VRegLoad;
  14902. CreateDefineVReg(result);
  14903. }
  14904. break;
  14905. case BeInsertValueInst::TypeId:
  14906. {
  14907. auto castedInst = (BeInsertValueInst*)inst;
  14908. auto mcAgg = GetOperand(castedInst->mAggVal);
  14909. auto mcValue = GetOperand(castedInst->mMemberVal);
  14910. auto aggType = GetType(mcAgg);
  14911. BF_ASSERT(aggType->IsStruct());
  14912. BeStructType* structType = (BeStructType*)aggType;
  14913. auto& structMember = structType->mMembers[castedInst->mIdx];
  14914. BF_ASSERT(mcAgg.mKind = BeMCOperandKind_VReg);
  14915. auto mcAggRef = mcAgg;
  14916. mcAggRef.mKind = BeMCOperandKind_VRegAddr;
  14917. auto memberPtrType = mModule->mContext->GetPointerTo(structMember.mType);
  14918. auto mcMemberRef = AllocRelativeVirtualReg(memberPtrType, mcAggRef, BeMCOperand::FromImmediate(structMember.mByteOffset), 1);
  14919. CreateDefineVReg(mcMemberRef);
  14920. mcMemberRef.mKind = BeMCOperandKind_VRegLoad;
  14921. AllocInst(BeMCInstKind_Mov, mcMemberRef, mcValue);
  14922. // Our InsertValue always modifies the source aggregate, it does not make a copy like LLVM's InsertValue would infer.
  14923. // This is okay because of Beef front end knowledge, but is not general purpose.
  14924. result = mcAgg;
  14925. }
  14926. break;
  14927. case BeNumericCastInst::TypeId:
  14928. {
  14929. auto castedInst = (BeNumericCastInst*)inst;
  14930. auto mcValue = GetOperand(castedInst->mValue);
  14931. auto fromType = GetType(mcValue);
  14932. if (fromType == castedInst->mToType)
  14933. {
  14934. // If it's just a sign change then leave it alone
  14935. result = mcValue;
  14936. }
  14937. else
  14938. {
  14939. auto toType = castedInst->mToType;
  14940. auto toValue = AllocVirtualReg(castedInst->mToType);
  14941. CreateDefineVReg(toValue);
  14942. if ((toType->IsIntable()) && (fromType->IsIntable()) && (toType->mSize < fromType->mSize))
  14943. {
  14944. // For truncating values, no actual instructions are needed, so we can just do a vreg relto ref
  14945. auto vregInfo = mVRegInfo[toValue.mVRegIdx];
  14946. vregInfo->mIsExpr = true;
  14947. vregInfo->mRelTo = mcValue;
  14948. }
  14949. else if ((toType->IsFloat()) && (fromType->IsIntable()) && (fromType->mSize == 8) && (!castedInst->mValSigned))
  14950. {
  14951. // uint64 to float - basically, when we are signed then we shift down one bit (so it's unsigned) and then double the result. There's a 1-bit correction factor.
  14952. AllocInst(BeMCInstKind_Test, mcValue, mcValue);
  14953. AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromLabel(mCurLabelIdx), BeMCOperand::FromCmpKind(BeCmpKind_Sign));
  14954. AllocInst(BeMCInstKind_MovSX, toValue, mcValue);
  14955. AllocInst(BeMCInstKind_Br, BeMCOperand::FromLabel(mCurLabelIdx + 1));
  14956. CreateLabel();
  14957. auto temp0 = AllocVirtualReg(GetType(mcValue));
  14958. CreateDefineVReg(temp0);
  14959. auto temp1 = AllocVirtualReg(GetType(mcValue));
  14960. CreateDefineVReg(temp1);
  14961. AllocInst(BeMCInstKind_Mov, temp0, mcValue);
  14962. AllocInst(BeMCInstKind_Shr, temp0, BeMCOperand::FromImmediate(1));
  14963. AllocInst(BeMCInstKind_Mov, temp1, mcValue);
  14964. AllocInst(BeMCInstKind_And, temp1, BeMCOperand::FromImmediate(1));
  14965. AllocInst(BeMCInstKind_Or, temp0, temp1);
  14966. AllocInst(BeMCInstKind_MovSX, toValue, temp0);
  14967. AllocInst(BeMCInstKind_Add, toValue, toValue);
  14968. CreateLabel();
  14969. }
  14970. else
  14971. {
  14972. bool doSignExtension = (toType->IsIntable()) && (fromType->IsIntable()) && (toType->mSize > fromType->mSize) && (castedInst->mToSigned) && (castedInst->mValSigned);
  14973. if ((toType->IsFloat()) && (fromType->IsIntable()) && (castedInst->mValSigned))
  14974. doSignExtension = true;
  14975. if (mcValue.IsImmediate())
  14976. doSignExtension = false;
  14977. if (doSignExtension)
  14978. {
  14979. AllocInst(BeMCInstKind_MovSX, toValue, mcValue);
  14980. }
  14981. else
  14982. AllocInst(BeMCInstKind_Mov, toValue, mcValue);
  14983. }
  14984. result = toValue;
  14985. }
  14986. }
  14987. break;
  14988. case BeNegInst::TypeId:
  14989. {
  14990. auto castedInst = (BeNumericCastInst*)inst;
  14991. auto mcValue = GetOperand(castedInst->mValue);
  14992. result = AllocVirtualReg(GetType(mcValue));
  14993. CreateDefineVReg(result);
  14994. AllocInst(BeMCInstKind_Mov, result, mcValue);
  14995. AllocInst(BeMCInstKind_Neg, result);
  14996. }
  14997. break;
  14998. case BeNotInst::TypeId:
  14999. {
  15000. auto castedInst = (BeNumericCastInst*)inst;
  15001. auto mcValue = GetOperand(castedInst->mValue, true);
  15002. // Phi's are easy - just make a new one with the true and false branches swapped
  15003. // if (mcValue.mKind == BeMCOperandKind_Phi)
  15004. // {
  15005. // BeMCPhi* origPhi = mcValue.mPhi;
  15006. // BeMCPhi* newPhi = mPhiAlloc.Alloc();
  15007. //
  15008. // *newPhi = *origPhi;
  15009. // BF_SWAP(newPhi->mBrTrue, newPhi->mBrFalse);
  15010. // result.mKind = BeMCOperandKind_Phi;
  15011. // result.mPhi = newPhi;
  15012. // break;
  15013. // }
  15014. //
  15015. // if (mcValue.mKind == BeMCOperandKind_CmpResult)
  15016. // {
  15017. // auto origCmpResult = mCmpResults[mcValue.mCmpResultIdx];
  15018. //
  15019. // auto cmpResultIdx = (int)mCmpResults.size();
  15020. // BeCmpResult cmpResult;
  15021. // cmpResult.mCmpKind = BeModule::InvertCmp(origCmpResult.mCmpKind);
  15022. // mCmpResults.push_back(cmpResult);
  15023. // result.mKind = BeMCOperandKind_CmpResult;
  15024. // result.mCmpResultIdx = cmpResultIdx;
  15025. // break;
  15026. // }
  15027. if (mcValue.mKind == BeMCOperandKind_NotResult)
  15028. {
  15029. // Double negative! Just unwrap the NotResult.
  15030. result = GetOperand(mcValue.mNotResult->mValue, true);
  15031. break;
  15032. }
  15033. else if ((mcValue.mKind == BeMCOperandKind_Phi) || (mcValue.mKind == BeMCOperandKind_CmpResult))
  15034. {
  15035. auto notResult = mAlloc.Alloc<BeNotResult>();
  15036. notResult->mValue = castedInst->mValue;
  15037. result.mKind = BeMCOperandKind_NotResult;
  15038. result.mNotResult = notResult;
  15039. break;
  15040. }
  15041. // LLVM does a weird thing for Not: val = (val ^ 0xFF) & 1
  15042. // Which turns a '2' into a '1' which is True to True - non-conformant to C standard?
  15043. // Visual Studio does an actual conditional branch. For Beef, bools are defined as 1 or 0.
  15044. /*result = AllocVirtualReg(GetType(mcValue));
  15045. CreateDefineVReg(result);
  15046. AllocInst(BeMCInstKind_Mov, result, mcValue);
  15047. BeMCOperand xorVal;
  15048. xorVal.mKind = BeMCOperandKind_Immediate_i8;
  15049. xorVal.mImmediate = 0xFF;
  15050. AllocInst(BeMCInstKind_Xor, result, xorVal);
  15051. BeMCOperand andVal;
  15052. andVal.mKind = BeMCOperandKind_Immediate_i8;
  15053. andVal.mImmediate = 0x1;
  15054. AllocInst(BeMCInstKind_And, result, andVal);*/
  15055. auto type = castedInst->mValue->GetType();
  15056. result = AllocVirtualReg(GetType(mcValue));
  15057. CreateDefineVReg(result);
  15058. AllocInst(BeMCInstKind_Mov, result, mcValue);
  15059. if (type->mTypeCode == BeTypeCode_Boolean)
  15060. {
  15061. BeMCOperand xorVal;
  15062. xorVal.mKind = BeMCOperandKind_Immediate_i8;
  15063. xorVal.mImmediate = 0x1;
  15064. AllocInst(BeMCInstKind_Xor, result, xorVal);
  15065. }
  15066. else
  15067. {
  15068. AllocInst(BeMCInstKind_Not, result);
  15069. }
  15070. }
  15071. break;
  15072. case BeBinaryOpInst::TypeId:
  15073. {
  15074. auto castedInst = (BeBinaryOpInst*)inst;
  15075. auto mcLHS = GetOperand(castedInst->mLHS);
  15076. auto mcRHS = GetOperand(castedInst->mRHS);
  15077. if (castedInst->mOpKind == BeBinaryOpKind_Subtract)
  15078. {
  15079. if (((mcLHS.IsImmediateFloat()) && (mcLHS.GetImmediateDouble() == 0.0)) ||
  15080. ((mcLHS.IsImmediateInt()) && (mcLHS.mImmediate == 0)))
  15081. {
  15082. auto castedInst = (BeNumericCastInst*)inst;
  15083. result = AllocVirtualReg(GetType(mcRHS));
  15084. CreateDefineVReg(result);
  15085. AllocInst(BeMCInstKind_Mov, result, mcRHS);
  15086. AllocInst(BeMCInstKind_Neg, result);
  15087. break;
  15088. }
  15089. }
  15090. auto type = GetType(mcLHS);
  15091. switch (castedInst->mOpKind)
  15092. {
  15093. case BeBinaryOpKind_Add: result = AllocBinaryOp(BeMCInstKind_Add, mcLHS, mcRHS, BeMCBinIdentityKind_Any_IsZero,
  15094. ((castedInst->mOverflowCheckKind & BfOverflowCheckKind_Signed) != 0) ? BeMCOverflowCheckKind_O :
  15095. ((castedInst->mOverflowCheckKind & BfOverflowCheckKind_Unsigned) != 0) ? BeMCOverflowCheckKind_B : BeMCOverflowCheckKind_None);
  15096. break;
  15097. case BeBinaryOpKind_Subtract: result = AllocBinaryOp(BeMCInstKind_Sub, mcLHS, mcRHS, BeMCBinIdentityKind_Right_IsZero,
  15098. ((castedInst->mOverflowCheckKind & BfOverflowCheckKind_Signed) != 0) ? BeMCOverflowCheckKind_O :
  15099. ((castedInst->mOverflowCheckKind & BfOverflowCheckKind_Unsigned) != 0) ? BeMCOverflowCheckKind_B : BeMCOverflowCheckKind_None);
  15100. break;
  15101. case BeBinaryOpKind_Multiply: result = AllocBinaryOp(((castedInst->mOverflowCheckKind & BfOverflowCheckKind_Unsigned) != 0) ? BeMCInstKind_Mul : BeMCInstKind_IMul, mcLHS, mcRHS, BeMCBinIdentityKind_Any_IsOne,
  15102. ((castedInst->mOverflowCheckKind & BfOverflowCheckKind_Signed) != 0) ? BeMCOverflowCheckKind_O :
  15103. ((castedInst->mOverflowCheckKind & BfOverflowCheckKind_Unsigned) != 0) ? BeMCOverflowCheckKind_O : BeMCOverflowCheckKind_None);
  15104. break;
  15105. case BeBinaryOpKind_SDivide: result = AllocBinaryOp(BeMCInstKind_IDiv, mcLHS, mcRHS, BeMCBinIdentityKind_Right_IsOne); break;
  15106. case BeBinaryOpKind_UDivide: result = AllocBinaryOp(BeMCInstKind_Div, mcLHS, mcRHS, BeMCBinIdentityKind_Right_IsOne); break;
  15107. case BeBinaryOpKind_SModulus: result = AllocBinaryOp(BeMCInstKind_IRem, mcLHS, mcRHS, type->IsFloat() ? BeMCBinIdentityKind_None : BeMCBinIdentityKind_Right_IsOne_Result_Zero); break;
  15108. case BeBinaryOpKind_UModulus: result = AllocBinaryOp(BeMCInstKind_Rem, mcLHS, mcRHS, type->IsFloat() ? BeMCBinIdentityKind_None : BeMCBinIdentityKind_Right_IsOne_Result_Zero); break;
  15109. case BeBinaryOpKind_BitwiseAnd: result = AllocBinaryOp(BeMCInstKind_And, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15110. case BeBinaryOpKind_BitwiseOr: result = AllocBinaryOp(BeMCInstKind_Or, mcLHS, mcRHS, BeMCBinIdentityKind_Any_IsZero); break;
  15111. case BeBinaryOpKind_ExclusiveOr: result = AllocBinaryOp(BeMCInstKind_Xor, mcLHS, mcRHS, BeMCBinIdentityKind_Any_IsZero); break;
  15112. case BeBinaryOpKind_LeftShift: result = AllocBinaryOp(BeMCInstKind_Shl, mcLHS, mcRHS, BeMCBinIdentityKind_Any_IsZero); break;
  15113. case BeBinaryOpKind_RightShift: result = AllocBinaryOp(BeMCInstKind_Shr, mcLHS, mcRHS, BeMCBinIdentityKind_Any_IsZero); break;
  15114. case BeBinaryOpKind_ARightShift: result = AllocBinaryOp(BeMCInstKind_Sar, mcLHS, mcRHS, BeMCBinIdentityKind_Any_IsZero); break;
  15115. }
  15116. }
  15117. break;
  15118. case BeBitCastInst::TypeId:
  15119. {
  15120. auto castedInst = (BeBitCastInst*)inst;
  15121. auto mcValue = GetOperand(castedInst->mValue);
  15122. if (castedInst->mToType->IsInt())
  15123. {
  15124. BF_ASSERT(castedInst->mToType->mSize == 8);
  15125. }
  15126. else
  15127. BF_ASSERT(castedInst->mToType->IsPointer());
  15128. auto toType = castedInst->mToType;
  15129. if (mcValue.IsImmediate())
  15130. {
  15131. if (mcValue.mImmediate == 0)
  15132. {
  15133. BeMCOperand newImmediate;
  15134. newImmediate.mKind = BeMCOperandKind_Immediate_Null;
  15135. newImmediate.mType = toType;
  15136. result = newImmediate;
  15137. }
  15138. else
  15139. {
  15140. // Non-zero constant. Weird case, just do an actual MOV
  15141. result = AllocVirtualReg(toType);
  15142. CreateDefineVReg(result);
  15143. auto vregInfo = GetVRegInfo(result);
  15144. AllocInst(BeMCInstKind_Mov, result, mcValue);
  15145. if (mcValue.mKind == BeMCOperandKind_VRegAddr)
  15146. {
  15147. auto srcVRegInfo = GetVRegInfo(mcValue);
  15148. srcVRegInfo->mForceMem = true;
  15149. CheckForce(srcVRegInfo);
  15150. }
  15151. }
  15152. }
  15153. else
  15154. {
  15155. result = AllocVirtualReg(toType);
  15156. CreateDefineVReg(result);
  15157. auto vregInfo = GetVRegInfo(result);
  15158. vregInfo->mRelTo = mcValue;
  15159. vregInfo->mIsExpr = true;
  15160. if (mcValue.mKind == BeMCOperandKind_VRegAddr)
  15161. {
  15162. auto srcVRegInfo = GetVRegInfo(mcValue);
  15163. srcVRegInfo->mForceMem = true;
  15164. CheckForce(srcVRegInfo);
  15165. }
  15166. }
  15167. }
  15168. break;
  15169. case BeCmpInst::TypeId:
  15170. {
  15171. auto castedInst = (BeCmpInst*)inst;
  15172. auto mcLHS = GetOperand(castedInst->mLHS);
  15173. auto mcRHS = GetOperand(castedInst->mRHS);
  15174. auto valType = castedInst->mLHS->GetType();
  15175. auto mcInst = AllocInst(BeMCInstKind_Cmp, mcLHS, mcRHS);
  15176. auto cmpResultIdx = (int)mCmpResults.size();
  15177. BeCmpResult cmpResult;
  15178. cmpResult.mCmpKind = castedInst->mCmpKind;
  15179. if (valType->IsFloat())
  15180. {
  15181. // These operations are set up to properly handle NaN comparisons
  15182. switch (cmpResult.mCmpKind)
  15183. {
  15184. case BeCmpKind_SLT:
  15185. cmpResult.mCmpKind = BeCmpKind_OLE;
  15186. cmpResult.mInverted = true;
  15187. BF_SWAP(mcInst->mArg0, mcInst->mArg1);
  15188. break;
  15189. case BeCmpKind_SLE:
  15190. cmpResult.mCmpKind = BeCmpKind_OLT;
  15191. cmpResult.mInverted = true;
  15192. BF_SWAP(mcInst->mArg0, mcInst->mArg1);
  15193. break;
  15194. case BeCmpKind_SGT:
  15195. cmpResult.mCmpKind = BeCmpKind_OGT;
  15196. break;
  15197. case BeCmpKind_SGE:
  15198. cmpResult.mCmpKind = BeCmpKind_OGE;
  15199. break;
  15200. case BeCmpKind_EQ:
  15201. cmpResult.mCmpKind = BeCmpKind_UNE;
  15202. cmpResult.mInverted = true;
  15203. break;
  15204. case BeCmpKind_NE:
  15205. cmpResult.mCmpKind = BeCmpKind_UNE;
  15206. break;
  15207. }
  15208. }
  15209. mCmpResults.push_back(cmpResult);
  15210. result.mKind = BeMCOperandKind_CmpResult;
  15211. result.mCmpResultIdx = cmpResultIdx;
  15212. mcInst->mResult = result;
  15213. break;
  15214. }
  15215. break;
  15216. case BeObjectAccessCheckInst::TypeId:
  15217. {
  15218. auto castedInst = (BeObjectAccessCheckInst*)inst;
  15219. auto mcValue = GetOperand(castedInst->mValue);
  15220. auto int8Type = mModule->mContext->GetPrimitiveType(BeTypeCode_Int8);
  15221. auto int8PtrType = mModule->mContext->GetPointerTo(int8Type);
  15222. auto int8PtrVal = AllocVirtualReg(int8PtrType);
  15223. CreateDefineVReg(int8PtrVal);
  15224. auto vregInfo = GetVRegInfo(int8PtrVal);
  15225. vregInfo->mRelTo = mcValue;
  15226. vregInfo->mIsExpr = true;
  15227. int labelNum = mCurLabelIdx++;
  15228. BeMCOperand mcImm;
  15229. mcImm.mKind = BeMCOperandKind_Immediate_i8;
  15230. mcImm.mImmediate = -0x80;
  15231. BeMCOperand int8Val;
  15232. int8Val.mKind = BeMCOperandKind_VRegLoad;
  15233. int8Val.mVRegIdx = int8PtrVal.mVRegIdx;
  15234. AllocInst(BeMCInstKind_Cmp, int8Val, mcImm);
  15235. AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromLabel(labelNum), BeMCOperand::FromCmpKind(BeCmpKind_ULT));
  15236. AllocInst(BeMCInstKind_DbgBreak);
  15237. AllocInst(BeMCInstKind_Label, BeMCOperand::FromLabel(labelNum));
  15238. }
  15239. break;
  15240. case BeAllocaInst::TypeId:
  15241. {
  15242. //int homeSize = BF_ALIGN(BF_MAX(mMaxCallParamCount, 4) * 8, 16);
  15243. auto castedInst = (BeAllocaInst*)inst;
  15244. auto mcSize = BeMCOperand::FromImmediate(castedInst->mType->mSize);
  15245. bool isAligned16 = false;
  15246. int align = castedInst->mAlign;
  15247. BeType* allocType = castedInst->mType;
  15248. bool preservedVolatiles = false;
  15249. bool doPtrCast = false;
  15250. if (castedInst->mArraySize != NULL)
  15251. {
  15252. mcSize = BeMCOperand::FromImmediate(castedInst->mType->GetStride());
  15253. auto mcArraySize = GetOperand(castedInst->mArraySize);
  15254. if (mcArraySize.IsImmediate())
  15255. {
  15256. mcSize.mImmediate = mcSize.mImmediate * mcArraySize.mImmediate;
  15257. allocType = mModule->mContext->CreateSizedArrayType(castedInst->mType, mcArraySize.mImmediate);
  15258. doPtrCast = true;
  15259. }
  15260. else
  15261. {
  15262. preservedVolatiles = true;
  15263. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX));
  15264. inHeadAlloca = false;
  15265. if (mcSize.mImmediate == 1)
  15266. {
  15267. mcSize = mcArraySize;
  15268. }
  15269. else
  15270. {
  15271. auto mcInst = AllocInst(BeMCInstKind_IMul, mcArraySize, mcSize);
  15272. mcInst->mResult = BeMCOperand::FromReg(X64Reg_RAX);
  15273. mcSize = mcInst->mResult;
  15274. }
  15275. }
  15276. }
  15277. // The stack is 16-byte aligned on entry - we have to manually adjust for any alignment greater than that
  15278. if ((inHeadAlloca) && (align <= 16))
  15279. {
  15280. result = AllocVirtualReg(allocType);
  15281. auto vregInfo = mVRegInfo[result.mVRegIdx];
  15282. vregInfo->mAlign = castedInst->mAlign;
  15283. vregInfo->mHasDynLife = true;
  15284. if (castedInst->mForceMem)
  15285. vregInfo->mForceMem = true;
  15286. if (allocType->IsNonVectorComposite())
  15287. vregInfo->mForceMem = true;
  15288. result.mKind = BeMCOperandKind_VRegAddr;
  15289. if (doPtrCast)
  15290. {
  15291. BF_ASSERT(allocType->IsSizedArray());
  15292. auto resultType = mModule->mContext->GetPointerTo(castedInst->mType);
  15293. auto ptrResult = AllocVirtualReg(resultType);
  15294. auto vregInfo = mVRegInfo[ptrResult.mVRegIdx];
  15295. vregInfo->mIsExpr = true;
  15296. vregInfo->mRelTo = result;
  15297. vregInfo->mType = resultType;
  15298. vregInfo->mAlign = resultType->mSize;
  15299. CreateDefineVReg(ptrResult);
  15300. result = ptrResult;
  15301. }
  15302. }
  15303. else
  15304. {
  15305. bool needsChkStk = !castedInst->mNoChkStk;
  15306. bool doFastChkStk = false;
  15307. if (instIdx < (int)beBlock->mInstructions.size() - 1)
  15308. {
  15309. if (auto memSetInstr = BeValueDynCast<BeMemSetInst>(beBlock->mInstructions[instIdx + 1]))
  15310. {
  15311. if (memSetInstr->mAddr == inst)
  15312. {
  15313. // If we're clearing out this memory immediately after allocation then we don't
  15314. // need to do stack probing - the memset will ensure stack pages are committed
  15315. needsChkStk = false;
  15316. if (mcSize.IsImmediate())
  15317. {
  15318. if (auto sizeConst = BeValueDynCast<BeConstant>(memSetInstr->mSize))
  15319. {
  15320. if (sizeConst->mInt64 < mcSize.mImmediate)
  15321. {
  15322. // We haven't actually cleared out everything so we still need to chkStk
  15323. needsChkStk = true;
  15324. }
  15325. }
  15326. }
  15327. }
  15328. }
  15329. }
  15330. int stackAlign = BF_MAX(align, 16);
  15331. BeMCOperand mcFunc;
  15332. mcFunc.mKind = BeMCOperandKind_SymbolAddr;
  15333. mcFunc.mSymbolIdx = mCOFFObject->GetSymbolRef("__chkstk")->mIdx;
  15334. if (mcSize.IsImmediate())
  15335. {
  15336. // Align to 16 bytes
  15337. mcSize.mImmediate = (mcSize.mImmediate + 0xF) & ~0xF;
  15338. }
  15339. if ((mcSize.IsImmediate()) && (!preservedVolatiles) && (!needsChkStk))
  15340. {
  15341. AllocInst(BeMCInstKind_Sub, BeMCOperand::FromReg(X64Reg_RSP), mcSize);
  15342. }
  15343. else
  15344. {
  15345. if (needsChkStk)
  15346. {
  15347. if ((mcSize.IsImmediate()) && (mcSize.mImmediate < 4096))
  15348. {
  15349. // We can do a fast __chkstk in this case since we have a max of one page to handle
  15350. doFastChkStk = true;
  15351. }
  15352. }
  15353. if (doFastChkStk)
  15354. {
  15355. AllocInst(BeMCInstKind_Sub, BeMCOperand::FromReg(X64Reg_RSP), mcSize);
  15356. AllocInst(BeMCInstKind_FastCheckStack, BeMCOperand::FromReg(X64Reg_RSP));
  15357. }
  15358. else
  15359. {
  15360. if (!preservedVolatiles)
  15361. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX));
  15362. if (mcSize.IsImmediate())
  15363. {
  15364. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RAX), mcSize);
  15365. // It's tempting to not do a chkstk when we do an alloc less than 4k, but this isn't valid
  15366. // because we could break the system by doing three 2k allocs and access the third one first
  15367. // and BOOM. We rely on the front-end to tell us when we can omit it.
  15368. }
  15369. else if (!isAligned16)
  15370. {
  15371. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RAX), mcSize);
  15372. AllocInst(BeMCInstKind_Add, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand::FromImmediate(0xF));
  15373. AllocInst(BeMCInstKind_And, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand::FromImmediate(~0xF));
  15374. }
  15375. if ((needsChkStk) && (!doFastChkStk))
  15376. {
  15377. AllocInst(BeMCInstKind_Call, mcFunc);
  15378. }
  15379. AllocInst(BeMCInstKind_Sub, BeMCOperand::FromReg(X64Reg_RSP), BeMCOperand::FromReg(X64Reg_RAX));
  15380. if (doFastChkStk)
  15381. {
  15382. AllocInst(BeMCInstKind_FastCheckStack, BeMCOperand::FromReg(X64Reg_RSP));
  15383. }
  15384. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX));
  15385. }
  15386. }
  15387. auto resultType = mModule->mContext->GetPointerTo(castedInst->mType);
  15388. auto intType = mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  15389. BeMCOperand ptrValue;
  15390. if (mUseBP)
  15391. {
  15392. ptrValue = AllocVirtualReg(intType);
  15393. auto vregInfo = mVRegInfo[ptrValue.mVRegIdx];
  15394. vregInfo->mIsExpr = true;
  15395. vregInfo->mRelTo = BeMCOperand::FromReg(X64Reg_RSP);
  15396. vregInfo->mRelOffset.mKind = BeMCOperandKind::BeMCOperandKind_Immediate_HomeSize;
  15397. CreateDefineVReg(ptrValue);
  15398. }
  15399. else
  15400. {
  15401. ptrValue = BeMCOperand::FromReg(X64Reg_RSP);
  15402. }
  15403. result = AllocVirtualReg(resultType);
  15404. auto vregInfo = mVRegInfo[result.mVRegIdx];
  15405. vregInfo->mHasDynLife = true;
  15406. CreateDefineVReg(result);
  15407. AllocInst(BeMCInstKind_Mov, result, ptrValue);
  15408. if (stackAlign > 16)
  15409. {
  15410. // We have to align after everything - note that we always have to keep the 'homeSize' space available from RSP for calls,
  15411. // so the ANDing for alignment must be done here
  15412. AllocInst(BeMCInstKind_And, result, BeMCOperand::FromImmediate(~(stackAlign - 1)));
  15413. AllocInst(BeMCInstKind_Sub, BeMCOperand::FromReg(X64Reg_RSP), BeMCOperand::FromImmediate(stackAlign - 16));
  15414. }
  15415. BF_ASSERT(mUseBP);
  15416. }
  15417. }
  15418. break;
  15419. case BeAliasValueInst::TypeId:
  15420. {
  15421. auto castedInst = (BeAliasValueInst*)inst;
  15422. auto mcPtr = GetOperand(castedInst->mPtr, false, true);
  15423. result = AllocVirtualReg(GetType(mcPtr));
  15424. auto vregInfo = mVRegInfo[result.mVRegIdx];
  15425. vregInfo->mIsExpr = true;
  15426. vregInfo->mRelTo = mcPtr;
  15427. vregInfo->mHasDynLife = true;
  15428. CreateDefineVReg(result);
  15429. }
  15430. break;
  15431. case BeLifetimeStartInst::TypeId:
  15432. {
  15433. auto castedInst = (BeLifetimeEndInst*)inst;
  15434. auto mcPtr = GetOperand(castedInst->mPtr, false, true, true);
  15435. if (mcPtr)
  15436. {
  15437. auto vregInfo = GetVRegInfo(mcPtr);
  15438. if ((vregInfo != NULL) && (vregInfo->mHasDynLife))
  15439. {
  15440. // This alloca had an assignment (ie: `mov vregX, [RBP+homeSize0]`) so it must be defined at the mov
  15441. // This may indicate incorrectly generated code where we thought an alloca would be in the head but it isn't
  15442. }
  15443. else
  15444. AllocInst(BeMCInstKind_LifetimeStart, mcPtr);
  15445. }
  15446. }
  15447. break;
  15448. case BeLifetimeExtendInst::TypeId:
  15449. {
  15450. auto castedInst = (BeLifetimeEndInst*)inst;
  15451. auto mcPtr = GetOperand(castedInst->mPtr, false, true, true);
  15452. if (mcPtr)
  15453. AllocInst(BeMCInstKind_LifetimeExtend, mcPtr);
  15454. }
  15455. break;
  15456. case BeLifetimeEndInst::TypeId:
  15457. {
  15458. auto castedInst = (BeLifetimeEndInst*)inst;
  15459. auto mcPtr = GetOperand(castedInst->mPtr, false, true, true);
  15460. if (mcPtr.IsVRegAny())
  15461. {
  15462. AllocInst(BeMCInstKind_LifetimeEnd, mcPtr);
  15463. }
  15464. }
  15465. break;
  15466. case BeLifetimeSoftEndInst::TypeId:
  15467. {
  15468. auto castedInst = (BeLifetimeSoftEndInst*)inst;
  15469. auto mcPtr = GetOperand(castedInst->mPtr, false, true, true);
  15470. if (mcPtr.IsVRegAny())
  15471. {
  15472. AllocInst(BeMCInstKind_LifetimeSoftEnd, mcPtr);
  15473. }
  15474. }
  15475. break;
  15476. case BeValueScopeStartInst::TypeId:
  15477. {
  15478. result = BeMCOperand::FromImmediate((int)mVRegInfo.size());
  15479. }
  15480. break;
  15481. case BeValueScopeRetainInst::TypeId:
  15482. {
  15483. auto castedInst = (BeValueScopeRetainInst*)inst;
  15484. auto mcValue = GetOperand(castedInst->mValue, false, true);
  15485. auto vregInfo = GetVRegInfo(mcValue);
  15486. if (vregInfo != NULL)
  15487. vregInfo->mValueScopeRetainedKind = BeMCValueScopeRetainKind_Soft;
  15488. }
  15489. break;
  15490. case BeValueScopeEndInst::TypeId:
  15491. {
  15492. auto castedInst = (BeValueScopeEndInst*)inst;
  15493. BeMCOperand mcScopeStart = GetOperand(castedInst->mScopeStart, false, true);
  15494. // There are some recordering cases where we have a ValueScopeStart moved after a ValueScopeEnd
  15495. // Just ignore those. This is just an optimization anyway.
  15496. if (mcScopeStart)
  15497. AllocInst(castedInst->mIsSoft ? BeMCInstKind_ValueScopeSoftEnd : BeMCInstKind_ValueScopeHardEnd, mcScopeStart, BeMCOperand::FromImmediate((int)mVRegInfo.size()));
  15498. }
  15499. break;
  15500. case BeLifetimeFenceInst::TypeId:
  15501. {
  15502. auto castedInst = (BeLifetimeFenceInst*)inst;
  15503. auto mcPtr = GetOperand(castedInst->mPtr, false);
  15504. auto fenceBlock = GetOperand(castedInst->mFenceBlock);
  15505. auto vregInfo = GetVRegInfo(mcPtr);
  15506. vregInfo->mChainLifetimeEnd = true;
  15507. SetAndRestoreValue<BeMCBlock*> prevBlock(mActiveBlock, fenceBlock.mBlock);
  15508. auto lifetimeStart = AllocInst(BeMCInstKind_LifetimeStart, mcPtr);
  15509. lifetimeStart->mDbgLoc = NULL;
  15510. }
  15511. break;
  15512. case BeLoadInst::TypeId:
  15513. {
  15514. auto castedInst = (BeLoadInst*)inst;
  15515. auto mcTarget = GetOperand(castedInst->mTarget, false, false, true);
  15516. result = CreateLoad(mcTarget);
  15517. }
  15518. break;
  15519. case BeStoreInst::TypeId:
  15520. {
  15521. auto castedInst = (BeStoreInst*)inst;
  15522. auto mcVal = GetOperand(castedInst->mVal);
  15523. auto mcPtr = GetOperand(castedInst->mPtr, false, false, true);
  15524. bool handled = false;
  15525. CreateStore(BeMCInstKind_Mov, mcVal, mcPtr);
  15526. }
  15527. break;
  15528. case BeSetCanMergeInst::TypeId:
  15529. {
  15530. auto castedInst = (BeSetCanMergeInst*)inst;
  15531. auto mcVal = GetOperand(castedInst->mVal, false, false, true);
  15532. auto vregInfo = GetVRegInfo(mcVal);
  15533. vregInfo->mForceMerge = true;
  15534. }
  15535. break;
  15536. case BeMemSetInst::TypeId:
  15537. {
  15538. auto castedInst = (BeMemSetInst*)inst;
  15539. if (auto constVal = BeValueDynCast<BeConstant>(castedInst->mVal))
  15540. {
  15541. if (auto constSize = BeValueDynCast<BeConstant>(castedInst->mSize))
  15542. {
  15543. CreateMemSet(GetOperand(castedInst->mAddr), constVal->mUInt8, constSize->mInt64, castedInst->mAlignment);
  15544. break;
  15545. }
  15546. }
  15547. SizedArray<BeValue*, 3> args = { castedInst->mAddr, castedInst->mVal, castedInst->mSize };
  15548. auto mcFunc = BeMCOperand::FromSymbolAddr(mCOFFObject->GetSymbolRef("memset")->mIdx);
  15549. CreateCall(mcFunc, args);
  15550. }
  15551. break;
  15552. case BeFenceInst::TypeId:
  15553. {
  15554. AllocInst(BeMCInstKind_MFence);
  15555. }
  15556. break;
  15557. case BeStackSaveInst::TypeId:
  15558. {
  15559. auto stackVReg = BeMCOperand::FromVReg(stackSaveVRegs.back());
  15560. stackSaveVRegs.pop_back();
  15561. CreateDefineVReg(stackVReg);
  15562. AllocInst(BeMCInstKind_Mov, stackVReg, BeMCOperand::FromReg(X64Reg_RSP));
  15563. result = stackVReg;
  15564. }
  15565. break;
  15566. case BeStackRestoreInst::TypeId:
  15567. {
  15568. auto castedInst = (BeStackRestoreInst*)inst;
  15569. auto mcStackVal = GetOperand(castedInst->mStackVal);
  15570. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RSP), mcStackVal);
  15571. }
  15572. break;
  15573. case BeGEPInst::TypeId:
  15574. {
  15575. auto castedInst = (BeGEPInst*)inst;
  15576. auto mcVal = GetOperand(castedInst->mPtr);
  15577. auto mcIdx0 = GetOperand(castedInst->mIdx0);
  15578. BePointerType* ptrType = (BePointerType*)GetType(mcVal);
  15579. BF_ASSERT(ptrType->mTypeCode == BeTypeCode_Pointer);
  15580. result = mcVal;
  15581. if (castedInst->mIdx1 != NULL)
  15582. {
  15583. // We assume we never do both an idx0 and idx1 at once. Fix if we change that.
  15584. BF_ASSERT(castedInst->mIdx0);
  15585. auto mcIdx1 = GetOperand(castedInst->mIdx1);
  15586. if (!mcIdx1.IsImmediate())
  15587. {
  15588. // This path is used when we have a const array that gets indexed by a non-const index value
  15589. if (ptrType->mElementType->mTypeCode == BeTypeCode_SizedArray)
  15590. {
  15591. auto arrayType = (BeSizedArrayType*)ptrType->mElementType;
  15592. auto elementPtrType = mModule->mContext->GetPointerTo(arrayType->mElementType);
  15593. auto ptrValue = AllocVirtualReg(elementPtrType);
  15594. auto ptrInfo = GetVRegInfo(ptrValue);
  15595. ptrInfo->mIsExpr = true;
  15596. ptrInfo->mRelTo = result;
  15597. CreateDefineVReg(ptrValue);
  15598. result = ptrValue;
  15599. BeMCOperand mcRelOffset;
  15600. int relScale = 1;
  15601. if (mcIdx1.IsImmediate())
  15602. {
  15603. mcRelOffset = BeMCOperand::FromImmediate(mcIdx1.mImmediate * arrayType->mElementType->GetStride());
  15604. }
  15605. else
  15606. {
  15607. mcRelOffset = mcIdx1;
  15608. relScale = arrayType->mElementType->GetStride();
  15609. }
  15610. result = AllocRelativeVirtualReg(elementPtrType, result, mcRelOffset, relScale);
  15611. // The def is primary to create a single 'master location' for the GEP vreg to become legalized before use
  15612. CreateDefineVReg(result);
  15613. //TODO: Always correct?
  15614. result.mKind = BeMCOperandKind_VReg;
  15615. }
  15616. else
  15617. SoftFail("Invalid GEP", inst->mDbgLoc);
  15618. }
  15619. else
  15620. {
  15621. BF_ASSERT(mcIdx1.IsImmediate());
  15622. int byteOffset = 0;
  15623. BeType* elementType = NULL;
  15624. if (ptrType->mElementType->mTypeCode == BeTypeCode_Struct)
  15625. {
  15626. BeStructType* structType = (BeStructType*)ptrType->mElementType;
  15627. auto& structMember = structType->mMembers[mcIdx1.mImmediate];
  15628. elementType = structMember.mType;
  15629. byteOffset = structMember.mByteOffset;
  15630. }
  15631. else if (ptrType->mElementType->mTypeCode == BeTypeCode_SizedArray)
  15632. {
  15633. auto arrayType = (BeSizedArrayType*)ptrType->mElementType;
  15634. elementType = arrayType->mElementType;
  15635. byteOffset = mcIdx1.mImmediate * elementType->GetStride();
  15636. }
  15637. else if (ptrType->mElementType->mTypeCode == BeTypeCode_Vector)
  15638. {
  15639. auto arrayType = (BeVectorType*)ptrType->mElementType;
  15640. elementType = arrayType->mElementType;
  15641. byteOffset = mcIdx1.mImmediate * elementType->GetStride();
  15642. }
  15643. else
  15644. {
  15645. Fail("Invalid gep target");
  15646. }
  15647. auto elementPtrType = mModule->mContext->GetPointerTo(elementType);
  15648. result = AllocRelativeVirtualReg(elementPtrType, result, GetImmediate(byteOffset), 1);
  15649. // The def is primary to create a single 'master location' for the GEP vreg to become legalized before use
  15650. CreateDefineVReg(result);
  15651. result.mKind = BeMCOperandKind_VReg;
  15652. }
  15653. }
  15654. else
  15655. {
  15656. // It's temping to do a (IsNonZero) precondition, but if we make a reference to a VReg that is NOT in Addr form,
  15657. // then this will encode that so we will know we need to do a Load on that value at the Def during legalization
  15658. BeMCOperand mcRelOffset;
  15659. int relScale = 1;
  15660. if (mcIdx0.IsImmediate())
  15661. {
  15662. mcRelOffset = BeMCOperand::FromImmediate(mcIdx0.mImmediate * ptrType->mElementType->GetStride());
  15663. }
  15664. else
  15665. {
  15666. mcRelOffset = mcIdx0;
  15667. relScale = ptrType->mElementType->GetStride();
  15668. }
  15669. result = AllocRelativeVirtualReg(ptrType, result, mcRelOffset, relScale);
  15670. // The def is primary to create a single 'master location' for the GEP vreg to become legalized before use
  15671. CreateDefineVReg(result);
  15672. //TODO: Always correct?
  15673. result.mKind = BeMCOperandKind_VReg;
  15674. }
  15675. }
  15676. break;
  15677. case BeBrInst::TypeId:
  15678. {
  15679. auto castedInst = (BeBrInst*)inst;
  15680. auto mcInst = AllocInst();
  15681. mcInst->mKind = BeMCInstKind_Br;
  15682. mcInst->mArg0 = GetOperand(castedInst->mTargetBlock);
  15683. mcInst->mArg0.mBlock->AddPred(mcBlock);
  15684. if (castedInst->mNoCollapse)
  15685. {
  15686. mcInst->mArg1.mKind = BeMCOperandKind_Immediate_i8;
  15687. mcInst->mArg1.mImmediate = 1;
  15688. }
  15689. else if (castedInst->mIsFake)
  15690. {
  15691. mcInst->mArg1.mKind = BeMCOperandKind_Immediate_i8;
  15692. mcInst->mArg1.mImmediate = 2;
  15693. mcBlock->mHasFakeBr = true;
  15694. }
  15695. }
  15696. break;
  15697. case BeCondBrInst::TypeId:
  15698. {
  15699. auto castedInst = (BeCondBrInst*)inst;
  15700. auto testVal = GetOperand(castedInst->mCond, true);
  15701. auto trueBlock = GetOperand(castedInst->mTrueBlock);
  15702. auto falseBlock = GetOperand(castedInst->mFalseBlock);
  15703. trueBlock.mBlock->AddPred(mcBlock);
  15704. falseBlock.mBlock->AddPred(mcBlock);
  15705. CreateCondBr(mcBlock, testVal, trueBlock, falseBlock);
  15706. }
  15707. break;
  15708. case BePhiInst::TypeId:
  15709. {
  15710. auto castedInst = (BePhiInst*)inst;
  15711. BeMCPhi* mcPhi = mPhiAlloc.Alloc();
  15712. mcPhi->mBlock = mcBlock;
  15713. mcPhi->mIdx = mCurPhiIdx++;
  15714. //if (mDebugging)
  15715. {
  15716. for (auto phiIncoming : castedInst->mIncoming)
  15717. {
  15718. auto blockFrom = GetOperand(phiIncoming->mBlock).mBlock;;
  15719. int insertIdx = blockFrom->mInstructions.size() - 1;
  15720. SetAndRestoreValue<BeMCBlock*> prevActiveBlock(mActiveBlock, blockFrom);
  15721. SetAndRestoreValue<int*> prevInsertIdxPtr(mInsertInstIdxRef, &insertIdx);
  15722. BeMCPhiValue phiVal;
  15723. phiVal.mBlockFrom = blockFrom;
  15724. phiVal.mValue = GetOperand(phiIncoming->mValue, true);
  15725. if (phiVal.mValue.mKind == BeMCOperandKind_VRegAddr)
  15726. {
  15727. auto vregInfo = GetVRegInfo(phiVal.mValue);
  15728. if (!vregInfo->mIsExpr)
  15729. {
  15730. vregInfo->mForceMem = true;
  15731. CheckForce(vregInfo);
  15732. }
  15733. }
  15734. mcPhi->mValues.push_back(phiVal);
  15735. }
  15736. }
  15737. /*else
  15738. {
  15739. for (auto phiIncoming : castedInst->mIncoming)
  15740. {
  15741. BeMCPhiValue phiVal;
  15742. phiVal.mBlockFrom = GetOperand(phiIncoming->mBlock).mBlock;
  15743. phiVal.mValue = GetOperand(phiIncoming->mValue, true);
  15744. mcPhi->mValues.push_back(phiVal);
  15745. }
  15746. }*/
  15747. result.mKind = BeMCOperandKind_Phi;
  15748. result.mPhi = mcPhi;
  15749. // DefPhi is important because when we convert a CondBr of a PHI, because we will need to create jumps to the correct
  15750. // location when we create it as a value (specifically in the bool case)
  15751. AllocInst(BeMCInstKind_DefPhi, result);
  15752. }
  15753. break;
  15754. case BeSwitchInst::TypeId:
  15755. {
  15756. auto castedInst = (BeSwitchInst*)inst;
  15757. std::stable_sort(castedInst->mCases.begin(), castedInst->mCases.end(), [&](const BeSwitchCase& lhs, const BeSwitchCase& rhs)
  15758. {
  15759. return lhs.mValue->mInt64 < rhs.mValue->mInt64;
  15760. });
  15761. int numVals = castedInst->mCases.size();
  15762. if (numVals > 0)
  15763. {
  15764. int64 loVal = castedInst->mCases.front().mValue->mInt64;
  15765. int64 hiVal = castedInst->mCases.back().mValue->mInt64;
  15766. uint64 avgSpacing = (uint64)(hiVal - loVal) / numVals;
  15767. // Only use a table if we have a lot of values and the values are 'tight' enough
  15768. if ((numVals > 6) && (avgSpacing <= 8))
  15769. CreateTableSwitchSection(castedInst, 0, castedInst->mCases.size());
  15770. else
  15771. CreateBinarySwitchSection(castedInst, 0, castedInst->mCases.size());
  15772. }
  15773. auto mcDefaultBlock = GetOperand(castedInst->mDefaultBlock);
  15774. AllocInst(BeMCInstKind_Br, mcDefaultBlock);
  15775. mcDefaultBlock.mBlock->AddPred(mActiveBlock);
  15776. }
  15777. break;
  15778. case BeRetInst::TypeId:
  15779. {
  15780. auto castedInst = (BeRetInst*)inst;
  15781. if (castedInst->mRetValue != NULL)
  15782. {
  15783. auto retVal = GetOperand(castedInst->mRetValue);
  15784. auto retType = GetType(retVal);
  15785. if (retType->IsNonVectorComposite())
  15786. {
  15787. BF_ASSERT(mCompositeRetVRegIdx != -1);
  15788. BF_ASSERT(retVal.IsVReg());
  15789. auto vregInfo = GetVRegInfo(retVal);
  15790. vregInfo->SetRetVal();
  15791. }
  15792. else if (retType->IsVector())
  15793. {
  15794. X64CPURegister reg = X64Reg_M128_XMM0;
  15795. auto movInst = AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(reg), retVal);
  15796. }
  15797. else
  15798. {
  15799. X64CPURegister reg = X64Reg_RAX;
  15800. if (retType->IsIntable())
  15801. {
  15802. if (retType->mSize == 4)
  15803. reg = X64Reg_EAX;
  15804. else if (retType->mSize == 2)
  15805. reg = X64Reg_AX;
  15806. else if (retType->mSize == 1)
  15807. reg = X64Reg_AL;
  15808. }
  15809. else if (retType->mTypeCode == BeTypeCode_Float)
  15810. reg = X64Reg_XMM0_f32;
  15811. else if (retType->mTypeCode == BeTypeCode_Double)
  15812. reg = X64Reg_XMM0_f64;
  15813. auto movInst = AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(reg), retVal);
  15814. }
  15815. }
  15816. if (mBeFunction->HasStructRet())
  15817. {
  15818. for (int vregIdx = 0; vregIdx < (int)mVRegInfo.size(); vregIdx++)
  15819. {
  15820. auto vregInfo = mVRegInfo[vregIdx];
  15821. if (vregInfo->mIsRetVal)
  15822. {
  15823. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(X64Reg_RAX), BeMCOperand::FromVReg(vregIdx));
  15824. break;
  15825. }
  15826. }
  15827. }
  15828. auto mcInst = AllocInst();
  15829. mcInst->mKind = BeMCInstKind_Ret;
  15830. retCount++;
  15831. }
  15832. break;
  15833. case BeCallInst::TypeId:
  15834. {
  15835. auto castedInst = (BeCallInst*)inst;
  15836. BeMCOperand mcFunc;
  15837. BeType* returnType = NULL;
  15838. int varArgStart = -1;
  15839. bool useAltArgs = false;
  15840. SizedArray<BeValue*, 6> args;
  15841. if (auto intrin = BeValueDynCast<BeIntrinsic>(castedInst->mFunc))
  15842. {
  15843. switch (intrin->mKind)
  15844. {
  15845. case BfIRIntrinsic_Add:
  15846. case BfIRIntrinsic_And:
  15847. case BfIRIntrinsic_Div:
  15848. case BfIRIntrinsic_Eq:
  15849. case BfIRIntrinsic_Gt:
  15850. case BfIRIntrinsic_GtE:
  15851. case BfIRIntrinsic_Lt:
  15852. case BfIRIntrinsic_LtE:
  15853. case BfIRIntrinsic_Mod:
  15854. case BfIRIntrinsic_Mul:
  15855. case BfIRIntrinsic_Neq:
  15856. case BfIRIntrinsic_Or:
  15857. case BfIRIntrinsic_SAR:
  15858. case BfIRIntrinsic_SHL:
  15859. case BfIRIntrinsic_SHR:
  15860. case BfIRIntrinsic_Sub:
  15861. case BfIRIntrinsic_Xor:
  15862. {
  15863. auto mcLHS = TryToVector(castedInst->mArgs[0].mValue);
  15864. BeMCOperand mcRHS;
  15865. if ((intrin->mKind == BfIRIntrinsic_SAR) ||
  15866. (intrin->mKind == BfIRIntrinsic_SHL) ||
  15867. (intrin->mKind == BfIRIntrinsic_SHR))
  15868. {
  15869. mcRHS = GetOperand(castedInst->mArgs[1].mValue);
  15870. if (!mcRHS.IsImmediateInt())
  15871. mcRHS = BeMCOperand();
  15872. }
  15873. if (!mcRHS)
  15874. mcRHS = TryToVector(castedInst->mArgs[1].mValue);
  15875. switch (intrin->mKind)
  15876. {
  15877. case BfIRIntrinsic_Add:
  15878. result = AllocBinaryOp(BeMCInstKind_Add, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15879. break;
  15880. case BfIRIntrinsic_And:
  15881. result = AllocBinaryOp(BeMCInstKind_And, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15882. break;
  15883. case BfIRIntrinsic_Mul:
  15884. result = AllocBinaryOp(BeMCInstKind_IMul, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15885. break;
  15886. case BfIRIntrinsic_Or:
  15887. result = AllocBinaryOp(BeMCInstKind_Or, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15888. break;
  15889. case BfIRIntrinsic_SAR:
  15890. result = AllocBinaryOp(BeMCInstKind_Sar, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15891. break;
  15892. case BfIRIntrinsic_SHL:
  15893. result = AllocBinaryOp(BeMCInstKind_Shl, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15894. break;
  15895. case BfIRIntrinsic_SHR:
  15896. result = AllocBinaryOp(BeMCInstKind_Shr, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15897. break;
  15898. case BfIRIntrinsic_Sub:
  15899. result = AllocBinaryOp(BeMCInstKind_Sub, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15900. break;
  15901. case BfIRIntrinsic_Xor:
  15902. result = AllocBinaryOp(BeMCInstKind_Xor, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15903. break;
  15904. default:
  15905. SoftFail("Unhandled intrinsic");
  15906. }
  15907. }
  15908. break;
  15909. // case BfIRIntrinsic_Cast:
  15910. // {
  15911. //
  15912. // }
  15913. // break;
  15914. case BfIRIntrinsic_Not:
  15915. {
  15916. auto mcLHS = TryToVector(castedInst->mArgs[0].mValue);
  15917. BeMCOperand mcRHS = BeMCOperand::FromImmediate(-1);
  15918. result = AllocBinaryOp(BeMCInstKind_Xor, mcLHS, mcRHS, BeMCBinIdentityKind_None); break;
  15919. }
  15920. break;
  15921. case BfIRIntrinsic_Abs:
  15922. {
  15923. auto mcValue = GetOperand(castedInst->mArgs[0].mValue);
  15924. auto mcType = GetType(mcValue);
  15925. BeMCOperand andValue;
  15926. if (mcType->mSize == 4)
  15927. {
  15928. andValue.mKind = BeMCOperandKind_Immediate_f32_Packed128;
  15929. andValue.mImmediate = 0x7FFFFFFF;
  15930. }
  15931. else
  15932. {
  15933. andValue.mKind = BeMCOperandKind_Immediate_f64_Packed128;
  15934. andValue.mImmediate = 0x7FFFFFFFFFFFFFFFLL;
  15935. }
  15936. result = AllocVirtualReg(GetType(mcValue));
  15937. CreateDefineVReg(result);
  15938. AllocInst(BeMCInstKind_Mov, result, mcValue);
  15939. AllocInst(BeMCInstKind_And, result, andValue);
  15940. }
  15941. break;
  15942. case BfIRIntrinsic_AtomicAdd:
  15943. case BfIRIntrinsic_AtomicSub:
  15944. {
  15945. auto mcPtr = GetOperand(castedInst->mArgs[0].mValue);
  15946. auto mcVal = GetOperand(castedInst->mArgs[1].mValue);
  15947. auto valType = GetType(mcVal);
  15948. if (!valType->IsFloat())
  15949. {
  15950. auto mcMemKind = GetOperand(castedInst->mArgs[2].mValue);
  15951. if (!mcMemKind.IsImmediateInt())
  15952. {
  15953. SoftFail("Non-constant success ordering on AtomicLoad", castedInst->mDbgLoc);
  15954. break;
  15955. }
  15956. BeMCOperand scratchReg = AllocVirtualReg(valType);
  15957. auto vregInfo = GetVRegInfo(scratchReg);
  15958. vregInfo->mMustExist = true;
  15959. CreateDefineVReg(scratchReg);
  15960. if ((intrin->mKind == BfIRIntrinsic_AtomicSub) && (mcVal.IsImmediate()))
  15961. {
  15962. BeMCOperand mcNeg = mcVal;
  15963. if (mcVal.mKind == BeMCOperandKind_Immediate_f32)
  15964. mcNeg.mImmF32 = -mcNeg.mImmF32;
  15965. else if (mcVal.mKind == BeMCOperandKind_Immediate_f64)
  15966. mcNeg.mImmF64 = -mcNeg.mImmF64;
  15967. else
  15968. mcNeg.mImmediate = -mcNeg.mImmediate;
  15969. AllocInst(BeMCInstKind_Mov, scratchReg, mcNeg);
  15970. }
  15971. else
  15972. {
  15973. AllocInst(BeMCInstKind_Mov, scratchReg, mcVal);
  15974. if (intrin->mKind == BfIRIntrinsic_AtomicSub)
  15975. AllocInst(BeMCInstKind_Neg, scratchReg);
  15976. }
  15977. CreateStore(BeMCInstKind_XAdd, scratchReg, mcPtr);
  15978. if ((mcMemKind.mImmediate & BfIRAtomicOrdering_ReturnModified) != 0)
  15979. {
  15980. if (intrin->mKind == BfIRIntrinsic_AtomicSub)
  15981. AllocInst(BeMCInstKind_Sub, scratchReg, mcVal);
  15982. else
  15983. AllocInst(BeMCInstKind_Add, scratchReg, mcVal);
  15984. }
  15985. result = scratchReg;
  15986. break;
  15987. }
  15988. }
  15989. // Fallthrough
  15990. case BfIRIntrinsic_AtomicAnd:
  15991. case BfIRIntrinsic_AtomicMax:
  15992. case BfIRIntrinsic_AtomicMin:
  15993. case BfIRIntrinsic_AtomicNAnd:
  15994. case BfIRIntrinsic_AtomicOr:
  15995. case BfIRIntrinsic_AtomicUMax:
  15996. case BfIRIntrinsic_AtomicUMin:
  15997. case BfIRIntrinsic_AtomicXor:
  15998. {
  15999. auto mcPtr = GetOperand(castedInst->mArgs[0].mValue);
  16000. auto mcVal = GetOperand(castedInst->mArgs[1].mValue);
  16001. auto mcMemKind = GetOperand(castedInst->mArgs[2].mValue);
  16002. if (!mcMemKind.IsImmediateInt())
  16003. {
  16004. SoftFail("Non-constant ordering on atomic instruction", castedInst->mDbgLoc);
  16005. break;
  16006. }
  16007. auto valType = GetType(mcVal);
  16008. auto origValType = valType;
  16009. bool isFloat = valType->IsFloat();
  16010. auto mcTarget = BeMCOperand::ToLoad(mcPtr);
  16011. auto useReg = ResizeRegister(X64Reg_RAX, valType->mSize);
  16012. auto origTarget = mcTarget;
  16013. if (isFloat)
  16014. {
  16015. if (valType->mSize == 4)
  16016. valType = mModule->mContext->GetPrimitiveType(BeTypeCode_Int32);
  16017. else
  16018. valType = mModule->mContext->GetPrimitiveType(BeTypeCode_Int64);
  16019. BeMCOperand castedTarget = AllocVirtualReg(valType);
  16020. auto vregInfo = GetVRegInfo(castedTarget);
  16021. vregInfo->mMustExist = true;
  16022. vregInfo->mType = valType;
  16023. vregInfo->mAlign = valType->mAlign;
  16024. vregInfo->mIsExpr = true;
  16025. vregInfo->mRelTo = mcTarget;
  16026. CreateDefineVReg(castedTarget);
  16027. mcTarget = castedTarget;
  16028. }
  16029. int labelIdx = CreateLabel();
  16030. BeMCOperand mcPrev = AllocVirtualReg(valType);
  16031. auto vregInfo = GetVRegInfo(mcPrev);
  16032. vregInfo->mMustExist = true;
  16033. CreateDefineVReg(mcPrev);
  16034. BeMCOperand mcNext = AllocVirtualReg(valType);
  16035. vregInfo = GetVRegInfo(mcNext);
  16036. vregInfo->mMustExist = true;
  16037. vregInfo->mForceMem = isFloat;
  16038. CreateDefineVReg(mcNext);
  16039. AllocInst(BeMCInstKind_Mov, mcPrev, mcTarget);
  16040. AllocInst(BeMCInstKind_Mov, mcNext, mcPrev);
  16041. if (isFloat)
  16042. {
  16043. BeMCOperand mcFNext = AllocVirtualReg(origValType);
  16044. vregInfo = GetVRegInfo(mcFNext);
  16045. vregInfo->mIsExpr = true;
  16046. vregInfo->mRelTo = mcNext;
  16047. CreateDefineVReg(mcFNext);
  16048. if (intrin->mKind == BfIRIntrinsic_AtomicAdd)
  16049. AllocInst(BeMCInstKind_Add, mcFNext, mcVal);
  16050. else
  16051. AllocInst(BeMCInstKind_Sub, mcFNext, mcVal);
  16052. }
  16053. else
  16054. {
  16055. switch (intrin->mKind)
  16056. {
  16057. case BfIRIntrinsic_AtomicAdd:
  16058. AllocInst(BeMCInstKind_Add, mcNext, mcVal);
  16059. break;
  16060. case BfIRIntrinsic_AtomicAnd:
  16061. AllocInst(BeMCInstKind_Or, mcNext, mcVal);
  16062. break;
  16063. case BfIRIntrinsic_AtomicMax:
  16064. case BfIRIntrinsic_AtomicMin:
  16065. case BfIRIntrinsic_AtomicUMax:
  16066. case BfIRIntrinsic_AtomicUMin:
  16067. {
  16068. int cmpLabelIdx = mCurLabelIdx++;
  16069. AllocInst(BeMCInstKind_Cmp, mcNext, mcVal);
  16070. BeCmpKind cmpKind = BeCmpKind_None;
  16071. switch (intrin->mKind)
  16072. {
  16073. case BfIRIntrinsic_AtomicMax:
  16074. cmpKind = BeCmpKind_SGE;
  16075. break;
  16076. case BfIRIntrinsic_AtomicMin:
  16077. cmpKind = BeCmpKind_SLE;
  16078. break;
  16079. case BfIRIntrinsic_AtomicUMax:
  16080. cmpKind = BeCmpKind_UGE;
  16081. break;
  16082. case BfIRIntrinsic_AtomicUMin:
  16083. cmpKind = BeCmpKind_ULE;
  16084. break;
  16085. }
  16086. AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromLabel(cmpLabelIdx), BeMCOperand::FromCmpKind(cmpKind));
  16087. AllocInst(BeMCInstKind_Mov, mcNext, mcVal);
  16088. CreateLabel(-1, cmpLabelIdx);
  16089. }
  16090. break;
  16091. case BfIRIntrinsic_AtomicNAnd:
  16092. AllocInst(BeMCInstKind_And, mcNext, mcVal);
  16093. AllocInst(BeMCInstKind_Not, mcNext);
  16094. break;
  16095. case BfIRIntrinsic_AtomicOr:
  16096. AllocInst(BeMCInstKind_Or, mcNext, mcVal);
  16097. break;
  16098. case BfIRIntrinsic_AtomicSub:
  16099. AllocInst(BeMCInstKind_Sub, mcNext, mcVal);
  16100. break;
  16101. case BfIRIntrinsic_AtomicXor:
  16102. AllocInst(BeMCInstKind_Xor, mcNext, mcVal);
  16103. break;
  16104. }
  16105. }
  16106. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX));
  16107. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(useReg), mcPrev);
  16108. auto cmpXChgInst = AllocInst(BeMCInstKind_CmpXChg, mcTarget, mcNext);
  16109. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX));
  16110. AllocInst(BeMCInstKind_CondBr, BeMCOperand::FromLabel(labelIdx), BeMCOperand::FromCmpKind(BeCmpKind_NE));
  16111. if ((mcMemKind.mImmediate & BfIRAtomicOrdering_ReturnModified) != 0)
  16112. result = mcNext;
  16113. else
  16114. result = mcPrev;
  16115. }
  16116. break;
  16117. case BfIRIntrinsic_AtomicCmpStore:
  16118. case BfIRIntrinsic_AtomicCmpStore_Weak:
  16119. case BfIRIntrinsic_AtomicCmpXChg:
  16120. {
  16121. auto mcPtr = GetOperand(castedInst->mArgs[0].mValue);
  16122. auto mcComparand = GetOperand(castedInst->mArgs[1].mValue);
  16123. auto mcVal = GetOperand(castedInst->mArgs[2].mValue);
  16124. auto mcMemKind = GetOperand(castedInst->mArgs[3].mValue);
  16125. if (!mcMemKind.IsImmediate())
  16126. {
  16127. SoftFail("Non-constant success ordering on atomic operation", castedInst->mDbgLoc);
  16128. break;
  16129. }
  16130. auto valType = GetType(mcVal);
  16131. auto useReg = ResizeRegister(X64Reg_RAX, valType->mSize);
  16132. BeMCOperand scratchReg = AllocVirtualReg(GetType(mcVal));
  16133. auto vregInfo = GetVRegInfo(scratchReg);
  16134. vregInfo->mMustExist = true;
  16135. CreateDefineVReg(scratchReg);
  16136. AllocInst(BeMCInstKind_Mov, scratchReg, mcVal);
  16137. AllocInst(BeMCInstKind_PreserveVolatiles, BeMCOperand::FromReg(X64Reg_RAX));
  16138. AllocInst(BeMCInstKind_Mov, BeMCOperand::FromReg(useReg), mcComparand);
  16139. auto cmpXChgInst = AllocInst(BeMCInstKind_CmpXChg, BeMCOperand::ToLoad(mcPtr), scratchReg);
  16140. if (intrin->mKind == BfIRIntrinsic_AtomicCmpXChg)
  16141. {
  16142. AllocInst(BeMCInstKind_Mov, scratchReg, BeMCOperand::FromReg(useReg));
  16143. result = scratchReg;
  16144. }
  16145. else
  16146. {
  16147. auto cmpResultIdx = (int)mCmpResults.size();
  16148. BeCmpResult cmpResult;
  16149. cmpResult.mCmpKind = BeCmpKind_EQ;
  16150. mCmpResults.push_back(cmpResult);
  16151. result.mKind = BeMCOperandKind_CmpResult;
  16152. result.mCmpResultIdx = cmpResultIdx;
  16153. cmpXChgInst->mResult = result;
  16154. }
  16155. AllocInst(BeMCInstKind_RestoreVolatiles, BeMCOperand::FromReg(X64Reg_RAX));
  16156. }
  16157. break;
  16158. case BfIRIntrinsic_AtomicLoad:
  16159. {
  16160. auto mcTarget = GetOperand(castedInst->mArgs[0].mValue);
  16161. result = CreateLoad(mcTarget);
  16162. }
  16163. break;
  16164. case BfIRIntrinsic_AtomicFence:
  16165. {
  16166. if (castedInst->mArgs.size() == 0)
  16167. {
  16168. // Compiler fence- do nothing.
  16169. break;
  16170. }
  16171. auto mcMemKind = GetOperand(castedInst->mArgs[0].mValue);
  16172. if (!mcMemKind.IsImmediateInt())
  16173. {
  16174. SoftFail("Non-constant success ordering on AtomicFence", castedInst->mDbgLoc);
  16175. break;
  16176. }
  16177. if (mcMemKind.mImmediate == BfIRAtomicOrdering_SeqCst)
  16178. AllocInst(BeMCInstKind_MFence);
  16179. }
  16180. break;
  16181. case BfIRIntrinsic_AtomicStore:
  16182. {
  16183. auto mcPtr = GetOperand(castedInst->mArgs[0].mValue);
  16184. auto mcVal = GetOperand(castedInst->mArgs[1].mValue);
  16185. auto mcMemKind = GetOperand(castedInst->mArgs[2].mValue);
  16186. if (!mcMemKind.IsImmediateInt())
  16187. {
  16188. SoftFail("Non-constant success ordering on AtomicLoad", castedInst->mDbgLoc);
  16189. break;
  16190. }
  16191. if (mcMemKind.mImmediate == BfIRAtomicOrdering_SeqCst)
  16192. {
  16193. BeMCOperand scratchReg = AllocVirtualReg(GetType(mcVal));
  16194. auto vregInfo = GetVRegInfo(scratchReg);
  16195. vregInfo->mMustExist = true;
  16196. CreateDefineVReg(scratchReg);
  16197. AllocInst(BeMCInstKind_Mov, scratchReg, mcVal);
  16198. CreateStore(BeMCInstKind_XChg, scratchReg, mcPtr);
  16199. }
  16200. else
  16201. {
  16202. CreateStore(BeMCInstKind_Mov, mcVal, mcPtr);
  16203. }
  16204. }
  16205. break;
  16206. case BfIRIntrinsic_AtomicXChg:
  16207. {
  16208. auto mcPtr = GetOperand(castedInst->mArgs[0].mValue);
  16209. auto mcVal = GetOperand(castedInst->mArgs[1].mValue);
  16210. auto mcMemKind = GetOperand(castedInst->mArgs[2].mValue);
  16211. if (!mcMemKind.IsImmediateInt())
  16212. {
  16213. SoftFail("Non-constant success ordering on AtomicXChg", castedInst->mDbgLoc);
  16214. break;
  16215. }
  16216. BeMCOperand scratchReg = AllocVirtualReg(GetType(mcVal));
  16217. auto vregInfo = GetVRegInfo(scratchReg);
  16218. vregInfo->mMustExist = true;
  16219. CreateDefineVReg(scratchReg);
  16220. AllocInst(BeMCInstKind_Mov, scratchReg, mcVal);
  16221. CreateStore(BeMCInstKind_XChg, scratchReg, mcPtr);
  16222. result = scratchReg;
  16223. }
  16224. break;
  16225. case BfIRIntrinsic_Cast:
  16226. {
  16227. result = AllocVirtualReg(intrin->mReturnType);
  16228. CreateDefineVReg(result);
  16229. auto vregInfo = GetVRegInfo(result);
  16230. vregInfo->mRelTo = TryToVector(castedInst->mArgs[0].mValue);
  16231. vregInfo->mIsExpr = true;
  16232. }
  16233. break;
  16234. case BfIRIntrinsic_DebugTrap:
  16235. {
  16236. AllocInst(BeMCInstKind_DbgBreak);
  16237. }
  16238. break;
  16239. case BfIRIntrinsic_Index:
  16240. {
  16241. auto valPtr = GetOperand(castedInst->mArgs[0].mValue);
  16242. auto idx = GetOperand(castedInst->mArgs[1].mValue);
  16243. auto valType = GetType(valPtr);
  16244. if (!valType->IsPointer())
  16245. {
  16246. SoftFail("Non-pointer index target", castedInst->mDbgLoc);
  16247. break;
  16248. }
  16249. valType = ((BePointerType*)valType)->mElementType;
  16250. if (!valType->IsVector())
  16251. {
  16252. SoftFail("Non-vector index target", castedInst->mDbgLoc);
  16253. break;
  16254. }
  16255. auto vectorType = (BeVectorType*)valType;
  16256. auto elementPtrType = mModule->mContext->GetPointerTo(vectorType->mElementType);
  16257. result = AllocVirtualReg(elementPtrType);
  16258. CreateDefineVReg(result);
  16259. auto vregInfo = GetVRegInfo(result);
  16260. vregInfo->mRelTo = valPtr;
  16261. vregInfo->mRelOffset = idx;
  16262. vregInfo->mRelOffsetScale = vectorType->mElementType->mSize;
  16263. vregInfo->mIsExpr = true;
  16264. result = CreateLoad(result);
  16265. }
  16266. break;
  16267. case BfIRIntrinsic_MemSet:
  16268. {
  16269. if (auto constVal = BeValueDynCast<BeConstant>(castedInst->mArgs[1].mValue))
  16270. {
  16271. if (auto constSize = BeValueDynCast<BeConstant>(castedInst->mArgs[2].mValue))
  16272. {
  16273. if (auto constAlign = BeValueDynCast<BeConstant>(castedInst->mArgs[3].mValue))
  16274. {
  16275. CreateMemSet(GetOperand(castedInst->mArgs[0].mValue), constVal->mUInt8, constSize->mInt64, constAlign->mInt32);
  16276. break;
  16277. }
  16278. }
  16279. }
  16280. mcFunc = BeMCOperand::FromSymbolAddr(mCOFFObject->GetSymbolRef("memset")->mIdx);
  16281. for (int i = 0; i < 3; i++)
  16282. args.Add(castedInst->mArgs[i].mValue);
  16283. useAltArgs = true;
  16284. }
  16285. break;
  16286. case BfIRIntrinsic_MemCpy:
  16287. {
  16288. if (auto constSize = BeValueDynCast<BeConstant>(castedInst->mArgs[2].mValue))
  16289. {
  16290. if (auto constAlign = BeValueDynCast<BeConstant>(castedInst->mArgs[3].mValue))
  16291. {
  16292. CreateMemCpy(GetOperand(castedInst->mArgs[0].mValue), GetOperand(castedInst->mArgs[1].mValue), constSize->mInt64, constAlign->mInt32);
  16293. break;
  16294. }
  16295. }
  16296. mcFunc = BeMCOperand::FromSymbolAddr(mCOFFObject->GetSymbolRef("memcpy")->mIdx);
  16297. //altArgs.insert(altArgs.begin(), castedInst->mArgs.begin(), castedInst->mArgs.begin() + 3);
  16298. for (int i = 0; i < 3; i++)
  16299. args.Add(castedInst->mArgs[i].mValue);
  16300. useAltArgs = true;
  16301. }
  16302. break;
  16303. case BfIRIntrinsic_MemMove:
  16304. {
  16305. mcFunc = BeMCOperand::FromSymbolAddr(mCOFFObject->GetSymbolRef("memmove")->mIdx);
  16306. //altArgs.insert(altArgs.begin(), castedInst->mArgs.begin(), castedInst->mArgs.begin() + 3);
  16307. for (int i = 0; i < 3; i++)
  16308. args.Add(castedInst->mArgs[i].mValue);
  16309. useAltArgs = true;
  16310. }
  16311. break;
  16312. case BfIRIntrinsic_ReturnAddress:
  16313. {
  16314. result = AllocVirtualReg(intrin->mReturnType);
  16315. CreateDefineVReg(result);
  16316. auto vregInfo = GetVRegInfo(result);
  16317. vregInfo->mFrameOffset = 0;
  16318. }
  16319. break;
  16320. case BfIRIntrinsic_VAArg:
  16321. {
  16322. auto mcListPtr = GetOperand(castedInst->mArgs[0].mValue);
  16323. auto mcDestVoidPtr = GetOperand(castedInst->mArgs[1].mValue);
  16324. auto mcType = GetOperand(castedInst->mArgs[2].mValue);
  16325. BeType* beType = mModule->mBeIRCodeGen->GetBeTypeById((int32)mcType.mImmediate);
  16326. auto mcList = AllocVirtualReg(mModule->mContext->GetPointerTo(mModule->mContext->GetPrimitiveType(BeTypeCode_NullPtr)));
  16327. CreateDefineVReg(mcList);
  16328. auto listVRegInfo = GetVRegInfo(mcList);
  16329. listVRegInfo->mRelTo = mcListPtr;
  16330. listVRegInfo->mIsExpr = true;
  16331. auto mcSrc = AllocVirtualReg(mModule->mContext->GetPointerTo(beType));
  16332. CreateDefineVReg(mcSrc);
  16333. auto srcVRegInfo = GetVRegInfo(mcSrc);
  16334. srcVRegInfo->mRelTo = BeMCOperand::ToLoad(mcList);
  16335. srcVRegInfo->mIsExpr = true;
  16336. auto mcDest = AllocVirtualReg(mModule->mContext->GetPointerTo(beType));
  16337. CreateDefineVReg(mcDest);
  16338. auto destVRegInfo = GetVRegInfo(mcDest);
  16339. destVRegInfo->mRelTo = mcDestVoidPtr;
  16340. destVRegInfo->mIsExpr = true;
  16341. AllocInst(BeMCInstKind_Mov, BeMCOperand::ToLoad(mcDest), BeMCOperand::ToLoad(mcSrc));
  16342. AllocInst(BeMCInstKind_Add, BeMCOperand::ToLoad(mcList), BeMCOperand::FromImmediate(8));
  16343. }
  16344. break;
  16345. case BfIRIntrinsic_VAEnd:
  16346. break;
  16347. case BfIRIntrinsic_VAStart:
  16348. {
  16349. auto mcTarget = GetOperand(castedInst->mArgs[0].mValue);
  16350. auto destVal = AllocVirtualReg(mModule->mContext->GetPointerTo(mModule->mContext->GetPrimitiveType(BeTypeCode_NullPtr)));
  16351. auto destVRegInfo = GetVRegInfo(destVal);
  16352. destVRegInfo->mRelTo = mcTarget;
  16353. destVRegInfo->mIsExpr = true;
  16354. CreateDefineVReg(destVal);
  16355. auto nullPtrType = mModule->mContext->GetPrimitiveType(BeTypeCode_NullPtr);
  16356. auto vaStartVal = AllocVirtualReg(nullPtrType);
  16357. auto vRegInfo = GetVRegInfo(vaStartVal);
  16358. vRegInfo->mMustExist = true;
  16359. vRegInfo->mForceMem = true;
  16360. vRegInfo->mFrameOffset = (int)mBeFunction->mParams.size() * 8 + 8;
  16361. vRegInfo->mRefCount++;
  16362. CreateDefineVReg(vaStartVal);
  16363. AllocInst(BeMCInstKind_Mov, BeMCOperand::ToLoad(destVal), BeMCOperand::FromVRegAddr(vaStartVal.mVRegIdx));
  16364. }
  16365. break;
  16366. default:
  16367. SoftFail(StrFormat("Intrinsic not handled: '%s'", intrin->mName.c_str()), castedInst->mDbgLoc);
  16368. break;
  16369. }
  16370. }
  16371. else
  16372. {
  16373. auto funcPtrType = castedInst->mFunc->GetType();
  16374. if (funcPtrType->IsPointer())
  16375. {
  16376. auto elementType = ((BePointerType*)funcPtrType)->mElementType;
  16377. if (elementType->mTypeCode == BeTypeCode_Function)
  16378. {
  16379. BeFunctionType* funcType = (BeFunctionType*)elementType;
  16380. if (funcType->mIsVarArg)
  16381. varArgStart = funcType->mParams.mSize;
  16382. }
  16383. }
  16384. returnType = castedInst->GetType();
  16385. mcFunc = GetOperand(castedInst->mFunc);
  16386. }
  16387. if (mcFunc)
  16388. {
  16389. if (!useAltArgs)
  16390. {
  16391. BF_ASSERT(args.IsEmpty());
  16392. for (auto& arg : castedInst->mArgs)
  16393. args.Add(arg.mValue);
  16394. }
  16395. result = CreateCall(mcFunc, args, returnType, castedInst->mCallingConv, castedInst->HasStructRet(), castedInst->mNoReturn, varArgStart);
  16396. }
  16397. }
  16398. break;
  16399. case BeDbgDeclareInst::TypeId:
  16400. {
  16401. auto castedInst = (BeDbgDeclareInst*)inst;
  16402. auto dbgVar = castedInst->mDbgVar;
  16403. auto mcValue = GetOperand(castedInst->mValue, false, false, true);
  16404. auto mcVReg = mcValue;
  16405. auto vregInfo = GetVRegInfo(mcVReg);
  16406. if ((vregInfo != NULL) && (vregInfo->mDbgVariable != NULL))
  16407. {
  16408. auto shadowVReg = AllocVirtualReg(vregInfo->mType);
  16409. CreateDefineVReg(shadowVReg);
  16410. auto shadowVRegInfo = GetVRegInfo(shadowVReg);
  16411. shadowVRegInfo->mIsExpr = true;
  16412. shadowVRegInfo->mValueScopeRetainedKind = BeMCValueScopeRetainKind_Hard;
  16413. shadowVRegInfo->mRelTo = mcVReg;
  16414. shadowVRegInfo->mRelTo.mKind = BeMCOperandKind_VReg;
  16415. shadowVReg.mKind = mcVReg.mKind;
  16416. mcValue = shadowVReg;
  16417. mcVReg = shadowVReg;
  16418. vregInfo = shadowVRegInfo;
  16419. mValueToOperand[dbgVar] = shadowVReg;
  16420. }
  16421. else
  16422. {
  16423. mValueToOperand[dbgVar] = mcVReg;
  16424. }
  16425. dbgVar->mValue = castedInst->mValue;
  16426. dbgVar->mDeclDbgLoc = inst->mDbgLoc;
  16427. dbgVar->mIsValue = castedInst->mIsValue;
  16428. if (!mcVReg.IsVRegAny())
  16429. {
  16430. mcVReg = AllocVirtualReg(mModule->mContext->GetPrimitiveType(BeTypeCode_None));
  16431. vregInfo = GetVRegInfo(mcVReg);
  16432. // Allocate a vreg just to properly track the lifetime of this local constant
  16433. /*auto constType = GetType(mcVReg);
  16434. auto newVReg = AllocVirtualReg(constType);
  16435. vregInfo = GetVRegInfo(newVReg);
  16436. vregInfo->mIsExpr = true;
  16437. vregInfo->mRelTo = mcVReg;
  16438. mcVReg = newVReg;*/
  16439. }
  16440. vregInfo->mDbgVariable = dbgVar;
  16441. vregInfo->mMustExist = true;
  16442. dbgVar->mScope = mCurDbgLoc->mDbgScope;
  16443. dbgVar->mDeclMCBlockId = mcBlock->mBlockIdx;
  16444. AllocInst(BeMCInstKind_DbgDecl, mcVReg);
  16445. }
  16446. break;
  16447. default:
  16448. Fail("Unhandled BeInst type");
  16449. break;
  16450. }
  16451. if (mFailed)
  16452. return;
  16453. if (result.mKind != BeMCOperandKind_None)
  16454. mValueToOperand[inst] = result;
  16455. }
  16456. inHeadAlloca = false;
  16457. isFirstBlock = false;
  16458. }
  16459. mCurDbgLoc = NULL;
  16460. BEMC_ASSERT(valueScopeStack.size() == 0);
  16461. BEMC_ASSERT(retCount == 1);
  16462. bool wantDebug = mDebugging;
  16463. wantDebug |= function->mName == "?GetVal@TestProgram@BeefTest@bf@@SATint@@M@Z";
  16464. //wantDebug |= function->mName == "?Testos@Fartso@@SAHPEA1@HH@Z";
  16465. //wantDebug |= function->mName == "?GetYoopA@Fartso@@QEAAUYoop@@XZ";
  16466. //"?TestVals@Fartso@@QEAATint@@XZ";
  16467. /*if (function->mName == "?TestVals@Fartso@@QEAATint@@PEA_J@Z")
  16468. {
  16469. //TODO: Temporary, force reg1 to spill
  16470. mVRegInfo[1]->mSpilled = true;
  16471. }*/
  16472. String str;
  16473. if (wantDebug)
  16474. {
  16475. str = "-------- Initial --------\n" + ToString();
  16476. OutputDebugStr(str.c_str());
  16477. } // Hi
  16478. DoTLSSetup();
  16479. DoChainedBlockMerge();
  16480. DoSplitLargeBlocks();
  16481. DetectLoops();
  16482. DoLastUsePass();
  16483. for (int pass = 0; pass < 3; pass++)
  16484. {
  16485. // Shouldn't have more than 2 passes
  16486. BF_ASSERT(pass != 2);
  16487. if (DoInitializedPass())
  16488. break;
  16489. }
  16490. if (wantDebug)
  16491. {
  16492. str = "-------- After DoDefPass --------\n" + ToString();
  16493. OutputDebugStr(str.c_str());
  16494. }
  16495. RefreshRefCounts();
  16496. GenerateLiveness();
  16497. if (wantDebug)
  16498. {
  16499. str = "--------After GenerateLiveness --------\n" + ToString();
  16500. OutputDebugStr(str.c_str());
  16501. }
  16502. DoInstCombinePass();
  16503. RefreshRefCounts();
  16504. if (wantDebug)
  16505. {
  16506. str = "--------After DoInstCombinePass--------\n" + ToString();
  16507. OutputDebugStr(str.c_str());
  16508. }
  16509. DoLoads();
  16510. for (int pass = 0; true; pass++)
  16511. {
  16512. DoRegAssignPass();
  16513. DoSanityChecking();
  16514. if (wantDebug)
  16515. {
  16516. str = "--------After RegAssignPass--------\n" + ToString();
  16517. OutputDebugStr(str.c_str());
  16518. }
  16519. bool regsDone = DoLegalization();
  16520. if (wantDebug)
  16521. {
  16522. str = "--------After Legalization--------\n" + ToString();
  16523. OutputDebugStr(str.c_str());
  16524. }
  16525. if (regsDone)
  16526. {
  16527. if (pass == 0)
  16528. {
  16529. if (wantDebug)
  16530. {
  16531. // The first pass may overestimate mem cost, so run another one
  16532. OutputDebugStr("Processing done on first pass, doing another register assign pass\n");
  16533. }
  16534. }
  16535. else
  16536. break;
  16537. }
  16538. //FOR TESTING
  16539. if (pass == 4)
  16540. {
  16541. NOP;
  16542. }
  16543. if (pass == 16)
  16544. {
  16545. Fail("Register assignment failed!");
  16546. }
  16547. // Reassign regs after legalization
  16548. //DoRegAssignPass();
  16549. }
  16550. DoRegFinalization();
  16551. if (wantDebug)
  16552. {
  16553. str = "--------After RegFinalization--------\n" + ToString();
  16554. OutputDebugStr(str);
  16555. }
  16556. DoBlockCombine();
  16557. if (wantDebug)
  16558. {
  16559. str = "--------After BlockCombine --------\n" + ToString();
  16560. OutputDebugStr(str);
  16561. }
  16562. while (true)
  16563. {
  16564. bool didWork = DoJumpRemovePass();
  16565. if (!didWork)
  16566. break;
  16567. RefreshRefCounts();
  16568. if (wantDebug)
  16569. {
  16570. str = "--------After DoJumpRemovePass --------\n" + ToString();
  16571. OutputDebugStr(str);
  16572. }
  16573. }
  16574. DoFrameObjPass(); // Must be (almost) last
  16575. if (wantDebug)
  16576. {
  16577. str = ToString();
  16578. OutputDebugStr(str.c_str());
  16579. }
  16580. DoCodeEmission();
  16581. }