BfIRCodeGen.cpp 193 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353
  1. #include "BfIRCodeGen.h"
  2. #include "BfModule.h"
  3. #include "BeefySysLib/util/BeefPerf.h"
  4. #include "BeefySysLib/util/Hash.h"
  5. #ifdef BF_PLATFORM_WINDOWS
  6. #include <io.h>
  7. #endif
  8. #pragma warning(push)
  9. #pragma warning(disable:4141)
  10. #pragma warning(disable:4146)
  11. #pragma warning(disable:4291)
  12. #pragma warning(disable:4244)
  13. #pragma warning(disable:4267)
  14. #pragma warning(disable:4624)
  15. #pragma warning(disable:4800)
  16. #pragma warning(disable:4996)
  17. #include "llvm/IR/Module.h"
  18. #include "llvm/IR/Constants.h"
  19. #include "llvm/IR/GlobalValue.h"
  20. #include "llvm/IR/GlobalVariable.h"
  21. #include "llvm/ADT/ArrayRef.h"
  22. #include "llvm/IR/InlineAsm.h"
  23. #include "llvm/IR/Attributes.h"
  24. #include "llvm/Support/FileSystem.h"
  25. #include "llvm/TargetParser/Host.h"
  26. //#include "llvm/Support/Dwarf.h"
  27. #include "llvm/IR/DIBuilder.h"
  28. //#include "llvm/ADT/Triple.h"
  29. //#include "llvm/CodeGen/CommandFlags.h"
  30. #include "llvm/CodeGen/LinkAllAsmWriterComponents.h"
  31. #include "llvm/CodeGen/LinkAllCodegenComponents.h"
  32. #include "llvm/IR/DataLayout.h"
  33. #include "llvm/IR/IRPrintingPasses.h"
  34. #include "llvm/IR/LLVMContext.h"
  35. #include "llvm/IR/Module.h"
  36. #include "llvm/IRReader/IRReader.h"
  37. //#include "llvm/MC/SubtargetFeature.h"
  38. #include "llvm/MC/MCObjectWriter.h"
  39. #include "llvm/Pass.h"
  40. //#include "llvm/Transforms/IPO/PassManagerBuilder.h"
  41. #include "llvm/Transforms/Utils.h"
  42. #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
  43. #include "llvm/IR/LegacyPassManager.h"
  44. #include "llvm/Support/CommandLine.h"
  45. #include "llvm/Support/Debug.h"
  46. #include "llvm/Support/FileSystem.h"
  47. #include "llvm/Support/FormattedStream.h"
  48. //#include "llvm/Support/Host.h"
  49. #include "llvm/Support/ManagedStatic.h"
  50. #include "llvm/Support/CodeGen.h"
  51. #include "llvm/Support/PluginLoader.h"
  52. #include "llvm/Support/PrettyStackTrace.h"
  53. #include "llvm/Support/Signals.h"
  54. #include "llvm/Support/SourceMgr.h"
  55. //#include "llvm/Support/TargetRegistry.h"
  56. #include "llvm/Support/TargetSelect.h"
  57. #include "llvm/Support/ToolOutputFile.h"
  58. //#include "llvm/Target/TargetLibraryInfo.h"
  59. #include "llvm/Target/TargetMachine.h"
  60. //#include "llvm/Target/TargetSubtargetInfo.h"
  61. //#include "llvm/Transforms/IPO/PassManagerBuilder.h"
  62. //#include "llvm-c/Transforms/PassManagerBuilder.h"
  63. #include "llvm/ADT/SmallVector.h"
  64. #include "llvm/Analysis/Passes.h"
  65. #include "llvm/IR/DataLayout.h"
  66. #include "llvm/IR/Verifier.h"
  67. #include "llvm/IR/LegacyPassManager.h"
  68. #include "llvm/IR/PassManager.h"
  69. #include "llvm/Support/CommandLine.h"
  70. #include "llvm/Support/ManagedStatic.h"
  71. #include "llvm/Analysis/BasicAliasAnalysis.h"
  72. //#include "llvm/Analysis/CFLAliasAnalysis.h"
  73. //#include "llvm/Analysis/CFLAndersAliasAnalysis.h"
  74. //#include "llvm/Analysis/CFLSteensAliasAnalysis.h"
  75. #include "llvm/Analysis/GlobalsModRef.h"
  76. #include "llvm/Analysis/ScopedNoAliasAA.h"
  77. #include "llvm/Analysis/TargetLibraryInfo.h"
  78. #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
  79. #include "llvm/Target/TargetMachine.h"
  80. #include "llvm/Transforms/IPO.h"
  81. #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
  82. #include "llvm/Transforms/IPO/FunctionAttrs.h"
  83. #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
  84. #include "llvm/Transforms/IPO/AlwaysInliner.h"
  85. #include "llvm/Transforms/Instrumentation.h"
  86. #include "llvm/Transforms/Scalar.h"
  87. #include "llvm/Transforms/Scalar/GVN.h"
  88. //#include "llvm/Transforms/Vectorize.h"
  89. #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
  90. #include "llvm/Transforms/InstCombine/InstCombine.h"
  91. #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
  92. #include "llvm/Passes/PassBuilder.h"
  93. //#include "llvm/Bitcode/ReaderWriter.h"
  94. #include "llvm/Analysis/Passes.h"
  95. #include "llvm/Transforms/IPO.h"
  96. #include "llvm/Transforms/Scalar.h"
  97. //#include "llvm/Transforms/Vectorize.h"
  98. #include "llvm/Pass.h"
  99. #include "llvm/CodeGen/MachineFunctionPass.h"
  100. #include "llvm/Support/raw_ostream.h"
  101. #include "llvm/MC/MCAsmBackend.h"
  102. #include "llvm/MC/MCCodeEmitter.h"
  103. #include "llvm/MC/TargetRegistry.h"
  104. #include "llvm/LTO/LTOBackend.h"
  105. #include "llvm/Bitcode/BitcodeWriter.h"
  106. #include "llvm/Bitcode/BitcodeReader.h"
  107. #include "llvm/Bitcode/BitcodeWriterPass.h"
  108. #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
  109. #include "llvm/Transforms/IPO/AlwaysInliner.h"
  110. #include "llvm/Transforms/IPO.h"
  111. #include "../LLVMUtils.h"
  112. #pragma warning(pop)
  113. void pm(llvm::Module* module);
  114. USING_NS_BF;
  115. #pragma warning(disable:4146)
  116. #pragma warning(disable:4996)
  117. struct BuiltinEntry
  118. {
  119. const char* mName;
  120. bool operator<(const StringImpl& rhs) const
  121. {
  122. return strcmp(mName, rhs.c_str()) < 0;
  123. }
  124. };
  125. static const BuiltinEntry gIntrinEntries[] =
  126. {
  127. {":PLATFORM"},
  128. {"abs"},
  129. {"add"},
  130. {"and"},
  131. {"atomic_add"},
  132. {"atomic_and"},
  133. {"atomic_cmpstore"},
  134. {"atomic_cmpstore_weak"},
  135. {"atomic_cmpxchg"},
  136. {"atomic_fence"},
  137. {"atomic_load"},
  138. {"atomic_max"},
  139. {"atomic_min"},
  140. {"atomic_nand"},
  141. {"atomic_or"},
  142. {"atomic_store"},
  143. {"atomic_sub"},
  144. {"atomic_umax"},
  145. {"atomic_umin"},
  146. {"atomic_xchg"},
  147. {"atomic_xor"},
  148. {"bswap"},
  149. {"cast"},
  150. {"cos"},
  151. {"cpuid"},
  152. {"debugtrap"},
  153. {"div"},
  154. {"eq"},
  155. {"floor"},
  156. {"free"},
  157. {"gt"},
  158. {"gte"},
  159. ("index"),
  160. {"log"},
  161. {"log10"},
  162. {"log2"},
  163. {"lt"},
  164. {"lte"},
  165. {"malloc"},
  166. {"max"},
  167. {"memcpy"},
  168. {"memmove"},
  169. {"memset"},
  170. {"min"},
  171. {"mod"},
  172. {"mul"},
  173. {"neq"},
  174. {"not"},
  175. {"or"},
  176. {"pow"},
  177. {"powi"},
  178. {"returnaddress"},
  179. {"round"},
  180. {"sar"},
  181. {"shl"},
  182. {"shr"},
  183. {"shuffle"},
  184. {"sin"},
  185. {"sqrt"},
  186. {"sub"},
  187. {"va_arg"},
  188. {"va_end"},
  189. {"va_start"},
  190. {"xgetbv"},
  191. {"xor"},
  192. };
  193. #define CMD_PARAM(ty, name) ty name; Read(name);
  194. #define CMD_PARAM_NOTRANS(ty, name) ty name; Read(name, NULL, BfIRSizeAlignKind_NoTransform);
  195. BF_STATIC_ASSERT(BF_ARRAY_COUNT(gIntrinEntries) == BfIRIntrinsic_COUNT);
  196. template <typename T>
  197. class CmdParamVec : public llvm::SmallVector<T, 8>
  198. {};
  199. static int GetLLVMCallingConv(BfIRCallingConv callingConv, BfTargetTriple& targetTriple)
  200. {
  201. int llvmCallingConv = llvm::CallingConv::C;
  202. if (targetTriple.GetMachineType() == BfMachineType_AArch64)
  203. {
  204. if (callingConv == BfIRCallingConv_CDecl)
  205. llvmCallingConv = llvm::CallingConv::C;
  206. else
  207. llvmCallingConv = llvm::CallingConv::PreserveMost;
  208. }
  209. else
  210. {
  211. if (callingConv == BfIRCallingConv_ThisCall)
  212. llvmCallingConv = llvm::CallingConv::X86_ThisCall;
  213. else if (callingConv == BfIRCallingConv_StdCall)
  214. llvmCallingConv = llvm::CallingConv::X86_StdCall;
  215. else if (callingConv == BfIRCallingConv_FastCall)
  216. llvmCallingConv = llvm::CallingConv::X86_FastCall;
  217. else if (callingConv == BfIRCallingConv_CDecl)
  218. llvmCallingConv = llvm::CallingConv::C;
  219. }
  220. return llvmCallingConv;
  221. }
  222. static llvm::GlobalValue::LinkageTypes LLVMMapLinkageType(BfIRLinkageType linkageType)
  223. {
  224. llvm::GlobalValue::LinkageTypes llvmLinkageType;
  225. if (linkageType == BfIRLinkageType_Internal)
  226. llvmLinkageType = llvm::GlobalValue::InternalLinkage;
  227. else
  228. llvmLinkageType = llvm::GlobalValue::ExternalLinkage;
  229. return llvmLinkageType;
  230. }
  231. static llvm::Attribute::AttrKind LLVMMapAttribute(BfIRAttribute attr)
  232. {
  233. switch (attr)
  234. {
  235. case BfIRAttribute_NoReturn: return llvm::Attribute::NoReturn;
  236. case BfIRAttribute_NoAlias: return llvm::Attribute::NoAlias;
  237. case BfIRAttribute_NoCapture: return llvm::Attribute::NoCapture;
  238. case BfIRAttribute_StructRet: return llvm::Attribute::StructRet;
  239. case BfIRAttribute_ZExt: return llvm::Attribute::ZExt;
  240. case BFIRAttribute_NoUnwind: return llvm::Attribute::NoUnwind;
  241. case BFIRAttribute_UWTable: return llvm::Attribute::UWTable;
  242. case BFIRAttribute_AlwaysInline: return llvm::Attribute::AlwaysInline;
  243. case BFIRAttribute_NoRecurse: return llvm::Attribute::NoRecurse;
  244. default: break;
  245. }
  246. return llvm::Attribute::None;
  247. }
  248. #ifdef BF_PLATFORM_WINDOWS
  249. struct BfTempFile
  250. {
  251. String mContents;
  252. String mFilePath;
  253. CritSect mCritSect;
  254. FILE* mFP;
  255. BfTempFile()
  256. {
  257. mFP = NULL;
  258. }
  259. ~BfTempFile()
  260. {
  261. if (mFP != NULL)
  262. fclose(mFP);
  263. if (!mFilePath.IsEmpty())
  264. ::DeleteFileW(UTF8Decode(mFilePath).c_str());
  265. }
  266. bool Create()
  267. {
  268. AutoCrit autoCrit(mCritSect);
  269. if (mFP != NULL)
  270. return false;
  271. WCHAR wPath[4096];
  272. wPath[0] = 0;
  273. ::GetTempPathW(4096, wPath);
  274. WCHAR wFilePath[4096];
  275. wFilePath[0] = 0;
  276. GetTempFileNameW(wPath, L"bftmp", 0, wFilePath);
  277. mFilePath = UTF8Encode(wFilePath);
  278. mFP = _wfopen(wFilePath, L"w+D");
  279. return mFP != NULL;
  280. }
  281. String GetContents()
  282. {
  283. AutoCrit autoCrit(mCritSect);
  284. if (mFP != NULL)
  285. {
  286. fseek(mFP, 0, SEEK_END);
  287. int size = (int)ftell(mFP);
  288. fseek(mFP, 0, SEEK_SET);
  289. char* str = new char[size];
  290. int readSize = (int)fread(str, 1, size, mFP);
  291. mContents.Append(str, readSize);
  292. delete [] str;
  293. fclose(mFP);
  294. mFP = NULL;
  295. ::DeleteFileW(UTF8Decode(mFilePath).c_str());
  296. }
  297. return mContents;
  298. }
  299. };
  300. static BfTempFile gTempFile;
  301. static void AddStdErrCrashInfo()
  302. {
  303. String tempContents = gTempFile.GetContents();
  304. if (!tempContents.IsEmpty())
  305. BfpSystem_AddCrashInfo(tempContents.c_str());
  306. }
  307. #endif
  308. ///
  309. BfIRCodeGen::BfIRCodeGen()
  310. {
  311. mStream = NULL;
  312. mBfIRBuilder = NULL;
  313. mLLVMTargetMachine = NULL;
  314. mNopInlineAsm = NULL;
  315. mObjectCheckAsm = NULL;
  316. mOverflowCheckAsm = NULL;
  317. mHasDebugLoc = false;
  318. mAttrSet = NULL;
  319. mIRBuilder = NULL;
  320. mDIBuilder = NULL;
  321. mDICompileUnit = NULL;
  322. mActiveFunction = NULL;
  323. mActiveFunctionType = NULL;
  324. mLLVMContext = new llvm::LLVMContext();
  325. mLLVMModule = NULL;
  326. mIsCodeView = false;
  327. mHadDLLExport = false;
  328. mConstValIdx = 0;
  329. mCmdCount = 0;
  330. mCurLine = -1;
  331. #ifdef BF_PLATFORM_WINDOWS
  332. if (::GetStdHandle(STD_ERROR_HANDLE) == 0)
  333. {
  334. if (gTempFile.Create())
  335. {
  336. _dup2(fileno(gTempFile.mFP), 2);
  337. BfpSystem_AddCrashInfoFunc(AddStdErrCrashInfo);
  338. }
  339. }
  340. #endif
  341. }
  342. BfIRCodeGen::~BfIRCodeGen()
  343. {
  344. mDebugLoc = llvm::DebugLoc();
  345. mSavedDebugLocs.Clear();
  346. for (auto typeEx : mIRTypeExs)
  347. delete typeEx;
  348. for (auto kv : mTypeCodeTypeExMap)
  349. delete kv.mValue;
  350. delete mStream;
  351. delete mIRBuilder;
  352. delete mDIBuilder;
  353. delete mLLVMTargetMachine;
  354. delete mLLVMModule;
  355. delete mLLVMContext;
  356. }
  357. void BfIRCodeGen::FatalError(const StringImpl &err)
  358. {
  359. String failStr = "Fatal Error in Module: ";
  360. failStr += mModuleName;
  361. failStr += "\n";
  362. if (mLLVMModule != NULL)
  363. {
  364. if (mActiveFunction != NULL)
  365. {
  366. failStr += "Function: ";
  367. failStr += mActiveFunction->getName().str();
  368. failStr += "\n";
  369. }
  370. auto loc = mIRBuilder->getCurrentDebugLocation();
  371. auto dbgLoc = loc.getAsMDNode();
  372. if (dbgLoc != NULL)
  373. {
  374. std::string str;
  375. llvm::raw_string_ostream os(str);
  376. dbgLoc->print(os);
  377. failStr += "DbgLoc: ";
  378. failStr += str;
  379. failStr += "\n";
  380. llvm::MDNode* scope = loc.getScope();
  381. if (scope != NULL)
  382. {
  383. std::string str;
  384. llvm::raw_string_ostream os(str);
  385. scope->print(os);
  386. failStr += "Scope: ";
  387. failStr += str;
  388. failStr += "\n";
  389. }
  390. }
  391. }
  392. failStr += err;
  393. BF_FATAL(failStr);
  394. }
  395. void BfIRCodeGen::Fail(const StringImpl& error)
  396. {
  397. if (mFailed)
  398. return;
  399. if (mHasDebugLoc)
  400. {
  401. auto dbgLoc = mIRBuilder->getCurrentDebugLocation();
  402. if (dbgLoc)
  403. {
  404. llvm::DIFile* file = NULL;
  405. if (llvm::DIScope* scope = llvm::dyn_cast<llvm::DIScope>(dbgLoc.getScope()))
  406. {
  407. BfIRCodeGenBase::Fail(StrFormat("%s at line %d:%d in %s/%s", error.c_str(), dbgLoc.getLine(), dbgLoc.getCol(), scope->getDirectory().data(), scope->getFilename().data()));
  408. return;
  409. }
  410. }
  411. }
  412. BfIRCodeGenBase::Fail(error);
  413. }
  414. void BfIRCodeGen::PrintModule()
  415. {
  416. Beefy::debug_ostream os;
  417. mLLVMModule->print(os, NULL, false, true);
  418. os << "\n";
  419. os.flush();
  420. }
  421. void BfIRCodeGen::PrintFunction()
  422. {
  423. Beefy::debug_ostream os;
  424. mActiveFunction->print(os);
  425. os << "\n";
  426. os.flush();
  427. }
  428. void pte(BfIRTypeEx* typeEx, int indent)
  429. {
  430. Beefy::debug_ostream os;
  431. typeEx->mLLVMType->print(os);
  432. os << "\n";
  433. os.flush();
  434. for (int i = 0; i < typeEx->mMembers.mSize; i++)
  435. {
  436. for (int i = 0; i < indent; i++)
  437. os << " ";
  438. os << i << ". ";
  439. os.flush();
  440. pte(typeEx->mMembers[i], indent + 1);
  441. }
  442. }
  443. void pte(BfIRTypeEx* typeEx)
  444. {
  445. if (typeEx == NULL)
  446. return;
  447. pte(typeEx, 0);
  448. }
  449. void pve(const BfIRTypedValue& typedValue)
  450. {
  451. Beefy::debug_ostream os;
  452. os << "Value: ";
  453. typedValue.mValue->print(os);
  454. os << "\nType: ";
  455. os.flush();
  456. pte(typedValue.mTypeEx);
  457. }
  458. void pirb(llvm::IRBuilder<>* irBuilder)
  459. {
  460. Beefy::debug_ostream os;
  461. os << "Debug loc: ";
  462. auto debugLoc = irBuilder->getCurrentDebugLocation();
  463. if (debugLoc.get() == NULL)
  464. os << "NULL";
  465. else
  466. debugLoc->print(os);
  467. os << "\n";
  468. os.flush();
  469. }
  470. void BfIRCodeGen::FixValues(llvm::StructType* structType, llvm::SmallVector<llvm::Value*, 8>& values)
  471. {
  472. if (values.size() >= structType->getNumElements())
  473. return;
  474. int readIdx = (int)values.size() - 1;
  475. values.resize(structType->getNumElements());
  476. for (int i = (int)values.size() - 1; i >= 0; i--)
  477. {
  478. if (values[readIdx]->getType() == structType->getElementType(i))
  479. {
  480. values[i] = values[readIdx];
  481. readIdx--;
  482. }
  483. else if (structType->getElementType(i)->isArrayTy())
  484. {
  485. values[i] = llvm::ConstantAggregateZero::get(structType->getElementType(i));
  486. }
  487. else
  488. {
  489. BF_FATAL("Malformed structure values");
  490. }
  491. }
  492. }
  493. void BfIRCodeGen::FixIndexer(llvm::Value*& val)
  494. {
  495. if ((int)val->getType()->getScalarSizeInBits() > mPtrSize * 8)
  496. val = mIRBuilder->CreateIntCast(val, llvm::Type::getInt32Ty(*mLLVMContext), false);
  497. }
  498. BfTypeCode BfIRCodeGen::GetTypeCode(llvm::Type* type, bool isSigned)
  499. {
  500. if (type->isIntegerTy())
  501. {
  502. switch (type->getIntegerBitWidth())
  503. {
  504. case 1:
  505. return BfTypeCode_Boolean;
  506. case 8:
  507. return isSigned ? BfTypeCode_Int8 : BfTypeCode_UInt8;
  508. case 16:
  509. return isSigned ? BfTypeCode_Int16 : BfTypeCode_UInt16;
  510. case 32:
  511. return isSigned ? BfTypeCode_Int32 : BfTypeCode_UInt32;
  512. case 64:
  513. return isSigned ? BfTypeCode_Int64 : BfTypeCode_UInt64;
  514. }
  515. }
  516. if (type->isFloatingPointTy())
  517. return BfTypeCode_Float;
  518. if (type->isDoubleTy())
  519. return BfTypeCode_Double;
  520. return BfTypeCode_None;
  521. }
  522. llvm::Type* BfIRCodeGen::GetLLVMType(BfTypeCode typeCode, bool& isSigned)
  523. {
  524. if ((typeCode == BfTypeCode_IntPtr) || (typeCode == BfTypeCode_UIntPtr))
  525. {
  526. /*isSigned = typeCode == BfTypeCode_IntPtr;
  527. if (mModule->mSystem->mPtrSize == 4)
  528. return llvm::Type::getInt32Ty(*mLLVMContext);
  529. else
  530. return llvm::Type::getInt64Ty(*mLLVMContext);*/
  531. BF_FATAL("Unsupported");
  532. }
  533. isSigned = false;
  534. switch (typeCode)
  535. {
  536. case BfTypeCode_None:
  537. return llvm::Type::getVoidTy(*mLLVMContext);
  538. case BfTypeCode_NullPtr:
  539. return llvm::PointerType::get(*mLLVMContext, 0);
  540. case BfTypeCode_Boolean:
  541. return llvm::Type::getInt1Ty(*mLLVMContext);
  542. case BfTypeCode_Int8:
  543. isSigned = true;
  544. return llvm::Type::getInt8Ty(*mLLVMContext);
  545. case BfTypeCode_UInt8:
  546. case BfTypeCode_Char8:
  547. return llvm::Type::getInt8Ty(*mLLVMContext);
  548. case BfTypeCode_Int16:
  549. isSigned = true;
  550. return llvm::Type::getInt16Ty(*mLLVMContext);
  551. case BfTypeCode_UInt16:
  552. case BfTypeCode_Char16:
  553. return llvm::Type::getInt16Ty(*mLLVMContext);
  554. case BfTypeCode_Int24:
  555. isSigned = true;
  556. return llvm::Type::getIntNTy(*mLLVMContext, 24);
  557. case BfTypeCode_UInt24:
  558. return llvm::Type::getIntNTy(*mLLVMContext, 24);
  559. case BfTypeCode_Int32:
  560. isSigned = true;
  561. return llvm::Type::getInt32Ty(*mLLVMContext);
  562. case BfTypeCode_UInt32:
  563. case BfTypeCode_Char32:
  564. return llvm::Type::getInt32Ty(*mLLVMContext);
  565. case BfTypeCode_Int40:
  566. isSigned = true;
  567. return llvm::Type::getIntNTy(*mLLVMContext, 40);
  568. case BfTypeCode_UInt40:
  569. return llvm::Type::getIntNTy(*mLLVMContext, 40);
  570. case BfTypeCode_Int48:
  571. isSigned = true;
  572. return llvm::Type::getIntNTy(*mLLVMContext, 48);
  573. case BfTypeCode_UInt48:
  574. return llvm::Type::getIntNTy(*mLLVMContext, 48);
  575. case BfTypeCode_Int56:
  576. isSigned = true;
  577. return llvm::Type::getIntNTy(*mLLVMContext, 56);
  578. case BfTypeCode_UInt56:
  579. return llvm::Type::getIntNTy(*mLLVMContext, 56);
  580. case BfTypeCode_Int64:
  581. isSigned = true;
  582. return llvm::Type::getInt64Ty(*mLLVMContext);
  583. case BfTypeCode_UInt64:
  584. return llvm::Type::getInt64Ty(*mLLVMContext);
  585. case BfTypeCode_Int128:
  586. isSigned = true;
  587. return llvm::Type::getInt128Ty(*mLLVMContext);
  588. case BfTypeCode_UInt128:
  589. return llvm::Type::getInt128Ty(*mLLVMContext);
  590. case BfTypeCode_IntPtr:
  591. BF_FATAL("Illegal");
  592. /*isSigned = true;
  593. if (mModule->mSystem->mPtrSize == 4)
  594. return llvm::Type::getInt32Ty(*mLLVMContext);
  595. else
  596. return llvm::Type::getInt64Ty(*mLLVMContext);*/
  597. case BfTypeCode_UIntPtr:
  598. BF_FATAL("Illegal");
  599. /*if (mModule->mSystem->mPtrSize == 4)
  600. return llvm::Type::getInt32Ty(*mLLVMContext);
  601. else
  602. return llvm::Type::getInt64Ty(*mLLVMContext);*/
  603. case BfTypeCode_Float:
  604. return llvm::Type::getFloatTy(*mLLVMContext);
  605. case BfTypeCode_Double:
  606. return llvm::Type::getDoubleTy(*mLLVMContext);
  607. case BfTypeCode_Float2:
  608. return llvm::FixedVectorType::get(llvm::Type::getFloatTy(*mLLVMContext), 2);
  609. case BfTypeCode_FloatX2:
  610. return llvm::ArrayType::get(llvm::Type::getFloatTy(*mLLVMContext), 2);
  611. case BfTypeCode_FloatX3:
  612. return llvm::ArrayType::get(llvm::Type::getFloatTy(*mLLVMContext), 3);
  613. case BfTypeCode_FloatX4:
  614. return llvm::ArrayType::get(llvm::Type::getFloatTy(*mLLVMContext), 4);
  615. case BfTypeCode_DoubleX2:
  616. return llvm::ArrayType::get(llvm::Type::getDoubleTy(*mLLVMContext), 2);
  617. case BfTypeCode_DoubleX3:
  618. return llvm::ArrayType::get(llvm::Type::getDoubleTy(*mLLVMContext), 3);
  619. case BfTypeCode_DoubleX4:
  620. return llvm::ArrayType::get(llvm::Type::getDoubleTy(*mLLVMContext), 4);
  621. case BfTypeCode_Int64X2:
  622. return llvm::ArrayType::get(llvm::Type::getInt64Ty(*mLLVMContext), 2);
  623. case BfTypeCode_Int64X3:
  624. return llvm::ArrayType::get(llvm::Type::getInt64Ty(*mLLVMContext), 3);
  625. case BfTypeCode_Int64X4:
  626. return llvm::ArrayType::get(llvm::Type::getInt64Ty(*mLLVMContext), 4);
  627. default: break;
  628. }
  629. return NULL;
  630. }
  631. BfIRTypeEx* BfIRCodeGen::GetTypeEx(BfTypeCode typeCode, bool& isSigned)
  632. {
  633. BfIRTypeEx** valuePtr = NULL;
  634. if (mTypeCodeTypeExMap.TryAdd(typeCode, NULL, &valuePtr))
  635. {
  636. BfIRTypeEx* typeEx = new BfIRTypeEx();
  637. typeEx->mLLVMType = GetLLVMType(typeCode, isSigned);
  638. if (typeEx->mLLVMType->isPointerTy())
  639. {
  640. // Make void* actually be an i8*
  641. typeEx->mMembers.Add(GetTypeEx(llvm::Type::getInt8Ty(*mLLVMContext)));
  642. }
  643. if (auto arrType = llvm::dyn_cast<llvm::ArrayType>(typeEx->mLLVMType))
  644. {
  645. typeEx->mMembers.Add(GetTypeEx(arrType->getElementType()));
  646. }
  647. if (auto vectorType = llvm::dyn_cast<llvm::VectorType>(typeEx->mLLVMType))
  648. {
  649. typeEx->mMembers.Add(GetTypeEx(vectorType->getElementType()));
  650. }
  651. *valuePtr = typeEx;
  652. }
  653. else
  654. {
  655. isSigned = false;
  656. switch (typeCode)
  657. {
  658. case BfTypeCode_Int8:
  659. case BfTypeCode_Int16:
  660. case BfTypeCode_Int24:
  661. case BfTypeCode_Int32:
  662. case BfTypeCode_Int40:
  663. case BfTypeCode_Int48:
  664. case BfTypeCode_Int56:
  665. case BfTypeCode_Int64:
  666. case BfTypeCode_Int128:
  667. isSigned = true;
  668. }
  669. }
  670. return *valuePtr;
  671. }
  672. BfIRTypeEx* BfIRCodeGen::GetTypeEx(llvm::Type* llvmType)
  673. {
  674. BfIRTypeEx** valuePtr = NULL;
  675. if (mLLVMTypeExMap.TryAdd(llvmType, NULL, &valuePtr))
  676. {
  677. BfIRTypeEx* typeEx = new BfIRTypeEx();
  678. mIRTypeExs.Add(typeEx);
  679. typeEx->mLLVMType = llvmType;
  680. *valuePtr = typeEx;
  681. }
  682. return *valuePtr;
  683. }
  684. BfIRTypeEx* BfIRCodeGen::CreateTypeEx(llvm::Type* llvmType)
  685. {
  686. BfIRTypeEx* typeEx = new BfIRTypeEx();
  687. mIRTypeExs.Add(typeEx);
  688. typeEx->mLLVMType = llvmType;
  689. return typeEx;
  690. }
  691. BfIRTypeEx* BfIRCodeGen::GetPointerTypeEx(BfIRTypeEx* elementType)
  692. {
  693. BF_ASSERT(elementType != NULL);
  694. BfIRTypeEx** valuePtr = NULL;
  695. if (mPointerTypeExMap.TryAdd(elementType, NULL, &valuePtr))
  696. {
  697. BfIRTypeEx* typeEx = new BfIRTypeEx();
  698. mIRTypeExs.Add(typeEx);
  699. typeEx->mLLVMType = llvm::PointerType::get(*mLLVMContext, 0);
  700. typeEx->mMembers.Add(elementType);
  701. *valuePtr = typeEx;
  702. }
  703. return *valuePtr;
  704. }
  705. BfIRTypeEx* BfIRCodeGen::GetTypeMember(BfIRTypeEx* typeEx, int idx)
  706. {
  707. if ((idx < 0) || (idx >= typeEx->mMembers.mSize))
  708. {
  709. Fail("BfIRTypeEx GetTypeMember OOB");
  710. bool isSigned;
  711. return GetTypeEx(BfTypeCode_Int8, isSigned);
  712. }
  713. return typeEx->mMembers[idx];
  714. }
  715. BfIRTypeEntry& BfIRCodeGen::GetTypeEntry(int typeId)
  716. {
  717. BfIRTypeEntry& typeEntry = mTypes[typeId];
  718. if (typeEntry.mTypeId == -1)
  719. typeEntry.mTypeId = typeId;
  720. return typeEntry;
  721. }
  722. BfIRTypeEntry* BfIRCodeGen::GetTypeEntry(BfIRTypeEx* type)
  723. {
  724. int typeId = 0;
  725. if (!mTypeToTypeIdMap.TryGetValue(type, &typeId))
  726. return NULL;
  727. return &GetTypeEntry(typeId);
  728. }
  729. void BfIRCodeGen::SetResult(int id, llvm::Value* value)
  730. {
  731. BF_ASSERT(!value->getType()->isAggregateType());
  732. BF_ASSERT(!value->getType()->isPointerTy());
  733. BfIRCodeGenEntry entry;
  734. entry.mKind = BfIRCodeGenEntryKind_LLVMValue;
  735. entry.mLLVMValue = value;
  736. mResults.TryAdd(id, entry);
  737. }
  738. void BfIRCodeGen::SetResult(int id, const BfIRTypedValue& value)
  739. {
  740. BfIRCodeGenEntry entry;
  741. entry.mKind = BfIRCodeGenEntryKind_TypedValue;
  742. entry.mTypedValue = value;
  743. mResults.TryAdd(id, entry);
  744. }
  745. void BfIRCodeGen::SetResultAligned(int id, llvm::Value* value)
  746. {
  747. BfIRCodeGenEntry entry;
  748. entry.mKind = BfIRCodeGenEntryKind_LLVMValue_Aligned;
  749. entry.mLLVMValue = value;
  750. mResults.TryAdd(id, entry);
  751. }
  752. void BfIRCodeGen::SetResultAligned(int id, const BfIRTypedValue& value)
  753. {
  754. BfIRCodeGenEntry entry;
  755. entry.mKind = BfIRCodeGenEntryKind_TypedValue_Aligned;
  756. entry.mTypedValue = value;
  757. mResults.TryAdd(id, entry);
  758. }
  759. void BfIRCodeGen::SetResult(int id, llvm::Type* type)
  760. {
  761. BfIRCodeGenEntry entry;
  762. entry.mKind = BfIRCodeGenEntryKind_LLVMType;
  763. entry.mLLVMType = type;
  764. mResults.TryAdd(id, entry);
  765. }
  766. void BfIRCodeGen::SetResult(int id, BfIRTypeEx* typeEx)
  767. {
  768. BfIRCodeGenEntry entry;
  769. entry.mKind = BfIRCodeGenEntryKind_TypeEx;
  770. entry.mTypeEx = typeEx;
  771. mResults.TryAdd(id, entry);
  772. }
  773. void BfIRCodeGen::SetResult(int id, llvm::BasicBlock* value)
  774. {
  775. BfIRCodeGenEntry entry;
  776. entry.mKind = BfIRCodeGenEntryKind_LLVMBasicBlock;
  777. entry.mLLVMBlock = value;
  778. mResults.TryAdd(id, entry);
  779. }
  780. void BfIRCodeGen::SetResult(int id, llvm::MDNode* md)
  781. {
  782. BfIRCodeGenEntry entry;
  783. entry.mKind = BfIRCodeGenEntryKind_LLVMMetadata;
  784. entry.mLLVMMetadata = md;
  785. mResults.TryAdd(id, entry);
  786. }
  787. void BfIRCodeGen::ProcessBfIRData(const BfSizedArray<uint8>& buffer)
  788. {
  789. // Diagnostic handlers were unified in LLVM change 5de2d189e6ad, so starting
  790. // with LLVM 13 this function is gone.
  791. /*struct InlineAsmErrorHook
  792. {
  793. static void StaticHandler(const llvm::SMDiagnostic& diag, void *context, unsigned locCookie)
  794. {
  795. if (diag.getKind() == llvm::SourceMgr::DK_Error)
  796. {
  797. BfIRCodeGen* irCodeGen = (BfIRCodeGen*)context;
  798. if (!irCodeGen->mErrorMsg.empty())
  799. irCodeGen->mErrorMsg += "\n";
  800. irCodeGen->mErrorMsg += StrFormat("Inline assembly error: \"%s\" : %s", diag.getMessage().data(), diag.getLineContents().data());
  801. }
  802. }
  803. };
  804. mLLVMContext->setInlineAsmDiagnosticHandler(InlineAsmErrorHook::StaticHandler, this);*/
  805. BF_ASSERT(mStream == NULL);
  806. mStream = new ChunkedDataBuffer();
  807. mStream->InitFlatRef(buffer.mVals, buffer.mSize);
  808. while (mStream->GetReadPos() < buffer.mSize)
  809. {
  810. if (mFailed)
  811. break;
  812. HandleNextCmd();
  813. }
  814. BF_ASSERT((mFailed) || (mStream->GetReadPos() == buffer.mSize));
  815. }
  816. int64 BfIRCodeGen::ReadSLEB128()
  817. {
  818. int64 val = 0;
  819. int64 shift = 0;
  820. uint8 byteVal;
  821. do
  822. {
  823. byteVal = mStream->Read();
  824. val |= ((int64)(byteVal & 0x7f)) << shift;
  825. shift += 7;
  826. } while (byteVal >= 128);
  827. // Sign extend negative numbers.
  828. if ((byteVal & 0x40) && (shift < 64))
  829. val |= (-1ULL) << shift;
  830. return val;
  831. }
  832. void BfIRCodeGen::Read(StringImpl& str)
  833. {
  834. int len = (int)ReadSLEB128();
  835. str.Append('?', len);
  836. mStream->Read((void*)str.c_str(), len);
  837. }
  838. void BfIRCodeGen::Read(int& i)
  839. {
  840. i = (int)ReadSLEB128();
  841. }
  842. void BfIRCodeGen::Read(int64& i)
  843. {
  844. i = ReadSLEB128();
  845. }
  846. void BfIRCodeGen::Read(Val128& i)
  847. {
  848. i.mLow = (uint64)ReadSLEB128();
  849. i.mHigh = (uint64)ReadSLEB128();
  850. }
  851. void BfIRCodeGen::Read(bool& val)
  852. {
  853. val = mStream->Read() != 0;
  854. }
  855. void BfIRCodeGen::Read(int8& val)
  856. {
  857. val = mStream->Read();
  858. }
  859. void BfIRCodeGen::Read(BfIRTypeEntry*& type)
  860. {
  861. int typeId = (int)ReadSLEB128();
  862. type = &GetTypeEntry(typeId);
  863. }
  864. void BfIRCodeGen::Read(BfIRTypeEx*& typeEx, BfIRTypeEntry** outTypeEntry)
  865. {
  866. typeEx = NULL;
  867. BfIRType::TypeKind typeKind = (BfIRType::TypeKind)mStream->Read();
  868. if (typeKind == BfIRType::TypeKind::TypeKind_None)
  869. return;
  870. if (typeKind == BfIRType::TypeKind::TypeKind_Stream)
  871. {
  872. int streamId = (int)ReadSLEB128();
  873. if (streamId == -1)
  874. {
  875. typeEx = NULL;
  876. return;
  877. }
  878. auto& result = mResults[streamId];
  879. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_TypeEx);
  880. typeEx = result.mTypeEx;
  881. return;
  882. }
  883. if (typeKind == BfIRType::TypeKind::TypeKind_SizedArray)
  884. {
  885. CMD_PARAM(BfIRTypeEx*, elementType);
  886. CMD_PARAM(int, length);
  887. typeEx = new BfIRTypeEx();
  888. typeEx->mLLVMType = llvm::ArrayType::get(elementType->mLLVMType, length);
  889. BF_ASSERT(elementType != NULL);
  890. typeEx->mMembers.Add(elementType);
  891. mIRTypeExs.Add(typeEx);
  892. return;
  893. }
  894. int typeId = (int)ReadSLEB128();
  895. if (typeKind == BfIRType::TypeKind::TypeKind_TypeCode)
  896. {
  897. bool isSigned = false;
  898. typeEx = GetTypeEx((BfTypeCode)typeId, isSigned);
  899. return;
  900. }
  901. auto& typeEntry = GetTypeEntry(typeId);
  902. if (typeKind == BfIRType::TypeKind::TypeKind_TypeId)
  903. typeEx = typeEntry.mType;
  904. else if (typeKind == BfIRType::TypeKind::TypeKind_TypeInstId)
  905. typeEx = typeEntry.mInstType;
  906. else if (typeKind == BfIRType::TypeKind::TypeKind_TypeInstPtrId)
  907. typeEx = GetPointerTypeEx(typeEntry.mInstType);
  908. if (outTypeEntry != NULL)
  909. *outTypeEntry = &typeEntry;
  910. }
  911. void BfIRCodeGen::Read(llvm::Type*& llvmType, BfIRTypeEntry** outTypeEntry)
  912. {
  913. BfIRTypeEx* typeEx = NULL;
  914. Read(typeEx, outTypeEntry);
  915. if (typeEx != NULL)
  916. {
  917. llvmType = typeEx->mLLVMType;
  918. }
  919. else
  920. llvmType = NULL;
  921. }
  922. void BfIRCodeGen::Read(llvm::FunctionType*& llvmType)
  923. {
  924. int streamId = (int)ReadSLEB128();
  925. auto& result = mResults[streamId];
  926. if (result.mKind == BfIRCodeGenEntryKind_TypeEx)
  927. {
  928. llvmType = (llvm::FunctionType*)result.mTypeEx->mLLVMType;
  929. return;
  930. }
  931. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMType);
  932. llvmType = (llvm::FunctionType*)result.mLLVMType;
  933. }
  934. void BfIRCodeGen::ReadFunctionType(BfIRTypeEx*& typeEx)
  935. {
  936. int streamId = (int)ReadSLEB128();
  937. auto& result = mResults[streamId];
  938. if (result.mKind == BfIRCodeGenEntryKind_TypeEx)
  939. {
  940. typeEx = result.mTypeEx;
  941. return;
  942. }
  943. BF_FATAL("Invalid path in ReadFunctionType");
  944. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMType);
  945. typeEx = GetTypeEx(result.mLLVMType);
  946. }
  947. void BfIRCodeGen::FixTypedValue(BfIRTypedValue& typedValue)
  948. {
  949. if ((typedValue.mValue != NULL) && (typedValue.mTypeEx == NULL))
  950. {
  951. typedValue.mTypeEx = GetTypeEx(typedValue.mValue->getType());
  952. BF_ASSERT(!typedValue.mValue->getType()->isStructTy());
  953. BF_ASSERT(!typedValue.mValue->getType()->isFunctionTy());
  954. }
  955. }
  956. void BfIRCodeGen::Read(BfIRTypedValue& typedValue, BfIRCodeGenEntry** codeGenEntry, BfIRSizeAlignKind sizeAlignKind)
  957. {
  958. typedValue.mValue = NULL;
  959. typedValue.mTypeEx = NULL;
  960. BfIRParamType paramType = (BfIRParamType)mStream->Read();
  961. if (paramType == BfIRParamType_None)
  962. {
  963. //
  964. }
  965. else if (paramType == BfIRParamType_Const)
  966. {
  967. BfTypeCode typeCode = (BfTypeCode)mStream->Read();
  968. BfConstType constType = (BfConstType)typeCode;
  969. if (constType == BfConstType_GlobalVar)
  970. {
  971. CMD_PARAM(int, streamId);
  972. if (streamId == -1)
  973. {
  974. int streamId = mCmdCount++;
  975. CMD_PARAM(BfIRTypeEx*, varType);
  976. CMD_PARAM(bool, isConstant);
  977. BfIRLinkageType linkageType = (BfIRLinkageType)mStream->Read();
  978. CMD_PARAM(llvm::Constant*, initializer);
  979. CMD_PARAM(String, name);
  980. CMD_PARAM(bool, isTLS);
  981. llvm::GlobalVariable* globalVariable = mLLVMModule->getGlobalVariable(name.c_str(), true);
  982. if (globalVariable == NULL)
  983. {
  984. globalVariable = mLLVMModule->getGlobalVariable(name.c_str());
  985. if (globalVariable == NULL)
  986. {
  987. globalVariable = new llvm::GlobalVariable(
  988. *mLLVMModule,
  989. varType->mLLVMType,
  990. isConstant,
  991. LLVMMapLinkageType(linkageType),
  992. initializer,
  993. name.c_str(), NULL, isTLS ? llvm::GlobalValue::GeneralDynamicTLSModel : llvm::GlobalValue::NotThreadLocal);
  994. }
  995. }
  996. typedValue.mTypeEx = GetPointerTypeEx(varType);
  997. typedValue.mValue = globalVariable;
  998. SetResult(streamId, typedValue);
  999. }
  1000. else
  1001. typedValue = GetTypedValue(streamId);
  1002. FixTypedValue(typedValue);
  1003. return;
  1004. }
  1005. /*else if (constType == BfConstType_GlobalVar_TypeInst)
  1006. {
  1007. CMD_PARAM(int, streamId);
  1008. if (streamId == -1)
  1009. {
  1010. int streamId = mStream->GetReadPos();
  1011. CMD_PARAM(int, varTypeId);
  1012. auto& typeEntry = GetTypeEntry(varTypeId);
  1013. auto varType = typeEntry.mInstLLVMType;
  1014. CMD_PARAM(bool, isConstant);
  1015. BfIRLinkageType linkageType = (BfIRLinkageType)mStream->Read();
  1016. CMD_PARAM(llvm::Constant*, initializer);
  1017. CMD_PARAM(String, name);
  1018. CMD_PARAM(bool, isTLS);
  1019. auto globalVariable = new llvm::GlobalVariable(
  1020. *mLLVMModule,
  1021. varType,
  1022. isConstant,
  1023. LLVMMapLinkageType(linkageType),
  1024. initializer,
  1025. name, NULL, isTLS ? llvm::GlobalValue::GeneralDynamicTLSModel : llvm::GlobalValue::NotThreadLocal);
  1026. llvmValue = globalVariable;
  1027. SetResult(streamId, globalVariable);
  1028. }
  1029. else
  1030. llvmValue = GetLLVMValue(streamId);
  1031. return;
  1032. }*/
  1033. else if ((constType == BfConstType_BitCast) || (constType == BfConstType_BitCastNull))
  1034. {
  1035. CMD_PARAM(llvm::Constant*, target);
  1036. CMD_PARAM(BfIRTypeEx*, toType);
  1037. typedValue.mTypeEx = toType;
  1038. if ((constType == BfConstType_BitCastNull) && (toType->mLLVMType->isIntegerTy()))
  1039. {
  1040. typedValue.mValue = llvm::ConstantInt::getNullValue(toType->mLLVMType);
  1041. }
  1042. else if (target->getType()->isIntegerTy())
  1043. typedValue.mValue = llvm::ConstantExpr::getIntToPtr(target, toType->mLLVMType);
  1044. else
  1045. typedValue.mValue = llvm::ConstantExpr::getBitCast(target, toType->mLLVMType);
  1046. FixTypedValue(typedValue);
  1047. return;
  1048. }
  1049. else if (constType == BfConstType_GEP32_1)
  1050. {
  1051. CMD_PARAM(BfIRTypedValue, target);
  1052. CMD_PARAM(int, idx0);
  1053. llvm::Value* gepArgs[] = {
  1054. llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), idx0)};
  1055. auto compositeType = GetTypeMember(target.mTypeEx, 0);
  1056. auto constant = llvm::dyn_cast<llvm::Constant>(target.mValue);
  1057. typedValue.mTypeEx = target.mTypeEx;
  1058. typedValue.mValue = llvm::ConstantExpr::getInBoundsGetElementPtr(compositeType->mLLVMType, constant, gepArgs);
  1059. FixTypedValue(typedValue);
  1060. return;
  1061. }
  1062. else if (constType == BfConstType_GEP32_2)
  1063. {
  1064. CMD_PARAM(BfIRTypedValue, target);
  1065. CMD_PARAM(int, idx0);
  1066. CMD_PARAM(int, idx1);
  1067. llvm::Value* gepArgs[] = {
  1068. llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), idx0),
  1069. llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), idx1)};
  1070. auto compositeType = GetTypeMember(target.mTypeEx, 0);
  1071. int elemIdx = BF_MAX(BF_MIN(idx1, (int)compositeType->mMembers.mSize - 1), 0);
  1072. auto elemType = GetTypeMember(compositeType, elemIdx);
  1073. auto constant = llvm::dyn_cast<llvm::Constant>(target.mValue);
  1074. typedValue.mValue = llvm::ConstantExpr::getInBoundsGetElementPtr(compositeType->mLLVMType, constant, gepArgs);
  1075. typedValue.mTypeEx = GetPointerTypeEx(elemType);
  1076. return;
  1077. }
  1078. else if (constType == BfConstType_ExtractValue)
  1079. {
  1080. CMD_PARAM(BfIRTypedValue, target);
  1081. CMD_PARAM(int, idx0);
  1082. auto compositeType = target.mTypeEx;
  1083. int elemIdx = BF_MIN(idx0, (int)compositeType->mMembers.mSize - 1);
  1084. auto elemType = GetTypeMember(compositeType, elemIdx);
  1085. typedValue.mTypeEx = elemType;
  1086. if (auto constant = llvm::dyn_cast<llvm::Constant>(target.mValue))
  1087. typedValue.mValue = constant->getAggregateElement(llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), idx0));
  1088. FixTypedValue(typedValue);
  1089. return;
  1090. }
  1091. else if (constType == BfConstType_PtrToInt)
  1092. {
  1093. CMD_PARAM(llvm::Constant*, target);
  1094. BfTypeCode toTypeCode = (BfTypeCode)mStream->Read();
  1095. bool isSigned;
  1096. BfIRTypeEx* toType = GetTypeEx(toTypeCode, isSigned);
  1097. typedValue.mTypeEx = toType;
  1098. typedValue.mValue = llvm::ConstantExpr::getPtrToInt(target, toType->mLLVMType);
  1099. FixTypedValue(typedValue);
  1100. return;
  1101. }
  1102. else if (constType == BfConstType_IntToPtr)
  1103. {
  1104. CMD_PARAM(llvm::Constant*, target);
  1105. CMD_PARAM(BfIRTypeEx*, toType);
  1106. typedValue.mTypeEx = toType;
  1107. typedValue.mValue = llvm::ConstantExpr::getIntToPtr(target, toType->mLLVMType);
  1108. FixTypedValue(typedValue);
  1109. return;
  1110. }
  1111. else if (constType == BfConstType_AggZero)
  1112. {
  1113. BfIRTypeEntry* typeEntry = NULL;
  1114. BfIRTypeEx* type = NULL;
  1115. Read(type, &typeEntry);
  1116. typedValue.mTypeEx = type;
  1117. if ((sizeAlignKind == BfIRSizeAlignKind_Aligned) && (typeEntry != NULL))
  1118. typedValue.mValue = llvm::ConstantAggregateZero::get(GetSizeAlignedType(typeEntry)->mLLVMType);
  1119. else
  1120. typedValue.mValue = llvm::ConstantAggregateZero::get(type->mLLVMType);
  1121. FixTypedValue(typedValue);
  1122. return;
  1123. }
  1124. else if (constType == BfConstType_ArrayZero8)
  1125. {
  1126. CMD_PARAM(int, count);
  1127. auto arrType = llvm::ArrayType::get(llvm::Type::getInt8Ty(*mLLVMContext), count);
  1128. typedValue.mValue = llvm::ConstantAggregateZero::get(arrType);
  1129. FixTypedValue(typedValue);
  1130. return;
  1131. }
  1132. else if (constType == BfConstType_Agg)
  1133. {
  1134. BfIRTypeEntry* typeEntry = NULL;
  1135. BfIRTypeEx* type = NULL;
  1136. Read(type, &typeEntry);
  1137. CmdParamVec<llvm::Constant*> values;
  1138. Read(values, type->mLLVMType->isArrayTy() ? BfIRSizeAlignKind_Aligned : BfIRSizeAlignKind_Original);
  1139. typedValue.mTypeEx = type;
  1140. if (auto arrayType = llvm::dyn_cast<llvm::ArrayType>(type->mLLVMType))
  1141. {
  1142. int fillCount = (int)(arrayType->getNumElements() - values.size());
  1143. if (fillCount > 0)
  1144. {
  1145. auto lastValue = values.back();
  1146. for (int i = 0; i < fillCount; i++)
  1147. values.push_back(lastValue);
  1148. }
  1149. typedValue.mValue = llvm::ConstantArray::get(arrayType, values);
  1150. }
  1151. else if (auto structType = llvm::dyn_cast<llvm::StructType>(type->mLLVMType))
  1152. {
  1153. for (int i = 0; i < (int)values.size(); i++)
  1154. {
  1155. if (values[i]->getType() != structType->getElementType(i))
  1156. {
  1157. auto valArrayType = llvm::dyn_cast<llvm::ArrayType>(values[i]->getType());
  1158. if (valArrayType != NULL)
  1159. {
  1160. if (valArrayType->getNumElements() == 0)
  1161. {
  1162. values[i] = llvm::ConstantAggregateZero::get(structType->getElementType(i));
  1163. }
  1164. }
  1165. }
  1166. }
  1167. if ((sizeAlignKind == BfIRSizeAlignKind_Aligned) && (typeEntry != NULL))
  1168. {
  1169. auto alignedTypeEx = GetSizeAlignedType(typeEntry);
  1170. auto alignedType = llvm::dyn_cast<llvm::StructType>(alignedTypeEx->mLLVMType);
  1171. if (type != alignedTypeEx)
  1172. values.push_back(llvm::ConstantAggregateZero::get(alignedType->getElementType(alignedType->getNumElements() - 1)));
  1173. typedValue.mTypeEx = alignedTypeEx;
  1174. typedValue.mValue = llvm::ConstantStruct::get(alignedType, values);
  1175. }
  1176. else
  1177. {
  1178. typedValue.mValue = llvm::ConstantStruct::get(structType, values);
  1179. }
  1180. }
  1181. else if (auto vecType = llvm::dyn_cast<llvm::VectorType>(type->mLLVMType))
  1182. {
  1183. typedValue.mValue = llvm::ConstantVector::get(values);
  1184. }
  1185. else
  1186. {
  1187. typedValue.mValue = NULL;
  1188. Fail("Bad type");
  1189. }
  1190. FixTypedValue(typedValue);
  1191. return;
  1192. }
  1193. else if (constType == BfConstType_Undef)
  1194. {
  1195. CMD_PARAM(BfIRTypeEx*, type);
  1196. typedValue.mTypeEx = type;
  1197. typedValue.mValue = llvm::UndefValue::get(type->mLLVMType);
  1198. return;
  1199. }
  1200. else if (constType == BfConstType_TypeOf)
  1201. {
  1202. CMD_PARAM(BfIRTypeEx*, type);
  1203. typedValue = mReflectDataMap[type];
  1204. BF_ASSERT(typedValue.mValue != NULL);
  1205. return;
  1206. }
  1207. else if (constType == BfConstType_TypeOf_WithData)
  1208. {
  1209. CMD_PARAM(BfIRTypeEx*, type);
  1210. CMD_PARAM(BfIRTypedValue, value);
  1211. mReflectDataMap[type] = value;
  1212. typedValue = value;
  1213. return;
  1214. }
  1215. bool isSigned;
  1216. llvm::Type* llvmConstType = GetLLVMType(typeCode, isSigned);
  1217. if (typeCode == BfTypeCode_Float)
  1218. {
  1219. float f;
  1220. mStream->Read(&f, sizeof(float));
  1221. typedValue.mValue = llvm::ConstantFP::get(llvmConstType, f);
  1222. }
  1223. else if (typeCode == BfTypeCode_Double)
  1224. {
  1225. double d;
  1226. mStream->Read(&d, sizeof(double));
  1227. typedValue.mValue = llvm::ConstantFP::get(llvmConstType, d);
  1228. }
  1229. else if (typeCode == BfTypeCode_Boolean)
  1230. {
  1231. CMD_PARAM(bool, boolVal);
  1232. typedValue.mValue = llvm::ConstantInt::get(llvmConstType, boolVal ? 1 : 0);
  1233. }
  1234. else if (typeCode == BfTypeCode_None)
  1235. {
  1236. typedValue.mValue = NULL;
  1237. }
  1238. else if (typeCode == BfTypeCode_NullPtr)
  1239. {
  1240. CMD_PARAM(llvm::Type*, nullType);
  1241. if (nullType != NULL)
  1242. typedValue.mValue = llvm::ConstantPointerNull::get((llvm::PointerType*)nullType);
  1243. else
  1244. typedValue.mValue = llvm::ConstantPointerNull::get((llvm::PointerType*)llvmConstType);
  1245. }
  1246. else if (BfIRBuilder::IsInt(typeCode))
  1247. {
  1248. int64 intVal = ReadSLEB128();
  1249. auto constVal = llvm::ConstantInt::get(llvmConstType, intVal);
  1250. auto constInt = (llvm::ConstantInt*)constVal;
  1251. typedValue.mValue = constInt;
  1252. }
  1253. else
  1254. {
  1255. BF_FATAL("Unhandled");
  1256. }
  1257. }
  1258. else if (paramType == BfIRParamType_Arg)
  1259. {
  1260. int argIdx = mStream->Read();
  1261. if (argIdx >= mActiveFunction->arg_size())
  1262. {
  1263. FatalError(StrFormat("ARG out of bounds %d", argIdx));
  1264. }
  1265. auto typeEx = mActiveFunctionType;
  1266. BF_ASSERT(argIdx < mActiveFunction->arg_size());
  1267. auto argItr = mActiveFunction->arg_begin();
  1268. for (int i = 0; i < argIdx; i++)
  1269. argItr++;
  1270. typedValue.mValue = &(*argItr);
  1271. typedValue.mTypeEx = GetTypeMember(typeEx, argIdx + 1);
  1272. }
  1273. else
  1274. {
  1275. int cmdId = -1;
  1276. if (paramType == BfIRParamType_StreamId_Abs8)
  1277. {
  1278. cmdId = mStream->Read();
  1279. }
  1280. else if (paramType == BfIRParamType_StreamId_Rel)
  1281. {
  1282. cmdId = mCmdCount - (int)ReadSLEB128();
  1283. }
  1284. else
  1285. {
  1286. cmdId = mCmdCount - (paramType - BfIRParamType_StreamId_Back1) - 1;
  1287. }
  1288. auto& result = mResults[cmdId];
  1289. if ((codeGenEntry != NULL) && (result.mKind != BfIRCodeGenEntryKind_None))
  1290. *codeGenEntry = &result;
  1291. if (result.mKind == BfIRCodeGenEntryKind_TypedValue_Aligned)
  1292. {
  1293. typedValue = result.mTypedValue;
  1294. BfIRTypeEx* normalType = NULL;
  1295. if (mAlignedTypeToNormalType.TryGetValue(typedValue.mTypeEx, &normalType))
  1296. typedValue.mTypeEx = normalType;
  1297. return;
  1298. }
  1299. if (result.mKind == BfIRCodeGenEntryKind_TypedValue)
  1300. {
  1301. typedValue = result.mTypedValue;
  1302. return;
  1303. }
  1304. if (result.mKind != BfIRCodeGenEntryKind_LLVMValue)
  1305. {
  1306. if ((codeGenEntry != NULL) && (result.mKind != BfIRCodeGenEntryKind_None))
  1307. {
  1308. *codeGenEntry = &result;
  1309. return;
  1310. }
  1311. }
  1312. if (result.mKind == BfIRCodeGenEntryKind_LLVMValue_Aligned)
  1313. {
  1314. typedValue.mValue = result.mLLVMValue;
  1315. if (sizeAlignKind != BfIRSizeAlignKind_Original)
  1316. return;
  1317. llvm::Type* normalType = NULL;
  1318. //TODO: if (auto ptrType = llvm::dyn_cast<llvm::PointerType>(llvmValue->getType()))
  1319. {
  1320. // if (mAlignedTypeToNormalType.TryGetValue(ptrType->getElementType(), &normalType))
  1321. // {
  1322. // llvmValue = mIRBuilder->CreateBitCast(llvmValue, normalType->getPointerTo());
  1323. // return;
  1324. // }
  1325. }
  1326. }
  1327. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMValue);
  1328. typedValue.mValue = result.mLLVMValue;
  1329. }
  1330. FixTypedValue(typedValue);
  1331. }
  1332. void BfIRCodeGen::Read(llvm::Value*& llvmValue, BfIRCodeGenEntry** codeGenEntry, BfIRSizeAlignKind sizeAlignKind)
  1333. {
  1334. BfIRTypedValue typedValue;
  1335. Read(typedValue, codeGenEntry, sizeAlignKind);
  1336. llvmValue = typedValue.mValue;
  1337. }
  1338. void BfIRCodeGen::Read(llvm::Constant*& llvmConstant, BfIRSizeAlignKind sizeAlignKind)
  1339. {
  1340. llvm::Value* value;
  1341. Read(value, NULL, sizeAlignKind);
  1342. if (value == NULL)
  1343. {
  1344. llvmConstant = NULL;
  1345. }
  1346. else
  1347. {
  1348. BF_ASSERT(llvm::isa<llvm::Constant>(value));
  1349. llvmConstant = (llvm::Constant*)value;
  1350. }
  1351. }
  1352. void BfIRCodeGen::Read(llvm::Function*& llvmFunc)
  1353. {
  1354. int streamId = (int)ReadSLEB128();
  1355. if (streamId == -1)
  1356. {
  1357. llvmFunc = NULL;
  1358. return;
  1359. }
  1360. auto& result = mResults[streamId];
  1361. if (result.mKind == BfIRCodeGenEntryKind_TypedValue)
  1362. {
  1363. llvmFunc = (llvm::Function*)result.mTypedValue.mValue;
  1364. return;
  1365. }
  1366. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMValue);
  1367. BF_ASSERT(llvm::isa<llvm::Function>(result.mLLVMValue));
  1368. llvmFunc = (llvm::Function*)result.mLLVMValue;
  1369. }
  1370. void BfIRCodeGen::ReadFunction(BfIRTypedValue& typedValue)
  1371. {
  1372. int streamId = (int)ReadSLEB128();
  1373. if (streamId == -1)
  1374. {
  1375. typedValue.mValue = NULL;
  1376. typedValue.mTypeEx = NULL;
  1377. return;
  1378. }
  1379. auto& result = mResults[streamId];
  1380. if (result.mKind == BfIRCodeGenEntryKind_TypedValue)
  1381. {
  1382. typedValue = result.mTypedValue;
  1383. return;
  1384. }
  1385. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMValue);
  1386. BF_ASSERT(llvm::isa<llvm::Function>(result.mLLVMValue));
  1387. typedValue.mValue = result.mLLVMValue;
  1388. FixTypedValue(typedValue);
  1389. }
  1390. void BfIRCodeGen::Read(llvm::BasicBlock*& llvmBlock)
  1391. {
  1392. int streamId = (int)ReadSLEB128();
  1393. auto& result = mResults[streamId];
  1394. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMBasicBlock);
  1395. llvmBlock = (llvm::BasicBlock*)result.mLLVMType;
  1396. }
  1397. void BfIRCodeGen::Read(llvm::MDNode*& llvmMD)
  1398. {
  1399. int streamId = (int)ReadSLEB128();
  1400. if (streamId == -1)
  1401. {
  1402. llvmMD = NULL;
  1403. return;
  1404. }
  1405. auto& result = mResults[streamId];
  1406. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMMetadata);
  1407. llvmMD = result.mLLVMMetadata;
  1408. }
  1409. void BfIRCodeGen::Read(llvm::Metadata*& llvmMD)
  1410. {
  1411. int streamId = (int)ReadSLEB128();
  1412. if (streamId == -1)
  1413. {
  1414. llvmMD = NULL;
  1415. return;
  1416. }
  1417. auto& result = mResults[streamId];
  1418. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMMetadata);
  1419. llvmMD = result.mLLVMMetadata;
  1420. }
  1421. void BfIRCodeGen::AddNop()
  1422. {
  1423. if ((mTargetTriple.GetMachineType() != BfMachineType_x86) && (mTargetTriple.GetMachineType() != BfMachineType_x64))
  1424. return;
  1425. if (mNopInlineAsm == NULL)
  1426. {
  1427. llvm::SmallVector<llvm::Type*, 8> paramTypes;
  1428. llvm::FunctionType* funcType = llvm::FunctionType::get(llvm::Type::getVoidTy(*mLLVMContext), paramTypes, false);
  1429. mNopInlineAsm = llvm::InlineAsm::get(funcType,
  1430. "nop", "", true, false, llvm::InlineAsm::AD_ATT);
  1431. }
  1432. llvm::CallInst* callInst = mIRBuilder->CreateCall(mNopInlineAsm);
  1433. callInst->addFnAttr(llvm::Attribute::NoUnwind);
  1434. }
  1435. llvm::Value* BfIRCodeGen::TryToVector(const BfIRTypedValue& value)
  1436. {
  1437. auto valueType = value.mTypeEx->mLLVMType;
  1438. if (llvm::isa<llvm::VectorType>(valueType))
  1439. return value.mValue;
  1440. if (auto ptrType = llvm::dyn_cast<llvm::PointerType>(valueType))
  1441. {
  1442. auto ptrElemType = GetTypeMember(value.mTypeEx, 0);
  1443. if (auto arrType = llvm::dyn_cast<llvm::ArrayType>(ptrElemType->mLLVMType))
  1444. {
  1445. auto vecType = llvm::FixedVectorType::get(arrType->getArrayElementType(), (uint)arrType->getArrayNumElements());
  1446. auto vecPtrType = vecType->getPointerTo();
  1447. auto ptrVal0 = mIRBuilder->CreateBitCast(value.mValue, vecPtrType);
  1448. return mIRBuilder->CreateAlignedLoad(vecType, ptrVal0, llvm::MaybeAlign(1));
  1449. }
  1450. if (auto vecType = llvm::dyn_cast<llvm::VectorType>(ptrElemType->mLLVMType))
  1451. {
  1452. return mIRBuilder->CreateAlignedLoad(vecType, value.mValue, llvm::MaybeAlign(1));
  1453. }
  1454. }
  1455. return NULL;
  1456. }
  1457. bool BfIRCodeGen::TryMemCpy(const BfIRTypedValue& ptr, llvm::Value* val)
  1458. {
  1459. auto valType = val->getType();
  1460. auto dataLayout = llvm::DataLayout(mLLVMModule);
  1461. int arrayBytes = (int)dataLayout.getTypeSizeInBits(valType) / 8;
  1462. // LLVM has perf issues with large aggregates - it treats each element as a unique value,
  1463. // which is great for optimizing small data but is a perf killer for large data.
  1464. if (arrayBytes < 256)
  1465. return false;
  1466. auto int8Ty = llvm::Type::getInt8Ty(*mLLVMContext);
  1467. auto int32Ty = llvm::Type::getInt32Ty(*mLLVMContext);
  1468. auto int8PtrTy = int8Ty->getPointerTo();
  1469. if (auto loadInst = llvm::dyn_cast<llvm::LoadInst>(val))
  1470. {
  1471. mIRBuilder->CreateMemCpy(
  1472. mIRBuilder->CreateBitCast(ptr.mValue, int8PtrTy),
  1473. llvm::MaybeAlign(1),
  1474. mIRBuilder->CreateBitCast(loadInst->getPointerOperand(), int8PtrTy),
  1475. llvm::MaybeAlign(1),
  1476. llvm::ConstantInt::get(int32Ty, arrayBytes));
  1477. return true;
  1478. }
  1479. auto constVal = llvm::dyn_cast<llvm::Constant>(val);
  1480. if (constVal == NULL)
  1481. return false;
  1482. if (llvm::isa<llvm::ConstantAggregateZero>(constVal))
  1483. {
  1484. mIRBuilder->CreateMemSet(
  1485. mIRBuilder->CreateBitCast(ptr.mValue, int8PtrTy),
  1486. llvm::ConstantInt::get(int8Ty, 0),
  1487. llvm::ConstantInt::get(int32Ty, arrayBytes),
  1488. llvm::MaybeAlign(1));
  1489. return true;
  1490. }
  1491. auto globalVariable = new llvm::GlobalVariable(
  1492. *mLLVMModule,
  1493. valType,
  1494. true,
  1495. llvm::GlobalValue::InternalLinkage,
  1496. constVal,
  1497. StrFormat("__ConstVal__%d", mConstValIdx++).c_str(),
  1498. NULL,
  1499. llvm::GlobalValue::NotThreadLocal);
  1500. mIRBuilder->CreateMemCpy(
  1501. mIRBuilder->CreateBitCast(ptr.mValue, int8PtrTy),
  1502. llvm::MaybeAlign(1),
  1503. mIRBuilder->CreateBitCast(globalVariable, int8PtrTy),
  1504. llvm::MaybeAlign(1),
  1505. llvm::ConstantInt::get(int32Ty, arrayBytes));
  1506. return true;
  1507. }
  1508. bool BfIRCodeGen::TryVectorCpy(const BfIRTypedValue& ptr, llvm::Value* val)
  1509. {
  1510. if (GetTypeMember(ptr.mTypeEx, 0)->mLLVMType == val->getType())
  1511. return false;
  1512. if (!llvm::isa<llvm::VectorType>(val->getType()))
  1513. {
  1514. return false;
  1515. }
  1516. auto usePtr = mIRBuilder->CreateBitCast(ptr.mValue, val->getType()->getPointerTo());
  1517. mIRBuilder->CreateAlignedStore(val, usePtr, llvm::MaybeAlign(1));
  1518. return true;
  1519. }
  1520. llvm::Type* BfIRCodeGen::GetLLVMPointerElementType(BfIRTypeEx* typeEx)
  1521. {
  1522. BF_ASSERT(typeEx != NULL);
  1523. BF_ASSERT(typeEx->mLLVMType->isPointerTy());
  1524. return GetTypeMember(typeEx, 0)->mLLVMType;
  1525. }
  1526. BfIRTypeEx* BfIRCodeGen::GetSizeAlignedType(BfIRTypeEntry* typeEntry)
  1527. {
  1528. if ((typeEntry->mAlignType == NULL) && ((typeEntry->mSize & (typeEntry->mAlign - 1)) != 0))
  1529. {
  1530. auto structType = llvm::dyn_cast<llvm::StructType>(typeEntry->mType->mLLVMType);
  1531. if (structType != NULL)
  1532. {
  1533. //TODO: Fill out properly
  1534. BF_ASSERT(structType->isPacked());
  1535. auto alignTypeEx = new BfIRTypeEx();
  1536. mIRTypeExs.Add(alignTypeEx);
  1537. auto alignType = llvm::StructType::create(*mLLVMContext, (structType->getName().str() + "_ALIGNED").c_str());
  1538. llvm::SmallVector<llvm::Type*, 8> members;
  1539. for (int elemIdx = 0; elemIdx < (int)structType->getNumElements(); elemIdx++)
  1540. {
  1541. members.push_back(structType->getElementType(elemIdx));
  1542. }
  1543. int alignSize = BF_ALIGN(typeEntry->mSize, typeEntry->mAlign);
  1544. int fillSize = alignSize - typeEntry->mSize;
  1545. members.push_back(llvm::ArrayType::get(llvm::Type::getInt8Ty(*mLLVMContext), fillSize));
  1546. alignType->setBody(members, structType->isPacked());
  1547. alignTypeEx->mLLVMType = alignType;
  1548. typeEntry->mAlignType = alignTypeEx;
  1549. mAlignedTypeToNormalType[alignTypeEx] = typeEntry->mType;
  1550. }
  1551. }
  1552. if (typeEntry->mAlignType != NULL)
  1553. return typeEntry->mAlignType;
  1554. return typeEntry->mType;
  1555. }
  1556. BfIRTypedValue BfIRCodeGen::GetAlignedPtr(const BfIRTypedValue& val)
  1557. {
  1558. BfIRTypedValue result = val;
  1559. if (auto ptrType = llvm::dyn_cast<llvm::PointerType>(val.mTypeEx->mLLVMType))
  1560. {
  1561. auto elemType = GetTypeMember(val.mTypeEx, 0);
  1562. auto typeEntry = GetTypeEntry(elemType);
  1563. if (typeEntry != NULL)
  1564. {
  1565. auto alignedType = GetSizeAlignedType(typeEntry);
  1566. if (alignedType != elemType)
  1567. result.mTypeEx = GetPointerTypeEx(alignedType);
  1568. }
  1569. }
  1570. return result;
  1571. }
  1572. llvm::Value* BfIRCodeGen::DoCheckedIntrinsic(llvm::Intrinsic::ID intrin, llvm::Value* lhs, llvm::Value* rhs, bool useAsm)
  1573. {
  1574. if ((mTargetTriple.GetMachineType() != BfMachineType_x86) && (mTargetTriple.GetMachineType() != BfMachineType_x64))
  1575. useAsm = false;
  1576. CmdParamVec<llvm::Type*> useParams;
  1577. useParams.push_back(lhs->getType());
  1578. auto func = llvm::Intrinsic::getDeclaration(mLLVMModule, intrin, useParams);
  1579. CmdParamVec<llvm::Value*> args;
  1580. args.push_back(lhs);
  1581. args.push_back(rhs);
  1582. llvm::FunctionType* funcType = func->getFunctionType();
  1583. auto aggResult = mIRBuilder->CreateCall(funcType, func, args);
  1584. auto valResult = mIRBuilder->CreateExtractValue(aggResult, 0);
  1585. auto failResult = mIRBuilder->CreateExtractValue(aggResult, 1);
  1586. if (!useAsm)
  1587. {
  1588. mLockedBlocks.Add(mIRBuilder->GetInsertBlock());
  1589. auto failBB = llvm::BasicBlock::Create(*mLLVMContext, "access.fail");
  1590. auto passBB = llvm::BasicBlock::Create(*mLLVMContext, "access.pass");
  1591. mIRBuilder->CreateCondBr(failResult, failBB, passBB);
  1592. mActiveFunction->insert(mActiveFunction->end(), failBB);
  1593. mIRBuilder->SetInsertPoint(failBB);
  1594. auto trapDecl = llvm::Intrinsic::getDeclaration(mLLVMModule, llvm::Intrinsic::trap);
  1595. auto callInst = mIRBuilder->CreateCall(trapDecl);
  1596. callInst->addFnAttr(llvm::Attribute::NoReturn);
  1597. mIRBuilder->CreateBr(passBB);
  1598. mActiveFunction->insert(mActiveFunction->end(), passBB);
  1599. mIRBuilder->SetInsertPoint(passBB);
  1600. }
  1601. else
  1602. {
  1603. if (mOverflowCheckAsm == NULL)
  1604. {
  1605. std::vector<llvm::Type*> paramTypes;
  1606. paramTypes.push_back(llvm::Type::getInt8Ty(*mLLVMContext));
  1607. auto funcType = llvm::FunctionType::get(llvm::Type::getVoidTy(*mLLVMContext), paramTypes, false);
  1608. String asmStr =
  1609. "testb $$1, $0\n"
  1610. "jz 1f\n"
  1611. "int $$3\n"
  1612. "1:";
  1613. mOverflowCheckAsm = llvm::InlineAsm::get(funcType,
  1614. asmStr.c_str(), "r,~{dirflag},~{fpsr},~{flags}", true,
  1615. false, llvm::InlineAsm::AD_ATT);
  1616. }
  1617. llvm::SmallVector<llvm::Value*, 1> llvmArgs;
  1618. llvmArgs.push_back(mIRBuilder->CreateIntCast(failResult, llvm::Type::getInt8Ty(*mLLVMContext), false));
  1619. llvm::CallInst* callInst = mIRBuilder->CreateCall(mOverflowCheckAsm, llvmArgs);
  1620. callInst->addFnAttr(llvm::Attribute::NoUnwind);
  1621. }
  1622. return valResult;
  1623. }
  1624. void BfIRCodeGen::CreateMemSet(llvm::Value* addr, llvm::Value* val, llvm::Value* size, int alignment, bool isVolatile)
  1625. {
  1626. auto sizeConst = llvm::dyn_cast<llvm::ConstantInt>(size);
  1627. auto valConst = llvm::dyn_cast<llvm::ConstantInt>(val);
  1628. if ((!mIsOptimized) && (sizeConst != NULL) && (valConst != NULL))
  1629. {
  1630. int64 sizeVal = sizeConst->getSExtValue();
  1631. uint8 setVal = (uint8)valConst->getSExtValue();
  1632. if (sizeVal <= 128)
  1633. {
  1634. //llvm::Value* intVal = mIRBuilder->CreatePtrToInt(addr, llvm::Type::getInt32Ty(*mLLVMContext))
  1635. int curOffset = 0;
  1636. int sizeLeft = (int)sizeVal;
  1637. llvm::Value* headVal;
  1638. if (mPtrSize >= 8)
  1639. {
  1640. headVal = NULL;
  1641. auto intTy = llvm::Type::getInt64Ty(*mLLVMContext);
  1642. auto constVal = llvm::ConstantInt::get(intTy,
  1643. ((int64)setVal << 56) | ((int64)setVal << 48) | ((int64)setVal << 40) | ((int64)setVal << 32) |
  1644. ((int64)setVal << 24) | ((int64)setVal << 16) | ((int64)setVal << 8) | ((int64)setVal));
  1645. while (sizeLeft >= 8)
  1646. {
  1647. if (headVal == NULL)
  1648. headVal = mIRBuilder->CreateBitCast(addr, intTy->getPointerTo());
  1649. llvm::Value* ptrVal = headVal;
  1650. if (curOffset != 0)
  1651. ptrVal = mIRBuilder->CreateConstInBoundsGEP1_32(intTy, headVal, curOffset / 8);
  1652. mIRBuilder->CreateStore(constVal, ptrVal, isVolatile);
  1653. curOffset += 8;
  1654. sizeLeft -= 8;
  1655. }
  1656. }
  1657. if (sizeLeft >= 4)
  1658. {
  1659. headVal = NULL;
  1660. auto intTy = llvm::Type::getInt32Ty(*mLLVMContext);
  1661. auto constVal = llvm::ConstantInt::get(intTy, ((int)setVal << 24) | ((int)setVal << 16) | ((int)setVal << 8) | ((int)setVal));
  1662. while (sizeLeft >= 4)
  1663. {
  1664. if (headVal == NULL)
  1665. headVal = mIRBuilder->CreateBitCast(addr, intTy->getPointerTo());
  1666. llvm::Value* ptrVal = headVal;
  1667. if (curOffset != 0)
  1668. ptrVal = mIRBuilder->CreateConstInBoundsGEP1_32(intTy, headVal, curOffset / 4);
  1669. mIRBuilder->CreateStore(constVal, ptrVal, isVolatile);
  1670. curOffset += 4;
  1671. sizeLeft -= 4;
  1672. }
  1673. }
  1674. if (sizeLeft >= 2)
  1675. {
  1676. headVal = NULL;
  1677. auto intTy = llvm::Type::getInt16Ty(*mLLVMContext);
  1678. auto constVal = llvm::ConstantInt::get(intTy, ((int)setVal << 8) | ((int)setVal));
  1679. while (sizeLeft >= 2)
  1680. {
  1681. if (headVal == NULL)
  1682. headVal = mIRBuilder->CreateBitCast(addr, intTy->getPointerTo());
  1683. llvm::Value* ptrVal = headVal;
  1684. if (curOffset != 0)
  1685. ptrVal = mIRBuilder->CreateConstInBoundsGEP1_32(intTy, headVal, curOffset / 2);
  1686. mIRBuilder->CreateStore(constVal, ptrVal, isVolatile);
  1687. curOffset += 2;
  1688. sizeLeft -= 2;
  1689. }
  1690. }
  1691. if (sizeLeft >= 1)
  1692. {
  1693. headVal = NULL;
  1694. auto intTy = llvm::Type::getInt8Ty(*mLLVMContext);
  1695. auto constVal = llvm::ConstantInt::get(intTy, ((int)setVal));
  1696. while (sizeLeft >= 1)
  1697. {
  1698. if (headVal == NULL)
  1699. headVal = mIRBuilder->CreateBitCast(addr, intTy->getPointerTo());
  1700. llvm::Value* ptrVal = headVal;
  1701. if (curOffset != 0)
  1702. ptrVal = mIRBuilder->CreateConstInBoundsGEP1_32(intTy, headVal, curOffset / 1);
  1703. mIRBuilder->CreateStore(constVal, ptrVal, isVolatile);
  1704. curOffset += 1;
  1705. sizeLeft -= 1;
  1706. }
  1707. }
  1708. return;
  1709. }
  1710. }
  1711. mIRBuilder->CreateMemSet(addr, val, size, llvm::MaybeAlign(alignment), isVolatile);
  1712. }
  1713. void BfIRCodeGen::InitTarget()
  1714. {
  1715. llvm::SMDiagnostic Err;
  1716. llvm::Triple theTriple = llvm::Triple(mLLVMModule->getTargetTriple());
  1717. llvm::CodeGenOptLevel optLvl = llvm::CodeGenOptLevel::None;
  1718. String cpuName = mTargetCPU;
  1719. String arch = "";
  1720. // Get the target specific parser.
  1721. std::string Error;
  1722. const llvm::Target *theTarget = llvm::TargetRegistry::lookupTarget(arch.c_str(), theTriple, Error);
  1723. if (!theTarget)
  1724. {
  1725. Fail(StrFormat("Failed to create LLVM Target: %s", Error.c_str()));
  1726. return;
  1727. }
  1728. llvm::TargetOptions Options = llvm::TargetOptions(); // InitTargetOptionsFromCodeGenFlags();
  1729. String featuresStr;
  1730. if (mCodeGenOptions.mOptLevel == BfOptLevel_O1)
  1731. {
  1732. //optLvl = CodeGenOpt::Less;
  1733. }
  1734. else if (mCodeGenOptions.mOptLevel == BfOptLevel_O2)
  1735. optLvl = llvm::CodeGenOptLevel::Default;
  1736. else if (mCodeGenOptions.mOptLevel == BfOptLevel_O3)
  1737. optLvl = llvm::CodeGenOptLevel::Aggressive;
  1738. if (theTriple.isWasm())
  1739. featuresStr = "+atomics,+bulk-memory,+mutable-globals,+sign-ext";
  1740. else if (mCodeGenOptions.mSIMDSetting == BfSIMDSetting_SSE)
  1741. featuresStr = "+sse";
  1742. else if (mCodeGenOptions.mSIMDSetting == BfSIMDSetting_SSE2)
  1743. featuresStr = "+sse2";
  1744. else if (mCodeGenOptions.mSIMDSetting == BfSIMDSetting_SSE3)
  1745. featuresStr = "+sse3";
  1746. else if (mCodeGenOptions.mSIMDSetting == BfSIMDSetting_SSE4)
  1747. featuresStr = "+sse4";
  1748. else if (mCodeGenOptions.mSIMDSetting == BfSIMDSetting_SSE41)
  1749. featuresStr = "+sse4.1";
  1750. else if (mCodeGenOptions.mSIMDSetting == BfSIMDSetting_AVX)
  1751. featuresStr = "+avx";
  1752. else if (mCodeGenOptions.mSIMDSetting == BfSIMDSetting_AVX2)
  1753. featuresStr = "+avx2";
  1754. std::optional<llvm::Reloc::Model> relocModel;
  1755. llvm::CodeModel::Model cmModel = llvm::CodeModel::Small;
  1756. switch (mCodeGenOptions.mRelocType)
  1757. {
  1758. case BfRelocType_Static:
  1759. relocModel = llvm::Reloc::Model::DynamicNoPIC;
  1760. break;
  1761. case BfRelocType_PIC:
  1762. relocModel = llvm::Reloc::Model::PIC_;
  1763. break;
  1764. case BfRelocType_DynamicNoPIC:
  1765. relocModel = llvm::Reloc::Model::DynamicNoPIC;
  1766. break;
  1767. case BfRelocType_ROPI:
  1768. relocModel = llvm::Reloc::Model::ROPI;
  1769. break;
  1770. case BfRelocType_RWPI:
  1771. relocModel = llvm::Reloc::Model::RWPI;
  1772. break;
  1773. case BfRelocType_ROPI_RWPI:
  1774. relocModel = llvm::Reloc::Model::ROPI_RWPI;
  1775. break;
  1776. default: break;
  1777. }
  1778. switch (mCodeGenOptions.mPICLevel)
  1779. {
  1780. case BfPICLevel_Not:
  1781. mLLVMModule->setPICLevel(llvm::PICLevel::Level::NotPIC);
  1782. break;
  1783. case BfPICLevel_Small:
  1784. mLLVMModule->setPICLevel(llvm::PICLevel::Level::SmallPIC);
  1785. break;
  1786. case BfPICLevel_Big:
  1787. mLLVMModule->setPICLevel(llvm::PICLevel::Level::BigPIC);
  1788. break;
  1789. default: break;
  1790. }
  1791. mLLVMTargetMachine =
  1792. theTarget->createTargetMachine(theTriple.getTriple(), cpuName.c_str(), featuresStr.c_str(),
  1793. Options, relocModel, cmModel, optLvl);
  1794. mLLVMModule->setDataLayout(mLLVMTargetMachine->createDataLayout());
  1795. }
  1796. void BfIRCodeGen::HandleNextCmd()
  1797. {
  1798. if (mFailed)
  1799. return;
  1800. int curId = mCmdCount;
  1801. BfIRCmd cmd = (BfIRCmd)mStream->Read();
  1802. mCmdCount++;
  1803. switch (cmd)
  1804. {
  1805. case BfIRCmd_Module_Start:
  1806. {
  1807. CMD_PARAM(String, moduleName);
  1808. CMD_PARAM(int, ptrSize);
  1809. CMD_PARAM(bool, isOptimized);
  1810. BF_ASSERT(mLLVMModule == NULL);
  1811. mModuleName = moduleName;
  1812. mPtrSize = ptrSize;
  1813. mIsOptimized = isOptimized;
  1814. mLLVMModule = new llvm::Module(moduleName.c_str(), *mLLVMContext);
  1815. mLLVMModule->setIsNewDbgInfoFormat(false);
  1816. mIRBuilder = new llvm::IRBuilder<>(*mLLVMContext);
  1817. //OutputDebugStrF("-------- Starting Module %s --------\n", moduleName.c_str());
  1818. }
  1819. break;
  1820. case BfIRCmd_Module_SetTargetTriple:
  1821. {
  1822. CMD_PARAM(String, targetTriple);
  1823. CMD_PARAM(String, targetCPU);
  1824. mTargetTriple.Set(targetTriple);
  1825. mTargetCPU = targetCPU;
  1826. if (targetTriple.IsEmpty())
  1827. mLLVMModule->setTargetTriple(llvm::sys::getDefaultTargetTriple());
  1828. else
  1829. mLLVMModule->setTargetTriple(targetTriple.c_str());
  1830. InitTarget();
  1831. }
  1832. break;
  1833. case BfIRCmd_Module_AddModuleFlag:
  1834. {
  1835. CMD_PARAM(String, flag);
  1836. CMD_PARAM(int, val);
  1837. mLLVMModule->addModuleFlag(llvm::Module::Warning, flag.c_str(), val);
  1838. if (flag == "CodeView")
  1839. mIsCodeView = true;
  1840. }
  1841. break;
  1842. case BfIRCmd_WriteIR:
  1843. {
  1844. CMD_PARAM(String, fileName);
  1845. std::error_code ec;
  1846. llvm::raw_fd_ostream outStream(fileName.c_str(), ec, llvm::sys::fs::OpenFlags::OF_Text);
  1847. if (ec)
  1848. {
  1849. Fail("Failed writing IR '" + fileName + "': " + ec.message());
  1850. }
  1851. else
  1852. mLLVMModule->print(outStream, NULL);
  1853. }
  1854. break;
  1855. case BfIRCmd_Abort:
  1856. Fail("Stream aborted");
  1857. break;
  1858. case BfIRCmd_SetType:
  1859. {
  1860. CMD_PARAM(int, typeId);
  1861. CMD_PARAM(BfIRTypeEx*, type);
  1862. //llvm::Type* type;
  1863. //llvm::Type* elementType;
  1864. auto& typeEntry = GetTypeEntry(typeId);
  1865. typeEntry.mType = type;
  1866. if (typeEntry.mInstType == NULL)
  1867. typeEntry.mInstType = type;
  1868. mTypeToTypeIdMap[type] = typeId;
  1869. }
  1870. break;
  1871. case BfIRCmd_SetInstType:
  1872. {
  1873. CMD_PARAM(int, typeId);
  1874. CMD_PARAM(BfIRTypeEx*, type);
  1875. GetTypeEntry(typeId).mInstType = type;
  1876. }
  1877. break;
  1878. case BfIRCmd_PrimitiveType:
  1879. {
  1880. BfTypeCode typeCode = (BfTypeCode)mStream->Read();
  1881. bool isSigned;
  1882. SetResult(curId, GetLLVMType(typeCode, isSigned));
  1883. }
  1884. break;
  1885. case BfIRCmd_CreateAnonymousStruct:
  1886. {
  1887. CMD_PARAM(CmdParamVec<BfIRTypeEx*>, members);
  1888. CmdParamVec<llvm::Type*> llvmMembers;
  1889. for (auto& memberType : members)
  1890. llvmMembers.push_back(memberType->mLLVMType);
  1891. auto structType = llvm::StructType::get(*mLLVMContext, llvmMembers);
  1892. auto typeEx = CreateTypeEx(structType);
  1893. for (auto& memberType : members)
  1894. {
  1895. BF_ASSERT(memberType != NULL);
  1896. typeEx->mMembers.Add(memberType);
  1897. }
  1898. SetResult(curId, typeEx);
  1899. }
  1900. break;
  1901. case BfIRCmd_CreateStruct:
  1902. {
  1903. CMD_PARAM(String, typeName);
  1904. auto structType = llvm::StructType::create(*mLLVMContext, typeName.c_str());
  1905. auto typeEx = CreateTypeEx(structType);
  1906. SetResult(curId, typeEx);
  1907. }
  1908. break;
  1909. case BfIRCmd_StructSetBody:
  1910. {
  1911. BfIRTypeEx* typeEx = NULL;
  1912. BfIRTypeEntry* typeEntry = NULL;
  1913. Read(typeEx, &typeEntry);
  1914. CMD_PARAM(CmdParamVec<BfIRTypeEx*>, members);
  1915. CMD_PARAM(int, instSize);
  1916. CMD_PARAM(int, instAlign);
  1917. CMD_PARAM(bool, isPacked);
  1918. typeEx->mMembers.clear();
  1919. auto type = typeEx->mLLVMType;
  1920. CmdParamVec<llvm::Type*> llvmMembers;
  1921. for (auto& memberType : members)
  1922. {
  1923. BF_ASSERT(memberType != NULL);
  1924. typeEx->mMembers.Add(memberType);
  1925. llvmMembers.push_back(memberType->mLLVMType);
  1926. }
  1927. BF_ASSERT(llvm::isa<llvm::StructType>(type));
  1928. auto structType = (llvm::StructType*)type;
  1929. if (structType->isOpaque())
  1930. structType->setBody(llvmMembers, isPacked);
  1931. if (typeEntry != NULL)
  1932. {
  1933. typeEntry->mSize = instSize;
  1934. typeEntry->mAlign = instAlign;
  1935. }
  1936. }
  1937. break;
  1938. case BfIRCmd_Type:
  1939. {
  1940. CMD_PARAM(BfIRTypeEntry*, typeEntry);
  1941. auto type = typeEntry->mType;
  1942. SetResult(curId, type);
  1943. }
  1944. break;
  1945. case BfIRCmd_TypeInst:
  1946. {
  1947. CMD_PARAM(BfIRTypeEntry*, typeEntry);
  1948. SetResult(curId, typeEntry->mInstType);
  1949. }
  1950. break;
  1951. case BfIRCmd_TypeInstPtr:
  1952. {
  1953. CMD_PARAM(BfIRTypeEntry*, typeEntry);
  1954. SetResult(curId, GetPointerTypeEx(typeEntry->mInstType));
  1955. }
  1956. break;
  1957. case BfIRCmd_GetType:
  1958. {
  1959. CMD_PARAM(BfIRTypedValue, typedValue);
  1960. BF_ASSERT(typedValue.mTypeEx != NULL);
  1961. SetResult(curId, typedValue.mTypeEx);
  1962. }
  1963. break;
  1964. case BfIRCmd_GetPointerToFuncType:
  1965. {
  1966. BfIRTypeEx* funcType = NULL;
  1967. ReadFunctionType(funcType);
  1968. SetResult(curId, GetPointerTypeEx(funcType));
  1969. }
  1970. break;
  1971. case BfIRCmd_GetPointerToType:
  1972. {
  1973. CMD_PARAM(BfIRTypeEx*, type);
  1974. SetResult(curId, GetPointerTypeEx(type));
  1975. }
  1976. break;
  1977. case BfIRCmd_GetSizedArrayType:
  1978. {
  1979. BfIRTypeEx* elementType = NULL;
  1980. BfIRTypeEntry* elementTypeEntry = NULL;
  1981. Read(elementType, &elementTypeEntry);
  1982. auto typeEx = new BfIRTypeEx();
  1983. typeEx->mMembers.Add(elementType);
  1984. mIRTypeExs.Add(typeEx);
  1985. CMD_PARAM(int, length);
  1986. if (elementTypeEntry != NULL)
  1987. typeEx->mLLVMType = llvm::ArrayType::get(GetSizeAlignedType(elementTypeEntry)->mLLVMType, length);
  1988. else
  1989. typeEx->mLLVMType = llvm::ArrayType::get(elementType->mLLVMType, length);
  1990. SetResult(curId, typeEx);
  1991. }
  1992. break;
  1993. case BfIRCmd_GetVectorType:
  1994. {
  1995. CMD_PARAM(BfIRTypeEx*, elementType);
  1996. CMD_PARAM(int, length);
  1997. auto llvmType = elementType->mLLVMType;
  1998. if (llvmType->getScalarSizeInBits() == 1)
  1999. {
  2000. llvmType = llvm::Type::getInt8Ty(*mLLVMContext);
  2001. }
  2002. auto typeEx = new BfIRTypeEx();
  2003. mIRTypeExs.Add(typeEx);
  2004. typeEx->mLLVMType = llvm::FixedVectorType::get(llvmType, length);
  2005. typeEx->mMembers.Add(elementType);
  2006. SetResult(curId, typeEx);
  2007. }
  2008. break;
  2009. case BfIRCmd_CreateConstAgg:
  2010. {
  2011. CMD_PARAM(BfIRTypeEx*, type);
  2012. CMD_PARAM(CmdParamVec<llvm::Value*>, values)
  2013. llvm::SmallVector<llvm::Constant*, 8> copyValues;
  2014. if (auto arrayType = llvm::dyn_cast<llvm::ArrayType>(type->mLLVMType))
  2015. {
  2016. for (auto val : values)
  2017. {
  2018. auto constValue = llvm::dyn_cast<llvm::Constant>(val);
  2019. BF_ASSERT(constValue != NULL);
  2020. copyValues.push_back(constValue);
  2021. }
  2022. int fillCount = (int)(arrayType->getNumElements() - copyValues.size());
  2023. if (fillCount > 0)
  2024. {
  2025. auto lastValue = copyValues.back();
  2026. for (int i = 0; i < fillCount; i++)
  2027. copyValues.push_back(lastValue);
  2028. }
  2029. BfIRTypedValue result;
  2030. result.mTypeEx = type;
  2031. result.mValue = llvm::ConstantArray::get(arrayType, copyValues);
  2032. SetResult(curId, result);
  2033. }
  2034. else if (auto structType = llvm::dyn_cast<llvm::StructType>(type->mLLVMType))
  2035. {
  2036. FixValues(structType, values);
  2037. for (auto val : values)
  2038. {
  2039. auto constValue = llvm::dyn_cast<llvm::Constant>(val);
  2040. BF_ASSERT(constValue != NULL);
  2041. copyValues.push_back(constValue);
  2042. }
  2043. BfIRTypedValue result;
  2044. result.mTypeEx = type;
  2045. result.mValue = llvm::ConstantStruct::get(structType, copyValues);
  2046. SetResult(curId, result);
  2047. }
  2048. else
  2049. Fail("Bad type");
  2050. }
  2051. break;
  2052. case BfIRCmd_CreateConstStructZero:
  2053. {
  2054. CMD_PARAM(BfIRTypeEx*, type);
  2055. BfIRTypedValue result;
  2056. result.mTypeEx = type;
  2057. result.mValue = llvm::ConstantAggregateZero::get(type->mLLVMType);
  2058. SetResult(curId, result);
  2059. }
  2060. break;
  2061. case BfIRCmd_CreateConstString:
  2062. {
  2063. CMD_PARAM(String, str);
  2064. BfIRTypedValue result;
  2065. result.mValue = llvm::ConstantDataArray::getString(*mLLVMContext, llvm::StringRef(str.c_str(), str.length()));
  2066. result.mTypeEx = GetTypeEx(result.mValue->getType());
  2067. SetResult(curId, result);
  2068. }
  2069. break;
  2070. case BfIRCmd_ConfigConst:
  2071. {
  2072. CMD_PARAM(int, constIdx);
  2073. BfTypeCode typeCode = (BfTypeCode)mStream->Read();
  2074. if (typeCode == BfTypeCode_IntPtr)
  2075. typeCode = (mPtrSize == 4) ? BfTypeCode_Int32 : BfTypeCode_Int64;
  2076. llvm::Constant* constVal = (typeCode == BfTypeCode_Int32) ?
  2077. mConfigConsts32[constIdx] :
  2078. mConfigConsts64[constIdx];
  2079. SetResult(curId, constVal);
  2080. }
  2081. break;
  2082. case BfIRCmd_SetName:
  2083. {
  2084. CMD_PARAM_NOTRANS(llvm::Value*, val);
  2085. CMD_PARAM(String, name);
  2086. if (!val->getType()->isVoidTy())
  2087. val->setName(name.c_str());
  2088. }
  2089. break;
  2090. case BfIRCmd_CreateUndefValue:
  2091. {
  2092. CMD_PARAM(BfIRTypeEx*, type);
  2093. BfIRTypedValue result;
  2094. result.mTypeEx = type;
  2095. result.mValue = llvm::UndefValue::get(type->mLLVMType);
  2096. SetResult(curId, result);
  2097. }
  2098. break;
  2099. case BfIRCmd_NumericCast:
  2100. {
  2101. CMD_PARAM(llvm::Value*, val);
  2102. CMD_PARAM(bool, valIsSigned);
  2103. BfTypeCode typeCode = (BfTypeCode)mStream->Read();
  2104. BfTypeCode valTypeCode = GetTypeCode(val->getType(), valIsSigned);
  2105. bool toSigned;
  2106. auto toLLVMType = GetLLVMType(typeCode, toSigned);
  2107. llvm::Value* retVal = NULL;
  2108. if (BfIRBuilder::IsIntable(typeCode))
  2109. {
  2110. // Int -> Int
  2111. if ((BfIRBuilder::IsIntable(valTypeCode)) || (valTypeCode == BfTypeCode_Boolean))
  2112. {
  2113. retVal = mIRBuilder->CreateIntCast(val, toLLVMType, toSigned && valIsSigned);
  2114. }
  2115. else // Float -> Int
  2116. {
  2117. if (BfIRBuilder::IsSigned(typeCode))
  2118. retVal = mIRBuilder->CreateFPToSI(val, toLLVMType);
  2119. else
  2120. retVal = mIRBuilder->CreateFPToUI(val, toLLVMType);
  2121. }
  2122. }
  2123. else
  2124. {
  2125. // Int -> Float
  2126. if ((BfIRBuilder::IsIntable(valTypeCode)) || (valTypeCode == BfTypeCode_Boolean))
  2127. {
  2128. if (BfIRBuilder::IsSigned(valTypeCode))
  2129. retVal = mIRBuilder->CreateSIToFP(val, toLLVMType);
  2130. else
  2131. retVal = mIRBuilder->CreateUIToFP(val, toLLVMType);
  2132. }
  2133. else // Float -> Float
  2134. {
  2135. retVal = mIRBuilder->CreateFPCast(val, toLLVMType);
  2136. }
  2137. }
  2138. SetResult(curId, retVal);
  2139. }
  2140. break;
  2141. case BfIRCmd_CmpEQ:
  2142. {
  2143. CMD_PARAM(llvm::Value*, lhs);
  2144. CMD_PARAM(llvm::Value*, rhs);
  2145. if (lhs->getType()->isFloatingPointTy())
  2146. SetResult(curId, mIRBuilder->CreateFCmpOEQ(lhs, rhs));
  2147. else
  2148. SetResult(curId, mIRBuilder->CreateICmpEQ(lhs, rhs));
  2149. }
  2150. break;
  2151. case BfIRCmd_CmpNE:
  2152. {
  2153. CMD_PARAM(llvm::Value*, lhs);
  2154. CMD_PARAM(llvm::Value*, rhs);
  2155. if (lhs->getType()->isFloatingPointTy())
  2156. SetResult(curId, mIRBuilder->CreateFCmpUNE(lhs, rhs));
  2157. else
  2158. SetResult(curId, mIRBuilder->CreateICmpNE(lhs, rhs));
  2159. }
  2160. break;
  2161. case BfIRCmd_CmpSLT:
  2162. {
  2163. CMD_PARAM(llvm::Value*, lhs);
  2164. CMD_PARAM(llvm::Value*, rhs);
  2165. if (lhs->getType()->isFloatingPointTy())
  2166. SetResult(curId, mIRBuilder->CreateFCmpOLT(lhs, rhs));
  2167. else
  2168. SetResult(curId, mIRBuilder->CreateICmpSLT(lhs, rhs));
  2169. }
  2170. break;
  2171. case BfIRCmd_CmpULT:
  2172. {
  2173. CMD_PARAM(llvm::Value*, lhs);
  2174. CMD_PARAM(llvm::Value*, rhs);
  2175. if (lhs->getType()->isFloatingPointTy())
  2176. SetResult(curId, mIRBuilder->CreateFCmpOLT(lhs, rhs));
  2177. else
  2178. SetResult(curId, mIRBuilder->CreateICmpULT(lhs, rhs));
  2179. }
  2180. break;
  2181. case BfIRCmd_CmpSLE:
  2182. {
  2183. CMD_PARAM(llvm::Value*, lhs);
  2184. CMD_PARAM(llvm::Value*, rhs);
  2185. if (lhs->getType()->isFloatingPointTy())
  2186. SetResult(curId, mIRBuilder->CreateFCmpOLE(lhs, rhs));
  2187. else
  2188. SetResult(curId, mIRBuilder->CreateICmpSLE(lhs, rhs));
  2189. }
  2190. break;
  2191. case BfIRCmd_CmpULE:
  2192. {
  2193. CMD_PARAM(llvm::Value*, lhs);
  2194. CMD_PARAM(llvm::Value*, rhs);
  2195. if (lhs->getType()->isFloatingPointTy())
  2196. SetResult(curId, mIRBuilder->CreateFCmpOLE(lhs, rhs));
  2197. else
  2198. SetResult(curId, mIRBuilder->CreateICmpULE(lhs, rhs));
  2199. }
  2200. break;
  2201. case BfIRCmd_CmpSGT:
  2202. {
  2203. CMD_PARAM(llvm::Value*, lhs);
  2204. CMD_PARAM(llvm::Value*, rhs);
  2205. if (lhs->getType()->isFloatingPointTy())
  2206. SetResult(curId, mIRBuilder->CreateFCmpOGT(lhs, rhs));
  2207. else
  2208. SetResult(curId, mIRBuilder->CreateICmpSGT(lhs, rhs));
  2209. }
  2210. break;
  2211. case BfIRCmd_CmpUGT:
  2212. {
  2213. CMD_PARAM(llvm::Value*, lhs);
  2214. CMD_PARAM(llvm::Value*, rhs);
  2215. if (lhs->getType()->isFloatingPointTy())
  2216. SetResult(curId, mIRBuilder->CreateFCmpOGT(lhs, rhs));
  2217. else
  2218. SetResult(curId, mIRBuilder->CreateICmpUGT(lhs, rhs));
  2219. }
  2220. break;
  2221. case BfIRCmd_CmpSGE:
  2222. {
  2223. CMD_PARAM(llvm::Value*, lhs);
  2224. CMD_PARAM(llvm::Value*, rhs);
  2225. if (lhs->getType()->isFloatingPointTy())
  2226. SetResult(curId, mIRBuilder->CreateFCmpOGE(lhs, rhs));
  2227. else
  2228. SetResult(curId, mIRBuilder->CreateICmpSGE(lhs, rhs));
  2229. }
  2230. break;
  2231. case BfIRCmd_CmpUGE:
  2232. {
  2233. CMD_PARAM(llvm::Value*, lhs);
  2234. CMD_PARAM(llvm::Value*, rhs);
  2235. if (lhs->getType()->isFloatingPointTy())
  2236. SetResult(curId, mIRBuilder->CreateFCmpOGE(lhs, rhs));
  2237. else
  2238. SetResult(curId, mIRBuilder->CreateICmpUGE(lhs, rhs));
  2239. }
  2240. break;
  2241. case BfIRCmd_Add:
  2242. {
  2243. CMD_PARAM(llvm::Value*, lhs);
  2244. CMD_PARAM(llvm::Value*, rhs);
  2245. CMD_PARAM(int8, overflowCheckKind);
  2246. if (lhs->getType()->isFloatingPointTy())
  2247. SetResult(curId, mIRBuilder->CreateFAdd(lhs, rhs));
  2248. else if ((overflowCheckKind & (BfOverflowCheckKind_Signed | BfOverflowCheckKind_Unsigned)) != 0)
  2249. SetResult(curId, DoCheckedIntrinsic(((overflowCheckKind & BfOverflowCheckKind_Signed) != 0) ? llvm::Intrinsic::sadd_with_overflow : llvm::Intrinsic::uadd_with_overflow,
  2250. lhs, rhs, (overflowCheckKind & BfOverflowCheckKind_Flag_UseAsm) != 0));
  2251. else
  2252. SetResult(curId, mIRBuilder->CreateAdd(lhs, rhs));
  2253. }
  2254. break;
  2255. case BfIRCmd_Sub:
  2256. {
  2257. CMD_PARAM(llvm::Value*, lhs);
  2258. CMD_PARAM(llvm::Value*, rhs);
  2259. CMD_PARAM(int8, overflowCheckKind);
  2260. if (lhs->getType()->isFloatingPointTy())
  2261. SetResult(curId, mIRBuilder->CreateFSub(lhs, rhs));
  2262. else if ((overflowCheckKind & (BfOverflowCheckKind_Signed | BfOverflowCheckKind_Unsigned)) != 0)
  2263. SetResult(curId, DoCheckedIntrinsic(((overflowCheckKind & BfOverflowCheckKind_Signed) != 0) ? llvm::Intrinsic::ssub_with_overflow : llvm::Intrinsic::usub_with_overflow,
  2264. lhs, rhs, (overflowCheckKind & BfOverflowCheckKind_Flag_UseAsm) != 0));
  2265. else
  2266. SetResult(curId, mIRBuilder->CreateSub(lhs, rhs));
  2267. }
  2268. break;
  2269. case BfIRCmd_Mul:
  2270. {
  2271. CMD_PARAM(llvm::Value*, lhs);
  2272. CMD_PARAM(llvm::Value*, rhs);
  2273. CMD_PARAM(int8, overflowCheckKind);
  2274. if (lhs->getType()->isFloatingPointTy())
  2275. SetResult(curId, mIRBuilder->CreateFMul(lhs, rhs));
  2276. else if ((overflowCheckKind & (BfOverflowCheckKind_Signed | BfOverflowCheckKind_Unsigned)) != 0)
  2277. SetResult(curId, DoCheckedIntrinsic(((overflowCheckKind & BfOverflowCheckKind_Signed) != 0) ? llvm::Intrinsic::smul_with_overflow : llvm::Intrinsic::umul_with_overflow,
  2278. lhs, rhs, (overflowCheckKind & BfOverflowCheckKind_Flag_UseAsm) != 0));
  2279. else
  2280. SetResult(curId, mIRBuilder->CreateMul(lhs, rhs));
  2281. }
  2282. break;
  2283. case BfIRCmd_SDiv:
  2284. {
  2285. CMD_PARAM(llvm::Value*, lhs);
  2286. CMD_PARAM(llvm::Value*, rhs);
  2287. if (lhs->getType()->isFloatingPointTy())
  2288. SetResult(curId, mIRBuilder->CreateFDiv(lhs, rhs));
  2289. else
  2290. SetResult(curId, mIRBuilder->CreateSDiv(lhs, rhs));
  2291. }
  2292. break;
  2293. case BfIRCmd_UDiv:
  2294. {
  2295. CMD_PARAM(llvm::Value*, lhs);
  2296. CMD_PARAM(llvm::Value*, rhs);
  2297. SetResult(curId, mIRBuilder->CreateUDiv(lhs, rhs));
  2298. }
  2299. break;
  2300. case BfIRCmd_SRem:
  2301. {
  2302. CMD_PARAM(llvm::Value*, lhs);
  2303. CMD_PARAM(llvm::Value*, rhs);
  2304. if (lhs->getType()->isFloatingPointTy())
  2305. SetResult(curId, mIRBuilder->CreateFRem(lhs, rhs));
  2306. else
  2307. SetResult(curId, mIRBuilder->CreateSRem(lhs, rhs));
  2308. }
  2309. break;
  2310. case BfIRCmd_URem:
  2311. {
  2312. CMD_PARAM(llvm::Value*, lhs);
  2313. CMD_PARAM(llvm::Value*, rhs);
  2314. SetResult(curId, mIRBuilder->CreateURem(lhs, rhs));
  2315. }
  2316. break;
  2317. case BfIRCmd_And:
  2318. {
  2319. CMD_PARAM(llvm::Value*, lhs);
  2320. CMD_PARAM(llvm::Value*, rhs);
  2321. SetResult(curId, mIRBuilder->CreateAnd(lhs, rhs));
  2322. }
  2323. break;
  2324. case BfIRCmd_Or:
  2325. {
  2326. CMD_PARAM(llvm::Value*, lhs);
  2327. CMD_PARAM(llvm::Value*, rhs);
  2328. SetResult(curId, mIRBuilder->CreateOr(lhs, rhs));
  2329. }
  2330. break;
  2331. case BfIRCmd_Xor:
  2332. {
  2333. CMD_PARAM(llvm::Value*, lhs);
  2334. CMD_PARAM(llvm::Value*, rhs);
  2335. SetResult(curId, mIRBuilder->CreateXor(lhs, rhs));
  2336. }
  2337. break;
  2338. case BfIRCmd_Shl:
  2339. {
  2340. CMD_PARAM(llvm::Value*, lhs);
  2341. CMD_PARAM(llvm::Value*, rhs);
  2342. SetResult(curId, mIRBuilder->CreateShl(lhs, rhs));
  2343. }
  2344. break;
  2345. case BfIRCmd_AShr:
  2346. {
  2347. CMD_PARAM(llvm::Value*, lhs);
  2348. CMD_PARAM(llvm::Value*, rhs);
  2349. SetResult(curId, mIRBuilder->CreateAShr(lhs, rhs));
  2350. }
  2351. break;
  2352. case BfIRCmd_LShr:
  2353. {
  2354. CMD_PARAM(llvm::Value*, lhs);
  2355. CMD_PARAM(llvm::Value*, rhs);
  2356. SetResult(curId, mIRBuilder->CreateLShr(lhs, rhs));
  2357. }
  2358. break;
  2359. case BfIRCmd_Neg:
  2360. {
  2361. CMD_PARAM(llvm::Value*, val);
  2362. if (val->getType()->isFloatingPointTy())
  2363. SetResult(curId, mIRBuilder->CreateFNeg(val));
  2364. else
  2365. SetResult(curId, mIRBuilder->CreateNeg(val));
  2366. }
  2367. break;
  2368. case BfIRCmd_Not:
  2369. {
  2370. CMD_PARAM(llvm::Value*, val);
  2371. SetResult(curId, mIRBuilder->CreateNot(val));
  2372. }
  2373. break;
  2374. case BfIRCmd_BitCast:
  2375. {
  2376. CMD_PARAM(BfIRTypedValue, val);
  2377. CMD_PARAM(BfIRTypeEx*, toType);
  2378. BfIRTypedValue result;
  2379. result.mTypeEx = toType;
  2380. auto fromType = val.mValue->getType();
  2381. if ((!fromType->isPointerTy()) || (!toType->mLLVMType->isPointerTy()))
  2382. {
  2383. if (fromType->isIntegerTy())
  2384. result.mValue = mIRBuilder->CreateIntToPtr(val.mValue, toType->mLLVMType);
  2385. else
  2386. result.mValue = mIRBuilder->CreatePtrToInt(val.mValue, toType->mLLVMType);
  2387. }
  2388. else
  2389. result.mValue = mIRBuilder->CreateBitCast(val.mValue, toType->mLLVMType);
  2390. SetResult(curId, result);
  2391. }
  2392. break;
  2393. case BfIRCmd_PtrToInt:
  2394. {
  2395. CMD_PARAM(llvm::Value*, val);
  2396. auto typeCode = (BfTypeCode)mStream->Read();
  2397. bool isSigned;
  2398. BfIRTypedValue result;
  2399. result.mTypeEx = GetTypeEx(typeCode, isSigned);
  2400. result.mValue = mIRBuilder->CreatePtrToInt(val, result.mTypeEx->mLLVMType);
  2401. SetResult(curId, result);
  2402. }
  2403. break;
  2404. case BfIRCmd_IntToPtr:
  2405. {
  2406. CMD_PARAM(llvm::Value*, val);
  2407. CMD_PARAM(BfIRTypeEx*, toType);
  2408. BfIRTypedValue result;
  2409. result.mTypeEx = toType;
  2410. result.mValue = mIRBuilder->CreateIntToPtr(val, toType->mLLVMType);
  2411. SetResult(curId, result);
  2412. }
  2413. break;
  2414. case BfIRCmd_InboundsGEP1_32:
  2415. {
  2416. CMD_PARAM(BfIRTypedValue, val);
  2417. CMD_PARAM(int, idx0);
  2418. BfIRTypedValue result;
  2419. result.mTypeEx = val.mTypeEx;
  2420. auto alignedPtr = GetAlignedPtr(val);
  2421. auto compositeType = GetTypeMember(alignedPtr.mTypeEx, 0);
  2422. result.mValue = mIRBuilder->CreateConstInBoundsGEP1_32(compositeType->mLLVMType, alignedPtr.mValue, idx0);
  2423. SetResult(curId, result);
  2424. }
  2425. break;
  2426. case BfIRCmd_InboundsGEP2_32:
  2427. {
  2428. CMD_PARAM(BfIRTypedValue, val);
  2429. CMD_PARAM(int, idx0);
  2430. CMD_PARAM(int, idx1);
  2431. auto compositeType = GetTypeMember(val.mTypeEx, 0);
  2432. int elemIdx = BF_MIN(idx1, (int)compositeType->mMembers.mSize - 1);
  2433. BfIRTypeEx* elemType = GetTypeMember(compositeType, elemIdx);
  2434. BfIRTypedValue result;
  2435. result.mValue = mIRBuilder->CreateConstInBoundsGEP2_32(compositeType->mLLVMType, val.mValue, idx0, idx1);
  2436. result.mTypeEx = GetPointerTypeEx(elemType);
  2437. SetResult(curId, result);
  2438. }
  2439. break;
  2440. case BfIRCmd_InBoundsGEP1:
  2441. {
  2442. CMD_PARAM(BfIRTypedValue, val);
  2443. CMD_PARAM(llvm::Value*, idx0);
  2444. BfIRTypedValue result;
  2445. auto alignedPtr = GetAlignedPtr(val);
  2446. auto compositeType = GetTypeMember(alignedPtr.mTypeEx, 0);
  2447. FixIndexer(idx0);
  2448. result.mValue = mIRBuilder->CreateInBoundsGEP(compositeType->mLLVMType, alignedPtr.mValue, idx0);
  2449. result.mTypeEx = val.mTypeEx;
  2450. SetResult(curId, result);
  2451. }
  2452. break;
  2453. case BfIRCmd_InBoundsGEP2:
  2454. {
  2455. CMD_PARAM(BfIRTypedValue, val);
  2456. CMD_PARAM(llvm::Value*, idx0);
  2457. CMD_PARAM(llvm::Value*, idx1);
  2458. FixIndexer(idx0);
  2459. FixIndexer(idx1);
  2460. llvm::Value* indices[2] = { idx0, idx1 };
  2461. int elemIdx = 0;
  2462. if (auto constInt = llvm::dyn_cast<llvm::ConstantInt>(idx1))
  2463. elemIdx = BF_MIN((int)constInt->getSExtValue(), (int)val.mTypeEx->mMembers.mSize - 1);
  2464. auto compositeType = GetTypeMember(val.mTypeEx, 0);
  2465. BfIRTypeEx* elemType = GetTypeMember(compositeType, elemIdx);
  2466. BfIRTypedValue result;
  2467. result.mValue = mIRBuilder->CreateInBoundsGEP(compositeType->mLLVMType, val.mValue, llvm::ArrayRef(indices));
  2468. result.mTypeEx = GetPointerTypeEx(elemType);
  2469. SetResult(curId, result);
  2470. }
  2471. break;
  2472. case BfIRCmd_IsNull:
  2473. {
  2474. CMD_PARAM(llvm::Value*, val);
  2475. SetResult(curId, mIRBuilder->CreateIsNull(val));
  2476. }
  2477. break;
  2478. case BfIRCmd_IsNotNull:
  2479. {
  2480. CMD_PARAM(llvm::Value*, val);
  2481. SetResult(curId, mIRBuilder->CreateIsNotNull(val));
  2482. }
  2483. break;
  2484. case BfIRCmd_ExtractValue:
  2485. {
  2486. CMD_PARAM(BfIRTypedValue, val);
  2487. CMD_PARAM(int, idx);
  2488. auto compositeType = val.mTypeEx;
  2489. int elemIdx = BF_MIN(idx, (int)compositeType->mMembers.mSize - 1);
  2490. auto elemType = GetTypeMember(compositeType, elemIdx);
  2491. BfIRTypedValue result;
  2492. result.mTypeEx = elemType;
  2493. result.mValue = mIRBuilder->CreateExtractValue(val.mValue, llvm::ArrayRef((unsigned)idx));
  2494. SetResult(curId, result);
  2495. }
  2496. break;
  2497. case BfIRCmd_InsertValue:
  2498. {
  2499. CMD_PARAM(BfIRTypedValue, agg);
  2500. CMD_PARAM(BfIRTypedValue, val);
  2501. CMD_PARAM(int, idx);
  2502. BfIRTypedValue result;
  2503. result.mTypeEx = agg.mTypeEx;
  2504. result.mValue = mIRBuilder->CreateInsertValue(agg.mValue, val.mValue, llvm::ArrayRef((unsigned)idx));
  2505. SetResult(curId, result);
  2506. }
  2507. break;
  2508. case BfIRCmd_Alloca:
  2509. {
  2510. CMD_PARAM(BfIRTypeEx*, type);
  2511. if (type->mLLVMType->isStructTy())
  2512. {
  2513. BF_ASSERT(!((llvm::StructType*)type->mLLVMType)->isOpaque());
  2514. }
  2515. BfIRTypedValue result;
  2516. result.mTypeEx = GetPointerTypeEx(type);
  2517. result.mValue = mIRBuilder->CreateAlloca(type->mLLVMType);
  2518. SetResult(curId, result);
  2519. }
  2520. break;
  2521. case BfIRCmd_AllocaArray:
  2522. {
  2523. CMD_PARAM(BfIRTypeEx*, type);
  2524. CMD_PARAM(llvm::Value*, arraySize);
  2525. auto origType = type;
  2526. auto typeEntry = GetTypeEntry(type);
  2527. if (typeEntry != NULL)
  2528. type = GetSizeAlignedType(typeEntry);
  2529. BfIRTypedValue typedValue;
  2530. typedValue.mTypeEx = GetPointerTypeEx(type);
  2531. if (origType != type)
  2532. {
  2533. typedValue.mValue = mIRBuilder->CreateAlloca(type->mLLVMType, arraySize);
  2534. SetResultAligned(curId, typedValue);
  2535. }
  2536. else
  2537. {
  2538. typedValue.mValue = mIRBuilder->CreateAlloca(type->mLLVMType, arraySize);
  2539. SetResult(curId, typedValue);
  2540. }
  2541. }
  2542. break;
  2543. case BfIRCmd_SetAllocaAlignment:
  2544. {
  2545. CMD_PARAM_NOTRANS(llvm::Value*, val);
  2546. CMD_PARAM(int, alignment);
  2547. auto inst = llvm::dyn_cast<llvm::AllocaInst>(val);
  2548. inst->setAlignment(llvm::Align(alignment));
  2549. }
  2550. break;
  2551. case BfIRCmd_SetAllocaNoChkStkHint:
  2552. {
  2553. CMD_PARAM_NOTRANS(llvm::Value*, val);
  2554. // LLVM does not support this
  2555. }
  2556. break;
  2557. case BfIRCmd_LifetimeStart:
  2558. {
  2559. CMD_PARAM_NOTRANS(llvm::Value*, val);
  2560. SetResult(curId, mIRBuilder->CreateLifetimeStart(val));
  2561. }
  2562. break;
  2563. case BfIRCmd_LifetimeEnd:
  2564. {
  2565. CMD_PARAM_NOTRANS(llvm::Value*, val);
  2566. SetResult(curId, mIRBuilder->CreateLifetimeEnd(val));
  2567. }
  2568. break;
  2569. case BfIRCmd_LifetimeSoftEnd:
  2570. {
  2571. CMD_PARAM_NOTRANS(llvm::Value*, val);
  2572. }
  2573. break;
  2574. case BfIRCmd_LifetimeExtend:
  2575. {
  2576. CMD_PARAM_NOTRANS(llvm::Value*, val);
  2577. }
  2578. break;
  2579. case BfIRCmd_Load:
  2580. {
  2581. CMD_PARAM(BfIRTypedValue, typedValue);
  2582. BF_ASSERT(typedValue.mTypeEx != NULL);
  2583. CMD_PARAM(bool, isVolatile);
  2584. BfIRTypedValue result;
  2585. result.mTypeEx = GetTypeMember(typedValue.mTypeEx, 0);
  2586. result.mValue = mIRBuilder->CreateLoad(result.mTypeEx->mLLVMType, typedValue.mValue, isVolatile);
  2587. SetResult(curId, result);
  2588. }
  2589. break;
  2590. case BfIRCmd_AlignedLoad:
  2591. {
  2592. CMD_PARAM(BfIRTypedValue, typedValue);
  2593. BF_ASSERT(typedValue.mTypeEx != NULL);
  2594. CMD_PARAM(int, alignment);
  2595. CMD_PARAM(bool, isVolatile);
  2596. BfIRTypedValue result;
  2597. result.mTypeEx = GetTypeMember(typedValue.mTypeEx, 0);
  2598. result.mValue = mIRBuilder->CreateAlignedLoad(result.mTypeEx->mLLVMType, typedValue.mValue, llvm::MaybeAlign(alignment), isVolatile);
  2599. SetResult(curId, result);
  2600. }
  2601. break;
  2602. case BfIRCmd_Store:
  2603. {
  2604. CMD_PARAM(BfIRTypedValue, val);
  2605. CMD_PARAM(BfIRTypedValue, ptr);
  2606. CMD_PARAM(bool, isVolatile);
  2607. if ((!TryMemCpy(ptr, val.mValue)) &&
  2608. (!TryVectorCpy(ptr, val.mValue)))
  2609. SetResult(curId, mIRBuilder->CreateStore(val.mValue, ptr.mValue, isVolatile));
  2610. }
  2611. break;
  2612. case BfIRCmd_AlignedStore:
  2613. {
  2614. CMD_PARAM(BfIRTypedValue, val);
  2615. CMD_PARAM(BfIRTypedValue, ptr);
  2616. CMD_PARAM(int, alignment);
  2617. CMD_PARAM(bool, isVolatile);
  2618. if ((!TryMemCpy(ptr, val.mValue)) &&
  2619. (!TryVectorCpy(ptr, val.mValue)))
  2620. SetResult(curId, mIRBuilder->CreateAlignedStore(val.mValue, ptr.mValue, llvm::MaybeAlign(alignment), isVolatile));
  2621. }
  2622. break;
  2623. case BfIRCmd_MemSet:
  2624. {
  2625. CMD_PARAM(llvm::Value*, addr);
  2626. CMD_PARAM(llvm::Value*, val);
  2627. CMD_PARAM(llvm::Value*, size);
  2628. CMD_PARAM(int, alignment);
  2629. CreateMemSet(addr, val, size, alignment);
  2630. }
  2631. break;
  2632. case BfIRCmd_Fence:
  2633. {
  2634. BfIRFenceType fenceType = (BfIRFenceType)mStream->Read();
  2635. if (fenceType == BfIRFenceType_AcquireRelease)
  2636. mIRBuilder->CreateFence(llvm::AtomicOrdering::AcquireRelease);
  2637. }
  2638. break;
  2639. case BfIRCmd_StackSave:
  2640. {
  2641. //auto intrin = llvm::Intrinsic::getDeclaration(mLLVMModule, llvm::Intrinsic::stacksave);
  2642. //CreateStackSave
  2643. //auto callInst = mIRBuilder->CreateCall(intrin);
  2644. BfIRTypedValue result;
  2645. result.mValue = mIRBuilder->CreateStackSave();
  2646. result.mTypeEx = GetTypeEx(result.mValue->getType());
  2647. SetResult(curId, result);
  2648. }
  2649. break;
  2650. case BfIRCmd_StackRestore:
  2651. {
  2652. CMD_PARAM(llvm::Value*, stackVal);
  2653. //auto intrin = llvm::Intrinsic::getDeclaration(mLLVMModule, llvm::Intrinsic::stackrestore);
  2654. //auto callInst = mIRBuilder->CreateCall(intrin, llvm::SmallVector<llvm::Value*, 1> {stackVal });
  2655. auto callInst = mIRBuilder->CreateStackRestore(stackVal);
  2656. SetResult(curId, callInst);
  2657. }
  2658. break;
  2659. case BfIRCmd_GlobalVariable:
  2660. {
  2661. CMD_PARAM(BfIRTypeEx*, varType);
  2662. CMD_PARAM(bool, isConstant);
  2663. BfIRLinkageType linkageType = (BfIRLinkageType)mStream->Read();
  2664. CMD_PARAM(String, name);
  2665. CMD_PARAM(bool, isTLS);
  2666. CMD_PARAM(llvm::Constant*, initializer);
  2667. auto globalVariable = mLLVMModule->getGlobalVariable(name.c_str());
  2668. if (globalVariable == NULL)
  2669. {
  2670. globalVariable = new llvm::GlobalVariable(
  2671. *mLLVMModule,
  2672. varType->mLLVMType,
  2673. isConstant,
  2674. LLVMMapLinkageType(linkageType),
  2675. initializer,
  2676. name.c_str(), NULL, isTLS ? llvm::GlobalValue::GeneralDynamicTLSModel : llvm::GlobalValue::NotThreadLocal);
  2677. }
  2678. BfIRTypedValue result;
  2679. result.mValue = globalVariable;
  2680. result.mTypeEx = GetPointerTypeEx(varType);
  2681. SetResult(curId, result);
  2682. }
  2683. break;
  2684. case BfIRCmd_GlobalVar_SetUnnamedAddr:
  2685. {
  2686. CMD_PARAM(llvm::Value*, val);
  2687. CMD_PARAM(bool, unnamedAddr);
  2688. ((llvm::GlobalVariable*)val)->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
  2689. }
  2690. break;
  2691. case BfIRCmd_GlobalVar_SetInitializer:
  2692. {
  2693. CMD_PARAM(llvm::Value*, val);
  2694. CMD_PARAM(llvm::Constant*, initializer);
  2695. ((llvm::GlobalVariable*)val)->setInitializer(initializer);
  2696. }
  2697. break;
  2698. case BfIRCmd_GlobalVar_SetAlignment:
  2699. {
  2700. CMD_PARAM(llvm::Value*, val);
  2701. CMD_PARAM(int, alignment);
  2702. ((llvm::GlobalVariable*)val)->setAlignment(llvm::Align(alignment));
  2703. }
  2704. break;
  2705. case BfIRCmd_GlobalVar_SetStorageKind:
  2706. {
  2707. CMD_PARAM(llvm::Value*, val);
  2708. CMD_PARAM(int, storageKind);
  2709. ((llvm::GlobalVariable*)val)->setDLLStorageClass((llvm::GlobalValue::DLLStorageClassTypes)storageKind);
  2710. }
  2711. break;
  2712. case BfIRCmd_GlobalStringPtr:
  2713. {
  2714. CMD_PARAM(String, str);
  2715. BfIRTypedValue result;
  2716. result.mValue = mIRBuilder->CreateGlobalStringPtr(llvm::StringRef(str.c_str(), str.length()));
  2717. result.mTypeEx = GetTypeEx(result.mValue->getType());
  2718. SetResult(curId, result);
  2719. }
  2720. break;
  2721. case BfIRCmd_SetReflectTypeData:
  2722. {
  2723. CMD_PARAM(BfIRTypeEx*, type);
  2724. CMD_PARAM(BfIRTypedValue, value);
  2725. mReflectDataMap[type] = value;
  2726. }
  2727. break;
  2728. case BfIRCmd_CreateBlock:
  2729. {
  2730. CMD_PARAM(String, name);
  2731. CMD_PARAM(bool, addNow);
  2732. auto block = llvm::BasicBlock::Create(*mLLVMContext, name.c_str());
  2733. if (addNow)
  2734. mActiveFunction->insert(mActiveFunction->end(), block);
  2735. SetResult(curId, block);
  2736. }
  2737. break;
  2738. case BfIRCmd_MaybeChainNewBlock:
  2739. {
  2740. CMD_PARAM(String, name);
  2741. auto newBlock = mIRBuilder->GetInsertBlock();
  2742. if (!newBlock->empty())
  2743. {
  2744. auto bb = llvm::BasicBlock::Create(*mLLVMContext, name.c_str());
  2745. mIRBuilder->CreateBr(bb);
  2746. mActiveFunction->insert(mActiveFunction->end(), bb);
  2747. mIRBuilder->SetInsertPoint(bb);
  2748. newBlock = bb;
  2749. }
  2750. SetResult(curId, newBlock);
  2751. }
  2752. break;
  2753. case BfIRCmd_AddBlock:
  2754. {
  2755. CMD_PARAM(llvm::BasicBlock*, block);
  2756. mActiveFunction->insert(mActiveFunction->end(), block);
  2757. }
  2758. break;
  2759. case BfIRCmd_DropBlocks:
  2760. {
  2761. //TODO: Not even needed
  2762. // CMD_PARAM(llvm::BasicBlock*, startingBlock);
  2763. // auto& basicBlockList = mActiveFunction->getBasicBlockList();
  2764. // int postExitBlockIdx = -1;
  2765. //
  2766. // auto itr = basicBlockList.rbegin();
  2767. // int blockIdx = (int)basicBlockList.size() - 1;
  2768. // while (itr != basicBlockList.rend())
  2769. // {
  2770. // auto& block = *itr++;
  2771. // block.dropAllReferences();
  2772. // if (&block == startingBlock)
  2773. // {
  2774. // postExitBlockIdx = blockIdx;
  2775. // break;
  2776. // }
  2777. // blockIdx--;
  2778. // }
  2779. //
  2780. // while ((int)basicBlockList.size() > postExitBlockIdx)
  2781. // {
  2782. // auto& block = basicBlockList.back();
  2783. // block.eraseFromParent();
  2784. // }
  2785. }
  2786. break;
  2787. case BfIRCmd_MergeBlockDown:
  2788. {
  2789. CMD_PARAM(llvm::BasicBlock*, fromBlock);
  2790. CMD_PARAM(llvm::BasicBlock*, intoBlock);
  2791. //llvm::BasicBlock::InstListType& fromInstList = fromBlock->getInstList();
  2792. //llvm::BasicBlock::InstListType& intoInstList = intoBlock->getInstList();
  2793. //intoInstList.splice(intoInstList.begin(), fromInstList, fromInstList.begin(), fromInstList.end());
  2794. intoBlock->splice(intoBlock->begin(), fromBlock);
  2795. fromBlock->eraseFromParent();
  2796. }
  2797. break;
  2798. case BfIRCmd_GetInsertBlock:
  2799. {
  2800. SetResult(curId, mIRBuilder->GetInsertBlock());
  2801. }
  2802. break;
  2803. case BfIRCmd_SetInsertPoint:
  2804. {
  2805. CMD_PARAM(llvm::BasicBlock*, block);
  2806. if (mLockedBlocks.Contains(block))
  2807. Fail("Attempt to modify locked block");
  2808. mIRBuilder->SetInsertPoint(block);
  2809. }
  2810. break;
  2811. case BfIRCmd_SetInsertPointAtStart:
  2812. {
  2813. CMD_PARAM(llvm::BasicBlock*, block);
  2814. auto itr = block->begin();
  2815. if (itr != block->end())
  2816. {
  2817. // Some instructructions MUST be at start of the block
  2818. auto instType = itr->getType();
  2819. if (llvm::PHINode::classof(&*itr))
  2820. ++itr;
  2821. }
  2822. mIRBuilder->SetInsertPoint(block, itr);
  2823. // SetInsertPoint can clear the debug loc so reset it here
  2824. mIRBuilder->SetCurrentDebugLocation(mDebugLoc);
  2825. }
  2826. break;
  2827. case BfIRCmd_EraseFromParent:
  2828. {
  2829. CMD_PARAM(llvm::BasicBlock*, block);
  2830. block->eraseFromParent();
  2831. }
  2832. break;
  2833. case BfIRCmd_DeleteBlock:
  2834. {
  2835. CMD_PARAM(llvm::BasicBlock*, block);
  2836. delete block;
  2837. }
  2838. break;
  2839. case BfIRCmd_EraseInstFromParent:
  2840. {
  2841. CMD_PARAM(llvm::Value*, instVal);
  2842. BF_ASSERT(llvm::isa<llvm::Instruction>(instVal));
  2843. ((llvm::Instruction*)instVal)->eraseFromParent();
  2844. }
  2845. break;
  2846. case BfIRCmd_CreateBr:
  2847. case BfIRCmd_CreateBr_NoCollapse:
  2848. {
  2849. CMD_PARAM(llvm::BasicBlock*, block);
  2850. mIRBuilder->CreateBr(block);
  2851. }
  2852. break;
  2853. case BfIRCmd_CreateBr_Fake:
  2854. {
  2855. CMD_PARAM(llvm::BasicBlock*, block);
  2856. // Do nothing
  2857. }
  2858. break;
  2859. case BfIRCmd_CreateCondBr:
  2860. {
  2861. CMD_PARAM(llvm::Value*, condVal);
  2862. CMD_PARAM(llvm::BasicBlock*, trueBlock);
  2863. CMD_PARAM(llvm::BasicBlock*, falseBlock);
  2864. mIRBuilder->CreateCondBr(condVal, trueBlock, falseBlock);
  2865. }
  2866. break;
  2867. case BfIRCmd_MoveBlockToEnd:
  2868. {
  2869. CMD_PARAM(llvm::BasicBlock*, block);
  2870. block->moveAfter(&block->getParent()->back());
  2871. }
  2872. break;
  2873. case BfIRCmd_CreateSwitch:
  2874. {
  2875. CMD_PARAM(llvm::Value*, val);
  2876. CMD_PARAM(llvm::BasicBlock*, dest);
  2877. CMD_PARAM(int, numCases);
  2878. SetResult(curId, mIRBuilder->CreateSwitch(val, dest, numCases));
  2879. }
  2880. break;
  2881. case BfIRCmd_AddSwitchCase:
  2882. {
  2883. CMD_PARAM(llvm::Value*, switchVal);
  2884. CMD_PARAM(llvm::Value*, caseVal);
  2885. CMD_PARAM(llvm::BasicBlock*, caseBlock);
  2886. BF_ASSERT(llvm::isa<llvm::SwitchInst>(switchVal));
  2887. BF_ASSERT(llvm::isa<llvm::ConstantInt>(caseVal));
  2888. ((llvm::SwitchInst*)switchVal)->addCase((llvm::ConstantInt*)caseVal, caseBlock);
  2889. }
  2890. break;
  2891. case BfIRCmd_SetSwitchDefaultDest:
  2892. {
  2893. CMD_PARAM(llvm::Value*, switchVal);
  2894. CMD_PARAM(llvm::BasicBlock*, caseBlock);
  2895. ((llvm::SwitchInst*)switchVal)->setDefaultDest(caseBlock);
  2896. }
  2897. break;
  2898. case BfIRCmd_CreatePhi:
  2899. {
  2900. CMD_PARAM(BfIRTypeEx*, type);
  2901. CMD_PARAM(int, incomingCount);
  2902. BfIRTypedValue result;
  2903. result.mTypeEx = type;
  2904. result.mValue = mIRBuilder->CreatePHI(type->mLLVMType, incomingCount);
  2905. SetResult(curId, result);
  2906. }
  2907. break;
  2908. case BfIRCmd_AddPhiIncoming:
  2909. {
  2910. CMD_PARAM(llvm::Value*, phiValue);
  2911. CMD_PARAM(llvm::Value*, value);
  2912. CMD_PARAM(llvm::BasicBlock*, comingFrom);
  2913. BF_ASSERT(llvm::isa<llvm::PHINode>(phiValue));
  2914. ((llvm::PHINode*)phiValue)->addIncoming(value, comingFrom);
  2915. }
  2916. break;
  2917. case BfIRCmd_GetIntrinsic:
  2918. {
  2919. CMD_PARAM(String, intrinName);
  2920. CMD_PARAM(int, intrinId);
  2921. CMD_PARAM(BfIRTypeEx*, returnType);
  2922. CMD_PARAM(CmdParamVec<BfIRTypeEx*>, paramTypes);
  2923. llvm::Function* func = NULL;
  2924. struct _Intrinsics
  2925. {
  2926. llvm::Intrinsic::ID mID;
  2927. int mArg0;
  2928. int mArg1;
  2929. int mArg2;
  2930. };
  2931. static _Intrinsics intrinsics[] =
  2932. {
  2933. { (llvm::Intrinsic::ID)-1, -1}, // PLATFORM,
  2934. { llvm::Intrinsic::fabs, 0, -1},
  2935. { (llvm::Intrinsic::ID)-2, -1}, // add,
  2936. { (llvm::Intrinsic::ID)-2, -1}, // and,
  2937. { (llvm::Intrinsic::ID)-2, -1}, // AtomicAdd,
  2938. { (llvm::Intrinsic::ID)-2, -1}, // AtomicAnd,
  2939. { (llvm::Intrinsic::ID)-2, -1}, // AtomicCmpStore,
  2940. { (llvm::Intrinsic::ID)-2, -1}, // AtomicCmpStore_Weak,
  2941. { (llvm::Intrinsic::ID)-2, -1}, // AtomicCmpXChg,
  2942. { (llvm::Intrinsic::ID)-2, -1}, // AtomicFence,
  2943. { (llvm::Intrinsic::ID)-2, -1}, // AtomicLoad,
  2944. { (llvm::Intrinsic::ID)-2, -1}, // AtomicMax,
  2945. { (llvm::Intrinsic::ID)-2, -1}, // AtomicMin,
  2946. { (llvm::Intrinsic::ID)-2, -1}, // AtomicNAnd,
  2947. { (llvm::Intrinsic::ID)-2, -1}, // AtomicOr,
  2948. { (llvm::Intrinsic::ID)-2, -1}, // AtomicStore,
  2949. { (llvm::Intrinsic::ID)-2, -1}, // AtomicSub,
  2950. { (llvm::Intrinsic::ID)-2, -1}, // AtomicUMax,
  2951. { (llvm::Intrinsic::ID)-2, -1}, // AtomicUMin,
  2952. { (llvm::Intrinsic::ID)-2, -1}, // AtomicXChg,
  2953. { (llvm::Intrinsic::ID)-2, -1}, // AtomicXor,
  2954. { llvm::Intrinsic::bswap, -1},
  2955. { (llvm::Intrinsic::ID)-2, -1}, // cast,
  2956. { llvm::Intrinsic::cos, 0, -1},
  2957. { (llvm::Intrinsic::ID)-2, -1}, // cpuid
  2958. { llvm::Intrinsic::debugtrap, -1}, // debugtrap,
  2959. { (llvm::Intrinsic::ID)-2, -1}, // div
  2960. { (llvm::Intrinsic::ID)-2, -1}, // eq
  2961. { llvm::Intrinsic::floor, 0, -1},
  2962. { (llvm::Intrinsic::ID)-2, -1}, // free
  2963. { (llvm::Intrinsic::ID)-2, -1}, // gt
  2964. { (llvm::Intrinsic::ID)-2, -1}, // gte
  2965. { (llvm::Intrinsic::ID)-2, -1}, // index
  2966. { llvm::Intrinsic::log, 0, -1},
  2967. { llvm::Intrinsic::log10, 0, -1},
  2968. { llvm::Intrinsic::log2, 0, -1},
  2969. { (llvm::Intrinsic::ID)-2, -1}, // lt
  2970. { (llvm::Intrinsic::ID)-2, -1}, // lte
  2971. { (llvm::Intrinsic::ID)-2}, // malloc
  2972. { (llvm::Intrinsic::ID)-2, -1}, // max
  2973. { llvm::Intrinsic::memcpy, 0, 1, 2},
  2974. { llvm::Intrinsic::memmove, 0, 2},
  2975. { llvm::Intrinsic::memset, 0, 2},
  2976. { (llvm::Intrinsic::ID)-2, -1}, // min
  2977. { (llvm::Intrinsic::ID)-2, -1}, // mod
  2978. { (llvm::Intrinsic::ID)-2, -1}, // mul
  2979. { (llvm::Intrinsic::ID)-2, -1}, // neq
  2980. { (llvm::Intrinsic::ID)-2, -1}, // not
  2981. { (llvm::Intrinsic::ID)-2, -1}, // or
  2982. { llvm::Intrinsic::pow, 0, -1},
  2983. { llvm::Intrinsic::powi, 0, -1},
  2984. { llvm::Intrinsic::returnaddress, -1},
  2985. { llvm::Intrinsic::round, 0, -1},
  2986. { (llvm::Intrinsic::ID)-2, -1}, // sar
  2987. { (llvm::Intrinsic::ID)-2, -1}, // shl
  2988. { (llvm::Intrinsic::ID)-2, -1}, // shr
  2989. { (llvm::Intrinsic::ID)-2, -1}, // shuffle
  2990. { llvm::Intrinsic::sin, 0, -1},
  2991. { llvm::Intrinsic::sqrt, 0, -1},
  2992. { (llvm::Intrinsic::ID)-2, -1}, // sub,
  2993. { (llvm::Intrinsic::ID)-2, -1}, // va_arg,
  2994. { llvm::Intrinsic::vaend, -1}, // va_end,
  2995. { llvm::Intrinsic::vastart, -1}, // va_start,
  2996. { (llvm::Intrinsic::ID)-2, -1}, // xgetbv
  2997. { (llvm::Intrinsic::ID)-2, -1}, // xor
  2998. };
  2999. BF_STATIC_ASSERT(BF_ARRAY_COUNT(intrinsics) == BfIRIntrinsic_COUNT);
  3000. CmdParamVec<llvm::Type*> useParams;
  3001. if (intrinsics[intrinId].mArg0 != -1)
  3002. {
  3003. useParams.push_back(paramTypes[0]->mLLVMType);
  3004. if (intrinsics[intrinId].mArg1 != -1)
  3005. {
  3006. useParams.push_back(paramTypes[1]->mLLVMType);
  3007. if (intrinsics[intrinId].mArg2 != -1)
  3008. {
  3009. useParams.push_back(paramTypes[2]->mLLVMType);
  3010. }
  3011. }
  3012. }
  3013. bool isFakeIntrinsic = (int)intrinsics[intrinId].mID == -2;
  3014. if (isFakeIntrinsic)
  3015. {
  3016. auto intrinsicData = mIntrinsicData.Alloc();
  3017. intrinsicData->mName = intrinName;
  3018. intrinsicData->mIntrinsic = (BfIRIntrinsic)intrinId;
  3019. intrinsicData->mReturnType = returnType;
  3020. BfIRCodeGenEntry entry;
  3021. entry.mKind = BfIRCodeGenEntryKind_IntrinsicData;
  3022. entry.mIntrinsicData = intrinsicData;
  3023. mResults.TryAdd(curId, entry);
  3024. break;
  3025. }
  3026. if (intrinId == BfIRIntrinsic__PLATFORM)
  3027. {
  3028. int colonPos = (int)intrinName.IndexOf(':');
  3029. String platName = intrinName.Substring(0, colonPos);
  3030. String platIntrinName = intrinName.Substring(colonPos + 1);
  3031. if (platName.IsEmpty())
  3032. {
  3033. auto intrinsicData = mIntrinsicData.Alloc();
  3034. intrinsicData->mName = platIntrinName;
  3035. intrinsicData->mIntrinsic = BfIRIntrinsic__PLATFORM;
  3036. intrinsicData->mReturnType = returnType;
  3037. BfIRCodeGenEntry entry;
  3038. entry.mKind = BfIRCodeGenEntryKind_IntrinsicData;
  3039. entry.mIntrinsicData = intrinsicData;
  3040. mResults.TryAdd(curId, entry);
  3041. break;
  3042. }
  3043. llvm::Intrinsic::ID intrin = llvm::Intrinsic::getIntrinsicForClangBuiltin(platName.c_str(), platIntrinName.c_str());
  3044. if ((int)intrin <= 0)
  3045. FatalError(StrFormat("Unable to find intrinsic '%s'", intrinName.c_str()));
  3046. else
  3047. func = llvm::Intrinsic::getDeclaration(mLLVMModule, intrinsics[intrinId].mID, useParams);
  3048. }
  3049. else
  3050. {
  3051. BF_ASSERT(intrinsics[intrinId].mID != (llvm::Intrinsic::ID)-1);
  3052. func = llvm::Intrinsic::getDeclaration(mLLVMModule, intrinsics[intrinId].mID, useParams);
  3053. }
  3054. mIntrinsicReverseMap[func] = intrinId;
  3055. auto funcTypeEx = CreateTypeEx(func->getFunctionType());
  3056. funcTypeEx->mMembers.Add(returnType);
  3057. for (auto typeEx : paramTypes)
  3058. funcTypeEx->mMembers.Add(typeEx);
  3059. BfIRTypedValue result;
  3060. result.mTypeEx = GetPointerTypeEx(funcTypeEx);
  3061. result.mValue = func;
  3062. SetResult(curId, result);
  3063. }
  3064. break;
  3065. case BfIRCmd_CreateFunctionType:
  3066. {
  3067. CMD_PARAM(BfIRTypeEx*, resultType);
  3068. CMD_PARAM(CmdParamVec<BfIRTypeEx*>, paramTypes);
  3069. CMD_PARAM(bool, isVarArg);
  3070. CmdParamVec<llvm::Type*> llvmTypes;
  3071. for (auto typeEx : paramTypes)
  3072. {
  3073. if (typeEx->mLLVMType->isPointerTy())
  3074. {
  3075. BF_ASSERT(!typeEx->mMembers.IsEmpty());
  3076. }
  3077. llvmTypes.push_back(typeEx->mLLVMType);
  3078. }
  3079. auto funcType = llvm::FunctionType::get(resultType->mLLVMType, llvmTypes, isVarArg);
  3080. auto typeEx = CreateTypeEx(funcType);
  3081. if (typeEx->mMembers.IsEmpty())
  3082. {
  3083. typeEx->mMembers.Add(resultType);
  3084. for (auto paramType : paramTypes)
  3085. typeEx->mMembers.Add(paramType);
  3086. }
  3087. SetResult(curId, typeEx);
  3088. }
  3089. break;
  3090. case BfIRCmd_CreateFunction:
  3091. {
  3092. BfIRTypeEx* type = NULL;
  3093. ReadFunctionType(type);
  3094. BfIRLinkageType linkageType = (BfIRLinkageType)mStream->Read();
  3095. CMD_PARAM(String, name);
  3096. BfIRTypedValue result;
  3097. result.mTypeEx = GetPointerTypeEx(type);
  3098. auto func = mLLVMModule->getFunction(name.c_str());
  3099. if ((func == NULL) || (func->getFunctionType() != type->mLLVMType))
  3100. func = llvm::Function::Create((llvm::FunctionType*)type->mLLVMType, LLVMMapLinkageType(linkageType), name.c_str(), mLLVMModule);
  3101. result.mValue = func;
  3102. SetResult(curId, result);
  3103. }
  3104. break;
  3105. case BfIRCmd_SetFunctionName:
  3106. {
  3107. CMD_PARAM(llvm::Value*, func);
  3108. CMD_PARAM(String, name);
  3109. llvm::Function* llvmFunc = llvm::dyn_cast<llvm::Function>(func);
  3110. llvmFunc->setName(name.c_str());
  3111. }
  3112. break;
  3113. case BfIRCmd_EnsureFunctionPatchable:
  3114. {
  3115. int minPatchSize = 5;
  3116. int guessInstBytes = 1; // ret
  3117. guessInstBytes += mActiveFunction->getFunctionType()->getNumParams() * 4;
  3118. if (guessInstBytes < 5)
  3119. {
  3120. for (auto& block : *mActiveFunction)
  3121. {
  3122. for (auto& inst : block)
  3123. {
  3124. if (auto loadInst = llvm::dyn_cast<llvm::LoadInst>(&inst))
  3125. guessInstBytes += 2;
  3126. else if (auto storeInst = llvm::dyn_cast<llvm::StoreInst>(&inst))
  3127. guessInstBytes += 2;
  3128. else if (auto callInst = llvm::dyn_cast<llvm::CallInst>(&inst))
  3129. {
  3130. auto calledValue = callInst->getCalledOperand();
  3131. if (calledValue == mNopInlineAsm)
  3132. guessInstBytes += 1;
  3133. else if (auto func = llvm::dyn_cast<llvm::Function>(calledValue))
  3134. {
  3135. if (!func->isIntrinsic())
  3136. guessInstBytes += 4;
  3137. }
  3138. else
  3139. guessInstBytes += 4;
  3140. }
  3141. if (guessInstBytes >= minPatchSize)
  3142. break;
  3143. }
  3144. }
  3145. }
  3146. for (int i = guessInstBytes; i < minPatchSize; i++)
  3147. AddNop();
  3148. }
  3149. break;
  3150. case BfIRCmd_RemapBindFunction:
  3151. {
  3152. CMD_PARAM(BfIRTypedValue, func);
  3153. // We need to store this value to a data segment so we get a symbol we can remap during hot swap
  3154. // We actually do this to ensure that we don't bind to the NEW method but rather the old one- so
  3155. // delegate equality checks still work
  3156. llvm::Function* llvmFunc = llvm::dyn_cast<llvm::Function>(func.mValue);
  3157. if (llvmFunc != NULL)
  3158. {
  3159. // I don't know why we mixed in HSPreserveIdx - that causes bound address to change after reloading, basically totally breaking
  3160. // the whole point of this.
  3161. //String funcName = StrFormat("bf_hs_preserve@%d@%s", mModule->mCompiler->mHSPreserveIdx++, func->getName());
  3162. String funcName = StrFormat("bf_hs_preserve@%s_%s", llvmFunc->getName().data(), mLLVMModule->getName().data());
  3163. llvm::GlobalVariable* globalVariable = mLLVMModule->getGlobalVariable(funcName.c_str());
  3164. if (globalVariable == NULL)
  3165. {
  3166. globalVariable = new llvm::GlobalVariable(*mLLVMModule, llvmFunc->getType(), true, llvm::GlobalValue::ExternalLinkage, (llvm::Constant*)llvmFunc, funcName.c_str());
  3167. }
  3168. BfIRTypedValue result;
  3169. result.mTypeEx = func.mTypeEx;
  3170. result.mValue = mIRBuilder->CreateLoad(result.mTypeEx->mLLVMType, globalVariable);
  3171. SetResult(curId, result);
  3172. }
  3173. else
  3174. SetResult(curId, func);
  3175. }
  3176. break;
  3177. case BfIRCmd_SetActiveFunction:
  3178. {
  3179. BfIRTypedValue func;
  3180. ReadFunction(func);
  3181. mActiveFunction = (llvm::Function*)func.mValue;
  3182. if (mActiveFunction == NULL)
  3183. mActiveFunctionType = NULL;
  3184. else
  3185. mActiveFunctionType = GetTypeMember(func.mTypeEx, 0);
  3186. }
  3187. break;
  3188. case BfIRCmd_CreateCall:
  3189. {
  3190. BfIRTypedValue func;
  3191. BfIRCodeGenEntry* codeGenEntry = NULL;
  3192. Read(func, &codeGenEntry);
  3193. CMD_PARAM(CmdParamVec<BfIRTypedValue>, args);
  3194. if ((func.mValue == NULL) && (codeGenEntry != NULL) && (codeGenEntry->mKind == BfIRCodeGenEntryKind_IntrinsicData))
  3195. {
  3196. auto intrinsicData = codeGenEntry->mIntrinsicData;
  3197. switch (intrinsicData->mIntrinsic)
  3198. {
  3199. case BfIRIntrinsic__PLATFORM:
  3200. {
  3201. if (intrinsicData->mName == "add_string_to_section")
  3202. {
  3203. llvm::StringRef strContent[2];
  3204. llvm::ConstantDataArray* dataArray;
  3205. for (int i = 0; i < 2; i++)
  3206. {
  3207. if (const llvm::ConstantExpr* ce = llvm::dyn_cast<llvm::ConstantExpr>(args[i].mValue))
  3208. {
  3209. llvm::Value* firstOperand = ce->getOperand(0);
  3210. if (llvm::GlobalVariable* gv = llvm::dyn_cast<llvm::GlobalVariable>(firstOperand))
  3211. {
  3212. if (gv->getType()->isPointerTy())
  3213. {
  3214. if (dataArray = llvm::dyn_cast<llvm::ConstantDataArray>(gv->getInitializer()))
  3215. strContent[i] = dataArray->getAsString();
  3216. }
  3217. }
  3218. }
  3219. else
  3220. FatalError("Value is not ConstantExpr");
  3221. }
  3222. static int symbolCount = 0;
  3223. symbolCount++;
  3224. auto charType = llvm::IntegerType::get(*mLLVMContext, 8);
  3225. std::vector<llvm::Constant*> chars(strContent[0].size());
  3226. for (unsigned int i = 0; i < strContent[0].size(); i++)
  3227. {
  3228. chars[i] = llvm::ConstantInt::get(charType, strContent[0][i]);;
  3229. }
  3230. chars.push_back(llvm::ConstantInt::get(charType, 0));
  3231. auto stringType = llvm::ArrayType::get(charType, chars.size());
  3232. std::string symbolName = strContent[1].str() + "_" + std::to_string(symbolCount);
  3233. llvm::StringRef resultStringRef(symbolName);
  3234. auto globalVar = (llvm::GlobalVariable*)mLLVMModule->getOrInsertGlobal(symbolName, stringType);
  3235. globalVar->setSection(strContent[1]);
  3236. globalVar->setInitializer(llvm::ConstantArray::get(stringType, chars));
  3237. globalVar->setConstant(true);
  3238. globalVar->setLinkage(llvm::GlobalValue::LinkageTypes::ExternalLinkage);
  3239. globalVar->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
  3240. SetResult(curId, llvm::ConstantExpr::getBitCast(globalVar, charType->getPointerTo()));
  3241. break;
  3242. }
  3243. FatalError(StrFormat("Unable to find intrinsic '%s'", intrinsicData->mName.c_str()));
  3244. break;
  3245. }
  3246. case BfIRIntrinsic_Add:
  3247. case BfIRIntrinsic_And:
  3248. case BfIRIntrinsic_Div:
  3249. case BfIRIntrinsic_Eq:
  3250. case BfIRIntrinsic_Gt:
  3251. case BfIRIntrinsic_GtE:
  3252. case BfIRIntrinsic_Lt:
  3253. case BfIRIntrinsic_LtE:
  3254. case BfIRIntrinsic_Mod:
  3255. case BfIRIntrinsic_Mul:
  3256. case BfIRIntrinsic_Neq:
  3257. case BfIRIntrinsic_Or:
  3258. case BfIRIntrinsic_Sub:
  3259. case BfIRIntrinsic_Xor:
  3260. {
  3261. auto val0 = TryToVector(args[0]);
  3262. if (val0 != NULL)
  3263. {
  3264. auto vecType = llvm::dyn_cast<llvm::VectorType>(val0->getType());
  3265. auto elemType = vecType->getElementType();
  3266. bool isFP = elemType->isFloatingPointTy();
  3267. llvm::Value* val1;
  3268. if (args.size() < 2)
  3269. {
  3270. llvm::Value* val;
  3271. if (isFP)
  3272. val = llvm::ConstantFP::get(elemType, 1);
  3273. else
  3274. val = llvm::ConstantInt::get(elemType, 1);
  3275. val1 = mIRBuilder->CreateInsertElement(llvm::UndefValue::get(vecType), val, (uint64)0);
  3276. val1 = mIRBuilder->CreateInsertElement(val1, val, (uint64)1);
  3277. val1 = mIRBuilder->CreateInsertElement(val1, val, (uint64)2);
  3278. val1 = mIRBuilder->CreateInsertElement(val1, val, (uint64)3);
  3279. }
  3280. else if (args[1].mValue->getType()->isPointerTy())
  3281. {
  3282. auto ptrVal1 = mIRBuilder->CreateBitCast(args[1].mValue, vecType->getPointerTo());
  3283. val1 = mIRBuilder->CreateAlignedLoad(vecType, ptrVal1, llvm::MaybeAlign(1));
  3284. }
  3285. else if (args[1].mValue->getType()->isVectorTy())
  3286. {
  3287. val1 = args[1].mValue;
  3288. }
  3289. else
  3290. {
  3291. val1 = mIRBuilder->CreateInsertElement(llvm::UndefValue::get(vecType), args[1].mValue, (uint64)0);
  3292. val1 = mIRBuilder->CreateInsertElement(val1, args[1].mValue, (uint64)1);
  3293. val1 = mIRBuilder->CreateInsertElement(val1, args[1].mValue, (uint64)2);
  3294. val1 = mIRBuilder->CreateInsertElement(val1, args[1].mValue, (uint64)3);
  3295. }
  3296. if (isFP)
  3297. {
  3298. llvm::Value* result = NULL;
  3299. switch (intrinsicData->mIntrinsic)
  3300. {
  3301. case BfIRIntrinsic_Add:
  3302. result = mIRBuilder->CreateFAdd(val0, val1);
  3303. break;
  3304. case BfIRIntrinsic_Div:
  3305. result = mIRBuilder->CreateFDiv(val0, val1);
  3306. break;
  3307. case BfIRIntrinsic_Eq:
  3308. result = mIRBuilder->CreateFCmpOEQ(val0, val1);
  3309. break;
  3310. case BfIRIntrinsic_Gt:
  3311. result = mIRBuilder->CreateFCmpOGT(val0, val1);
  3312. break;
  3313. case BfIRIntrinsic_GtE:
  3314. result = mIRBuilder->CreateFCmpOGE(val0, val1);
  3315. break;
  3316. case BfIRIntrinsic_Lt:
  3317. result = mIRBuilder->CreateFCmpOLT(val0, val1);
  3318. break;
  3319. case BfIRIntrinsic_LtE:
  3320. result = mIRBuilder->CreateFCmpOLE(val0, val1);
  3321. break;
  3322. case BfIRIntrinsic_Mod:
  3323. result = mIRBuilder->CreateFRem(val0, val1);
  3324. break;
  3325. case BfIRIntrinsic_Mul:
  3326. result = mIRBuilder->CreateFMul(val0, val1);
  3327. break;
  3328. case BfIRIntrinsic_Neq:
  3329. result = mIRBuilder->CreateFCmpONE(val0, val1);
  3330. break;
  3331. case BfIRIntrinsic_Sub:
  3332. result = mIRBuilder->CreateFSub(val0, val1);
  3333. break;
  3334. default:
  3335. FatalError("Intrinsic argument error");
  3336. }
  3337. if (result != NULL)
  3338. {
  3339. if (auto vecType = llvm::dyn_cast<llvm::FixedVectorType>(result->getType()))
  3340. {
  3341. if (auto intType = llvm::dyn_cast<llvm::IntegerType>(vecType->getElementType()))
  3342. {
  3343. if (intType->getBitWidth() == 1)
  3344. {
  3345. auto toType = llvm::FixedVectorType::get(llvm::IntegerType::get(*mLLVMContext, 8), vecType->getNumElements());
  3346. result = mIRBuilder->CreateZExt(result, toType);
  3347. }
  3348. }
  3349. }
  3350. SetResult(curId, result);
  3351. }
  3352. }
  3353. else
  3354. {
  3355. llvm::Value* result = NULL;
  3356. switch (intrinsicData->mIntrinsic)
  3357. {
  3358. case BfIRIntrinsic_And:
  3359. result = mIRBuilder->CreateAnd(val0, val1);
  3360. break;
  3361. case BfIRIntrinsic_Add:
  3362. result = mIRBuilder->CreateAdd(val0, val1);
  3363. break;
  3364. case BfIRIntrinsic_Div:
  3365. result = mIRBuilder->CreateSDiv(val0, val1);
  3366. break;
  3367. case BfIRIntrinsic_Eq:
  3368. result = mIRBuilder->CreateICmpEQ(val0, val1);
  3369. break;
  3370. case BfIRIntrinsic_Gt:
  3371. result = mIRBuilder->CreateICmpSGT(val0, val1);
  3372. break;
  3373. case BfIRIntrinsic_GtE:
  3374. result = mIRBuilder->CreateICmpSGE(val0, val1);
  3375. break;
  3376. case BfIRIntrinsic_Lt:
  3377. result = mIRBuilder->CreateICmpSLT(val0, val1);
  3378. break;
  3379. case BfIRIntrinsic_LtE:
  3380. result = mIRBuilder->CreateICmpSLE(val0, val1);
  3381. break;
  3382. case BfIRIntrinsic_Mod:
  3383. result = mIRBuilder->CreateSRem(val0, val1);
  3384. break;
  3385. case BfIRIntrinsic_Mul:
  3386. result = mIRBuilder->CreateMul(val0, val1);
  3387. break;
  3388. case BfIRIntrinsic_Neq:
  3389. result = mIRBuilder->CreateICmpNE(val0, val1);
  3390. break;
  3391. case BfIRIntrinsic_Or:
  3392. result = mIRBuilder->CreateOr(val0, val1);
  3393. break;
  3394. case BfIRIntrinsic_Sub:
  3395. result = mIRBuilder->CreateSub(val0, val1);
  3396. break;
  3397. case BfIRIntrinsic_Xor:
  3398. result = mIRBuilder->CreateXor(val0, val1);
  3399. break;
  3400. default:
  3401. FatalError("Intrinsic argument error");
  3402. }
  3403. if (result != NULL)
  3404. {
  3405. if (auto vecType = llvm::dyn_cast<llvm::FixedVectorType>(result->getType()))
  3406. {
  3407. if (auto intType = llvm::dyn_cast<llvm::IntegerType>(vecType->getElementType()))
  3408. {
  3409. if (intType->getBitWidth() == 1)
  3410. {
  3411. auto toType = llvm::FixedVectorType::get(llvm::IntegerType::get(*mLLVMContext, 8), vecType->getNumElements());
  3412. result = mIRBuilder->CreateZExt(result, toType);
  3413. }
  3414. }
  3415. }
  3416. SetResult(curId, result);
  3417. }
  3418. }
  3419. }
  3420. else if (auto ptrType = llvm::dyn_cast<llvm::PointerType>(args[1].mTypeEx->mLLVMType))
  3421. {
  3422. //auto ptrElemType = ptrType->getElementType();
  3423. auto ptrElemType = GetLLVMPointerElementType(args[1].mTypeEx);
  3424. if (auto arrType = llvm::dyn_cast<llvm::ArrayType>(ptrElemType))
  3425. {
  3426. auto vecType = llvm::FixedVectorType::get(arrType->getArrayElementType(), (uint)arrType->getArrayNumElements());
  3427. auto vecPtrType = vecType->getPointerTo();
  3428. llvm::Value* val0;
  3429. val0 = mIRBuilder->CreateInsertElement(llvm::UndefValue::get(vecType), args[0].mValue, (uint64)0);
  3430. val0 = mIRBuilder->CreateInsertElement(val0, args[0].mValue, (uint64)1);
  3431. val0 = mIRBuilder->CreateInsertElement(val0, args[0].mValue, (uint64)2);
  3432. val0 = mIRBuilder->CreateInsertElement(val0, args[0].mValue, (uint64)3);
  3433. auto ptrVal1 = mIRBuilder->CreateBitCast(args[1].mValue, vecPtrType);
  3434. auto val1 = mIRBuilder->CreateAlignedLoad(vecType, ptrVal1, llvm::MaybeAlign(1));
  3435. switch (intrinsicData->mIntrinsic)
  3436. {
  3437. case BfIRIntrinsic_Div:
  3438. SetResult(curId, mIRBuilder->CreateFDiv(val0, val1));
  3439. break;
  3440. case BfIRIntrinsic_Mod:
  3441. SetResult(curId, mIRBuilder->CreateFRem(val0, val1));
  3442. break;
  3443. default:
  3444. FatalError("Intrinsic argument error");
  3445. }
  3446. }
  3447. }
  3448. else
  3449. {
  3450. FatalError("Intrinsic argument error");
  3451. }
  3452. }
  3453. break;
  3454. case BfIRIntrinsic_Min:
  3455. case BfIRIntrinsic_Max:
  3456. {
  3457. // Get arguments as vectors
  3458. auto val0 = TryToVector(args[0]);
  3459. if (val0 == NULL)
  3460. FatalError("Intrinsic argument error");
  3461. auto val1 = TryToVector(args[1]);
  3462. if (val1 == NULL)
  3463. FatalError("Intrinsic argument error");
  3464. // Make sure both argument types are the same
  3465. auto vecType = llvm::dyn_cast<llvm::VectorType>(val0->getType());
  3466. if (vecType != llvm::dyn_cast<llvm::VectorType>(val1->getType()))
  3467. FatalError("Intrinsic argument error");
  3468. // Make sure the type is not scalable
  3469. if (vecType->getElementCount().isScalable())
  3470. FatalError("Intrinsic argument error");
  3471. // Make sure the element type is either float or double
  3472. auto elemType = vecType->getElementType();
  3473. if (!elemType->isFloatTy() && !elemType->isDoubleTy())
  3474. FatalError("Intrinsic argument error");
  3475. // Get some properties for easier access
  3476. bool isFloat = elemType->isFloatTy();
  3477. bool isMin = intrinsicData->mIntrinsic == BfIRIntrinsic_Min;
  3478. auto elemCount = vecType->getElementCount().getFixedValue();
  3479. // Get the intrinsic function
  3480. const char* funcName;
  3481. if (isFloat)
  3482. {
  3483. if (elemCount == 4)
  3484. {
  3485. funcName = isMin ? "llvm.x86.sse.min.ps" : "llvm.x86.sse.max.ps";
  3486. SetActiveFunctionSimdType(BfIRSimdType_SSE);
  3487. }
  3488. else if (elemCount == 8)
  3489. {
  3490. funcName = isMin ? "llvm.x86.avx.min.ps.256" : "llvm.x86.avx.max.ps.256";
  3491. SetActiveFunctionSimdType(BfIRSimdType_AVX2);
  3492. }
  3493. else if (elemCount == 16)
  3494. {
  3495. funcName = isMin ? "llvm.x86.avx512.min.ps.512" : "llvm.x86.avx512.max.ps.512";
  3496. SetActiveFunctionSimdType(BfIRSimdType_AVX512);
  3497. }
  3498. else
  3499. FatalError("Intrinsic argument error");
  3500. }
  3501. else
  3502. {
  3503. if (elemCount == 2)
  3504. {
  3505. funcName = isMin ? "llvm.x86.sse.min.pd" : "llvm.x86.sse.max.pd";
  3506. SetActiveFunctionSimdType(BfIRSimdType_SSE);
  3507. }
  3508. else if (elemCount == 4)
  3509. {
  3510. funcName = isMin ? "llvm.x86.avx.min.pd.256" : "llvm.x86.avx.max.pd.256";
  3511. SetActiveFunctionSimdType(BfIRSimdType_AVX2);
  3512. }
  3513. else if (elemCount == 8)
  3514. {
  3515. funcName = isMin ? "llvm.x86.avx512.min.pd.512" : "llvm.x86.avx512.max.pd.512";
  3516. SetActiveFunctionSimdType(BfIRSimdType_AVX512);
  3517. }
  3518. else
  3519. FatalError("Intrinsic argument error");
  3520. }
  3521. auto func = mLLVMModule->getOrInsertFunction(funcName, vecType, vecType, vecType);
  3522. // Call intrinsic
  3523. llvm::SmallVector<llvm::Value*, 2> args;
  3524. args.push_back(val0);
  3525. args.push_back(val1);
  3526. SetResult(curId, mIRBuilder->CreateCall(func, args));
  3527. }
  3528. break;
  3529. case BfIRIntrinsic_Cpuid:
  3530. {
  3531. llvm::Type* elemType = llvm::Type::getInt32Ty(*mLLVMContext);
  3532. // Check argument errors
  3533. if (args.size() != 6 || !args[0].mValue->getType()->isIntegerTy(32) || !args[1].mValue->getType()->isIntegerTy(32))
  3534. FatalError("Intrinsic argument error");
  3535. // for (int i = 2; i < 6; i++)
  3536. // {
  3537. // llvm::Type* type = args[i]->getType();
  3538. //
  3539. // if (!type->isPointerTy() || !GetPointerElementType(args[1])->isIntegerTy(32))
  3540. // FatalError("Intrinsic argument error");
  3541. // }
  3542. // Get asm return type
  3543. llvm::SmallVector<llvm::Type*, 4> asmReturnTypes;
  3544. asmReturnTypes.push_back(elemType);
  3545. asmReturnTypes.push_back(elemType);
  3546. asmReturnTypes.push_back(elemType);
  3547. asmReturnTypes.push_back(elemType);
  3548. llvm::Type* returnType = llvm::StructType::get(*mLLVMContext, asmReturnTypes);
  3549. // Get asm function
  3550. llvm::SmallVector<llvm::Type*, 2> funcParams;
  3551. funcParams.push_back(elemType);
  3552. funcParams.push_back(elemType);
  3553. llvm::FunctionType* funcType = llvm::FunctionType::get(returnType, funcParams, false);
  3554. llvm::InlineAsm* func = llvm::InlineAsm::get(funcType, "xchgq %rbx,${1:q}\ncpuid\nxchgq %rbx,${1:q}", "={ax},=r,={cx},={dx},0,2,~{dirflag},~{fpsr},~{flags}", false);
  3555. // Call asm function
  3556. llvm::SmallVector<llvm::Value*, 2> funcArgs;
  3557. funcArgs.push_back(args[0].mValue);
  3558. funcArgs.push_back(args[1].mValue);
  3559. llvm::Value* asmResult = mIRBuilder->CreateCall(func, funcArgs);
  3560. // Store results
  3561. mIRBuilder->CreateStore(mIRBuilder->CreateExtractValue(asmResult, 0), args[2].mValue);
  3562. mIRBuilder->CreateStore(mIRBuilder->CreateExtractValue(asmResult, 1), args[3].mValue);
  3563. mIRBuilder->CreateStore(mIRBuilder->CreateExtractValue(asmResult, 2), args[4].mValue);
  3564. mIRBuilder->CreateStore(mIRBuilder->CreateExtractValue(asmResult, 3), args[5].mValue);
  3565. }
  3566. break;
  3567. case BfIRIntrinsic_Xgetbv:
  3568. {
  3569. if (args.size() != 1 || !args[0].mValue->getType()->isIntegerTy(32))
  3570. FatalError("Intrinsic argument error");
  3571. auto func = mLLVMModule->getOrInsertFunction("llvm.x86.xgetbv", llvm::Type::getInt64Ty(*mLLVMContext), llvm::Type::getInt32Ty(*mLLVMContext));
  3572. SetResult(curId, mIRBuilder->CreateCall(func, args[0].mValue));
  3573. }
  3574. break;
  3575. case BfIRIntrinsic_Not:
  3576. {
  3577. auto val0 = TryToVector(args[0]);
  3578. SetResult(curId, mIRBuilder->CreateNot(val0));
  3579. }
  3580. break;
  3581. case BfIRIntrinsic_Shuffle:
  3582. {
  3583. llvm::SmallVector<int, 8> intMask;
  3584. for (int i = 7; i < (int)intrinsicData->mName.length(); i++)
  3585. intMask.push_back((int)(intrinsicData->mName[i] - '0'));
  3586. auto val0 = TryToVector(args[0]);
  3587. if (val0 != NULL)
  3588. {
  3589. SetResult(curId, mIRBuilder->CreateShuffleVector(val0, val0, intMask));
  3590. }
  3591. else
  3592. {
  3593. FatalError("Intrinsic argument error");
  3594. }
  3595. }
  3596. break;
  3597. case BfIRIntrinsic_Index:
  3598. {
  3599. llvm::Value* gepArgs[] = {
  3600. llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), 0),
  3601. args[1].mValue };
  3602. auto gep = mIRBuilder->CreateInBoundsGEP(GetLLVMPointerElementType(args[0].mTypeEx), args[0].mValue, llvm::ArrayRef(gepArgs));
  3603. if (args.size() >= 3)
  3604. mIRBuilder->CreateStore(args[2].mValue, gep);
  3605. else
  3606. {
  3607. BfIRTypedValue result;
  3608. result.mTypeEx = GetTypeMember(args[0].mTypeEx, 0);
  3609. result.mValue = mIRBuilder->CreateLoad(result.mTypeEx->mLLVMType, gep);
  3610. SetResult(curId, result);
  3611. }
  3612. }
  3613. break;
  3614. case BfIRIntrinsic_AtomicCmpStore:
  3615. case BfIRIntrinsic_AtomicCmpStore_Weak:
  3616. case BfIRIntrinsic_AtomicCmpXChg:
  3617. {
  3618. auto memoryKindConst = llvm::dyn_cast<llvm::ConstantInt>(args[3].mValue);
  3619. if (memoryKindConst == NULL)
  3620. {
  3621. FatalError("Non-constant success ordering on Atomic_CmpXChg");
  3622. break;
  3623. }
  3624. auto memoryKind = (BfIRAtomicOrdering)memoryKindConst->getSExtValue();
  3625. auto successOrdering = llvm::AtomicOrdering::Unordered;
  3626. auto failOrdering = llvm::AtomicOrdering::Unordered;
  3627. switch (memoryKind & BfIRAtomicOrdering_ORDERMASK)
  3628. {
  3629. case BfIRAtomicOrdering_Acquire:
  3630. successOrdering = llvm::AtomicOrdering::Acquire;
  3631. failOrdering = llvm::AtomicOrdering::Acquire;
  3632. break;
  3633. case BfIRAtomicOrdering_AcqRel:
  3634. successOrdering = llvm::AtomicOrdering::AcquireRelease;
  3635. failOrdering = llvm::AtomicOrdering::Acquire;
  3636. break;
  3637. case BfIRAtomicOrdering_Relaxed:
  3638. successOrdering = llvm::AtomicOrdering::Monotonic;
  3639. failOrdering = llvm::AtomicOrdering::Monotonic;
  3640. break;
  3641. case BfIRAtomicOrdering_Release:
  3642. successOrdering = llvm::AtomicOrdering::Release;
  3643. failOrdering = llvm::AtomicOrdering::Monotonic;
  3644. break;
  3645. case BfIRAtomicOrdering_SeqCst:
  3646. successOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
  3647. failOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
  3648. break;
  3649. default:
  3650. Fail("Invalid success ordering on Atomic_CmpXChg");
  3651. break;
  3652. }
  3653. if (args.size() >= 5)
  3654. {
  3655. auto memoryKindConst = llvm::dyn_cast<llvm::ConstantInt>(args[4].mValue);
  3656. if (memoryKindConst == NULL)
  3657. {
  3658. FatalError("Non-constant fail ordering on Atomic_CmpXChg");
  3659. break;
  3660. }
  3661. auto memoryKind = (BfIRAtomicOrdering)memoryKindConst->getSExtValue();
  3662. switch (memoryKind & BfIRAtomicOrdering_ORDERMASK)
  3663. {
  3664. case BfIRAtomicOrdering_Acquire:
  3665. failOrdering = llvm::AtomicOrdering::Acquire;
  3666. break;
  3667. case BfIRAtomicOrdering_Relaxed:
  3668. failOrdering = llvm::AtomicOrdering::Monotonic;
  3669. break;
  3670. case BfIRAtomicOrdering_SeqCst:
  3671. failOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
  3672. break;
  3673. default:
  3674. FatalError("Invalid fail ordering on Atomic_CmpXChg");
  3675. break;
  3676. }
  3677. }
  3678. auto inst = mIRBuilder->CreateAtomicCmpXchg(args[0].mValue, args[1].mValue, args[2].mValue, llvm::MaybeAlign(), successOrdering, failOrdering);
  3679. if (intrinsicData->mIntrinsic == BfIRIntrinsic_AtomicCmpStore_Weak)
  3680. inst->setWeak(true);
  3681. if ((memoryKind & BfIRAtomicOrdering_Volatile) != 0)
  3682. inst->setVolatile(true);
  3683. if (intrinsicData->mIntrinsic == BfIRIntrinsic_AtomicCmpXChg)
  3684. {
  3685. auto prevVal = mIRBuilder->CreateExtractValue(inst, 0);
  3686. SetResult(curId, prevVal);
  3687. }
  3688. else
  3689. {
  3690. auto successVal = mIRBuilder->CreateExtractValue(inst, 1);
  3691. SetResult(curId, successVal);
  3692. }
  3693. }
  3694. break;
  3695. case BfIRIntrinsic_AtomicFence:
  3696. {
  3697. if (args.size() == 0)
  3698. {
  3699. if ((mTargetTriple.GetMachineType() != BfMachineType_x86) && (mTargetTriple.GetMachineType() != BfMachineType_x64))
  3700. {
  3701. Fail("Unable to create compiler barrier on this platform");
  3702. }
  3703. else
  3704. {
  3705. // Compiler barrier
  3706. llvm::SmallVector<llvm::Type*, 8> paramTypes;
  3707. llvm::FunctionType* funcType = llvm::FunctionType::get(llvm::Type::getVoidTy(*mLLVMContext), paramTypes, false);
  3708. auto fenceFunc = llvm::InlineAsm::get(funcType,
  3709. "", "~{memory},~{dirflag},~{fpsr},~{flags}", true, false, llvm::InlineAsm::AD_ATT);
  3710. mIRBuilder->CreateCall(fenceFunc);
  3711. }
  3712. break;
  3713. }
  3714. auto memoryKindConst = llvm::dyn_cast<llvm::ConstantInt>(args[0].mValue);
  3715. if (memoryKindConst == NULL)
  3716. {
  3717. FatalError("Non-constant success ordering on AtomicFence");
  3718. break;
  3719. }
  3720. auto memoryKind = (BfIRAtomicOrdering)memoryKindConst->getSExtValue();
  3721. auto ordering = llvm::AtomicOrdering::SequentiallyConsistent;
  3722. switch (memoryKind & BfIRAtomicOrdering_ORDERMASK)
  3723. {
  3724. case BfIRAtomicOrdering_Acquire:
  3725. ordering = llvm::AtomicOrdering::Acquire;
  3726. break;
  3727. case BfIRAtomicOrdering_AcqRel:
  3728. ordering = llvm::AtomicOrdering::AcquireRelease;
  3729. break;
  3730. case BfIRAtomicOrdering_Release:
  3731. ordering = llvm::AtomicOrdering::Release;
  3732. break;
  3733. case BfIRAtomicOrdering_SeqCst:
  3734. ordering = llvm::AtomicOrdering::SequentiallyConsistent;
  3735. break;
  3736. default:
  3737. Fail("Invalid ordering on atomic operation");
  3738. break;
  3739. }
  3740. mIRBuilder->CreateFence(ordering);
  3741. }
  3742. break;
  3743. case BfIRIntrinsic_AtomicLoad:
  3744. {
  3745. auto memoryKindConst = llvm::dyn_cast<llvm::ConstantInt>(args[1].mValue);
  3746. if (memoryKindConst == NULL)
  3747. {
  3748. FatalError("Non-constant success ordering on AtomicLoad");
  3749. break;
  3750. }
  3751. BfIRTypedValue result;
  3752. result.mTypeEx = GetTypeMember(args[0].mTypeEx, 0);
  3753. auto memoryKind = (BfIRAtomicOrdering)memoryKindConst->getSExtValue();
  3754. auto ptrType = llvm::dyn_cast<llvm::PointerType>(args[0].mValue->getType());
  3755. auto loadInst = mIRBuilder->CreateAlignedLoad(result.mTypeEx->mLLVMType, args[0].mValue, llvm::MaybeAlign((uint)GetLLVMPointerElementType(args[0].mTypeEx)->getPrimitiveSizeInBits() / 8));
  3756. switch (memoryKind & BfIRAtomicOrdering_ORDERMASK)
  3757. {
  3758. case BfIRAtomicOrdering_Acquire:
  3759. loadInst->setAtomic(llvm::AtomicOrdering::Acquire);
  3760. break;
  3761. case BfIRAtomicOrdering_Relaxed:
  3762. loadInst->setAtomic(llvm::AtomicOrdering::Monotonic);
  3763. break;
  3764. case BfIRAtomicOrdering_SeqCst:
  3765. loadInst->setAtomic(llvm::AtomicOrdering::SequentiallyConsistent);
  3766. break;
  3767. default:
  3768. BF_FATAL("BadAtomic");
  3769. }
  3770. if ((memoryKind & BfIRAtomicOrdering_Volatile) != 0)
  3771. loadInst->setVolatile(true);
  3772. result.mValue = loadInst;
  3773. SetResult(curId, result);
  3774. }
  3775. break;
  3776. case BfIRIntrinsic_AtomicStore:
  3777. {
  3778. auto memoryKindConst = llvm::dyn_cast<llvm::ConstantInt>(args[2].mValue);
  3779. if (memoryKindConst == NULL)
  3780. {
  3781. FatalError("Non-constant success ordering on AtomicLoad");
  3782. break;
  3783. }
  3784. auto memoryKind = (BfIRAtomicOrdering)memoryKindConst->getSExtValue();
  3785. auto storeInst = mIRBuilder->CreateAlignedStore(args[1].mValue, args[0].mValue, llvm::MaybeAlign((uint)args[1].mValue->getType()->getPrimitiveSizeInBits() / 8));
  3786. switch (memoryKind & BfIRAtomicOrdering_ORDERMASK)
  3787. {
  3788. case BfIRAtomicOrdering_Relaxed:
  3789. storeInst->setAtomic(llvm::AtomicOrdering::Monotonic);
  3790. break;
  3791. case BfIRAtomicOrdering_Release:
  3792. storeInst->setAtomic(llvm::AtomicOrdering::Release);
  3793. break;
  3794. case BfIRAtomicOrdering_SeqCst:
  3795. storeInst->setAtomic(llvm::AtomicOrdering::SequentiallyConsistent);
  3796. break;
  3797. }
  3798. if ((memoryKind & BfIRAtomicOrdering_Volatile) != 0)
  3799. storeInst->setVolatile(true);
  3800. SetResult(curId, storeInst);
  3801. }
  3802. break;
  3803. case BfIRIntrinsic_AtomicAdd:
  3804. case BfIRIntrinsic_AtomicAnd:
  3805. case BfIRIntrinsic_AtomicMax:
  3806. case BfIRIntrinsic_AtomicMin:
  3807. case BfIRIntrinsic_AtomicNAnd:
  3808. case BfIRIntrinsic_AtomicOr:
  3809. case BfIRIntrinsic_AtomicSub:
  3810. case BfIRIntrinsic_AtomicUMax:
  3811. case BfIRIntrinsic_AtomicUMin:
  3812. case BfIRIntrinsic_AtomicXChg:
  3813. case BfIRIntrinsic_AtomicXor:
  3814. {
  3815. bool isFloat = args[1].mValue->getType()->isFloatingPointTy();
  3816. auto op = llvm::AtomicRMWInst::BinOp::Add;
  3817. switch (intrinsicData->mIntrinsic)
  3818. {
  3819. case BfIRIntrinsic_AtomicAdd:
  3820. op = llvm::AtomicRMWInst::BinOp::Add;
  3821. break;
  3822. case BfIRIntrinsic_AtomicAnd:
  3823. op = llvm::AtomicRMWInst::BinOp::And;
  3824. break;
  3825. case BfIRIntrinsic_AtomicMax:
  3826. op = llvm::AtomicRMWInst::BinOp::Max;
  3827. break;
  3828. case BfIRIntrinsic_AtomicMin:
  3829. op = llvm::AtomicRMWInst::BinOp::Min;
  3830. break;
  3831. case BfIRIntrinsic_AtomicNAnd:
  3832. op = llvm::AtomicRMWInst::BinOp::Nand;
  3833. break;
  3834. case BfIRIntrinsic_AtomicOr:
  3835. op = llvm::AtomicRMWInst::BinOp::Or;
  3836. break;
  3837. case BfIRIntrinsic_AtomicSub:
  3838. op = llvm::AtomicRMWInst::BinOp::Sub;
  3839. break;
  3840. case BfIRIntrinsic_AtomicUMax:
  3841. op = llvm::AtomicRMWInst::BinOp::UMax;
  3842. break;
  3843. case BfIRIntrinsic_AtomicUMin:
  3844. op = llvm::AtomicRMWInst::BinOp::UMin;
  3845. break;
  3846. case BfIRIntrinsic_AtomicXChg:
  3847. op = llvm::AtomicRMWInst::BinOp::Xchg;
  3848. break;
  3849. case BfIRIntrinsic_AtomicXor:
  3850. op = llvm::AtomicRMWInst::BinOp::Xor;
  3851. break;
  3852. default: break;
  3853. }
  3854. auto memoryKindConst = llvm::dyn_cast<llvm::ConstantInt>(args[2].mValue);
  3855. if (memoryKindConst == NULL)
  3856. {
  3857. FatalError("Non-constant ordering on atomic operation");
  3858. break;
  3859. }
  3860. auto memoryKind = (BfIRAtomicOrdering)memoryKindConst->getSExtValue();
  3861. auto ordering = llvm::AtomicOrdering::Unordered;
  3862. switch (memoryKind & BfIRAtomicOrdering_ORDERMASK)
  3863. {
  3864. case BfIRAtomicOrdering_Acquire:
  3865. ordering = llvm::AtomicOrdering::Acquire;
  3866. break;
  3867. case BfIRAtomicOrdering_AcqRel:
  3868. ordering = llvm::AtomicOrdering::AcquireRelease;
  3869. break;
  3870. case BfIRAtomicOrdering_Relaxed:
  3871. ordering = llvm::AtomicOrdering::Monotonic;
  3872. break;
  3873. case BfIRAtomicOrdering_Release:
  3874. ordering = llvm::AtomicOrdering::Release;
  3875. break;
  3876. case BfIRAtomicOrdering_SeqCst:
  3877. ordering = llvm::AtomicOrdering::SequentiallyConsistent;
  3878. break;
  3879. default:
  3880. Fail("Invalid ordering on atomic operation");
  3881. break;
  3882. }
  3883. auto atomicRMW = mIRBuilder->CreateAtomicRMW(op, args[0].mValue, args[1].mValue, llvm::MaybeAlign(), ordering);
  3884. if ((memoryKind & BfIRAtomicOrdering_Volatile) != 0)
  3885. atomicRMW->setVolatile(true);
  3886. llvm::Value* result = atomicRMW;
  3887. if ((memoryKind & BfIRAtomicOrdering_ReturnModified) != 0)
  3888. {
  3889. switch (intrinsicData->mIntrinsic)
  3890. {
  3891. case BfIRIntrinsic_AtomicAdd:
  3892. if (isFloat)
  3893. result = mIRBuilder->CreateFAdd(atomicRMW, args[1].mValue);
  3894. else
  3895. result = mIRBuilder->CreateAdd(atomicRMW, args[1].mValue);
  3896. break;
  3897. case BfIRIntrinsic_AtomicAnd:
  3898. result = mIRBuilder->CreateAnd(atomicRMW, args[1].mValue);
  3899. break;
  3900. case BfIRIntrinsic_AtomicMax:
  3901. case BfIRIntrinsic_AtomicMin:
  3902. case BfIRIntrinsic_AtomicUMax:
  3903. case BfIRIntrinsic_AtomicUMin:
  3904. {
  3905. llvm::Value* cmpVal = NULL;
  3906. switch (intrinsicData->mIntrinsic)
  3907. {
  3908. case BfIRIntrinsic_AtomicMax:
  3909. if (isFloat)
  3910. cmpVal = mIRBuilder->CreateFCmpOGE(atomicRMW, args[1].mValue);
  3911. else
  3912. cmpVal = mIRBuilder->CreateICmpSGE(atomicRMW, args[1].mValue);
  3913. break;
  3914. case BfIRIntrinsic_AtomicMin:
  3915. if (isFloat)
  3916. cmpVal = mIRBuilder->CreateFCmpOLE(atomicRMW, args[1].mValue);
  3917. else
  3918. cmpVal = mIRBuilder->CreateICmpSLE(atomicRMW, args[1].mValue);
  3919. break;
  3920. case BfIRIntrinsic_AtomicUMax:
  3921. cmpVal = mIRBuilder->CreateICmpUGE(atomicRMW, args[1].mValue);
  3922. break;
  3923. case BfIRIntrinsic_AtomicUMin:
  3924. cmpVal = mIRBuilder->CreateICmpULE(atomicRMW, args[1].mValue);
  3925. break;
  3926. default: break;
  3927. }
  3928. result = mIRBuilder->CreateSelect(cmpVal, atomicRMW, args[1].mValue);
  3929. }
  3930. break;
  3931. case BfIRIntrinsic_AtomicNAnd:
  3932. result = mIRBuilder->CreateAnd(atomicRMW, args[1].mValue);
  3933. result = mIRBuilder->CreateNot(result);
  3934. break;
  3935. case BfIRIntrinsic_AtomicOr:
  3936. result = mIRBuilder->CreateOr(atomicRMW, args[1].mValue);
  3937. break;
  3938. case BfIRIntrinsic_AtomicSub:
  3939. if (isFloat)
  3940. result = mIRBuilder->CreateFSub(atomicRMW, args[1].mValue);
  3941. else
  3942. result = mIRBuilder->CreateSub(atomicRMW, args[1].mValue);
  3943. break;
  3944. case BfIRIntrinsic_AtomicXor:
  3945. result = mIRBuilder->CreateXor(atomicRMW, args[1].mValue);
  3946. break;
  3947. case BfIRIntrinsic_AtomicXChg:
  3948. result = args[1].mValue;
  3949. break;
  3950. default: break;
  3951. }
  3952. }
  3953. SetResult(curId, result);
  3954. }
  3955. break;
  3956. case BfIRIntrinsic_Cast:
  3957. {
  3958. BfIRTypedValue result;
  3959. result.mTypeEx = intrinsicData->mReturnType;
  3960. auto arg0Type = args[0].mValue->getType();
  3961. if (arg0Type->isPointerTy())
  3962. {
  3963. if (intrinsicData->mReturnType->mLLVMType->isPointerTy())
  3964. {
  3965. result.mValue = mIRBuilder->CreateBitCast(args[0].mValue, intrinsicData->mReturnType->mLLVMType);
  3966. }
  3967. else
  3968. {
  3969. auto castedRes = mIRBuilder->CreateBitCast(args[0].mValue, intrinsicData->mReturnType->mLLVMType->getPointerTo());
  3970. result.mValue = mIRBuilder->CreateAlignedLoad(intrinsicData->mReturnType->mLLVMType, castedRes, llvm::MaybeAlign(1));
  3971. }
  3972. }
  3973. else if ((arg0Type->isVectorTy()) && (intrinsicData->mReturnType->mLLVMType->isVectorTy()))
  3974. {
  3975. result.mValue = mIRBuilder->CreateBitCast(args[0].mValue, intrinsicData->mReturnType->mLLVMType);
  3976. }
  3977. else
  3978. FatalError("Invalid cast intrinsic values");
  3979. SetResult(curId, result);
  3980. }
  3981. break;
  3982. case BfIRIntrinsic_VAArg:
  3983. {
  3984. auto constInt = llvm::dyn_cast<llvm::ConstantInt>(args[2].mValue);
  3985. auto argType = GetLLVMTypeById((int)constInt->getSExtValue());
  3986. auto vaArgVal = mIRBuilder->CreateVAArg(args[0].mValue, argType);
  3987. auto resultPtr = mIRBuilder->CreateBitCast(args[1].mValue, argType->getPointerTo());
  3988. mIRBuilder->CreateStore(vaArgVal, resultPtr);
  3989. }
  3990. break;
  3991. default:
  3992. FatalError("Unhandled intrinsic");
  3993. }
  3994. break;
  3995. }
  3996. if (auto funcPtr = llvm::dyn_cast<llvm::Function>(func.mValue))
  3997. {
  3998. // if (funcPtr->getName() == "__FAILCALL")
  3999. // {
  4000. // FatalError("__FAILCALL");
  4001. // }
  4002. int intrinId = -1;
  4003. if (mIntrinsicReverseMap.TryGetValue(funcPtr, &intrinId))
  4004. {
  4005. if (intrinId == BfIRIntrinsic_MemSet)
  4006. {
  4007. int align = 1;
  4008. BF_ASSERT(args.size() == 5);
  4009. auto alignConst = llvm::dyn_cast<llvm::ConstantInt>(args[3].mValue);
  4010. if (alignConst != NULL)
  4011. align = (int)alignConst->getSExtValue();
  4012. bool isVolatile = false;
  4013. auto volatileConst = llvm::dyn_cast<llvm::ConstantInt>(args[4].mValue);
  4014. if ((volatileConst != NULL) && (volatileConst->getSExtValue() != 0))
  4015. isVolatile = true;
  4016. CreateMemSet(args[0].mValue, args[1].mValue, args[2].mValue, align, isVolatile);
  4017. break;
  4018. }
  4019. else if ((intrinId == BfIRIntrinsic_MemCpy) || (intrinId == BfIRIntrinsic_MemMove))
  4020. {
  4021. int align = 1;
  4022. BF_ASSERT(args.size() == 5);
  4023. auto alignConst = llvm::dyn_cast<llvm::ConstantInt>(args[3].mValue);
  4024. if (alignConst != NULL)
  4025. align = (int)alignConst->getSExtValue();
  4026. bool isVolatile = false;
  4027. auto volatileConst = llvm::dyn_cast<llvm::ConstantInt>(args[4].mValue);
  4028. if ((volatileConst != NULL) && (volatileConst->getSExtValue() != 0))
  4029. isVolatile = true;
  4030. if (intrinId == BfIRIntrinsic_MemCpy)
  4031. mIRBuilder->CreateMemCpy(args[0].mValue, llvm::MaybeAlign(align), args[1].mValue, llvm::MaybeAlign(align), args[2].mValue, isVolatile);
  4032. else
  4033. mIRBuilder->CreateMemMove(args[0].mValue, llvm::MaybeAlign(align), args[1].mValue, llvm::MaybeAlign(align), args[2].mValue, isVolatile);
  4034. break;
  4035. }
  4036. }
  4037. }
  4038. llvm::Value* val0 = NULL;
  4039. llvm::Value* val1 = NULL;
  4040. if (args.size() > 0)
  4041. {
  4042. val0 = args[0].mValue;
  4043. }
  4044. if (args.size() > 1)
  4045. {
  4046. val1 = args[1].mValue;
  4047. }
  4048. llvm::FunctionType* funcType = NULL;
  4049. if (auto ptrType = llvm::dyn_cast<llvm::PointerType>(func.mValue->getType()))
  4050. funcType = llvm::dyn_cast<llvm::FunctionType>(GetLLVMPointerElementType(func.mTypeEx));
  4051. CmdParamVec<llvm::Value*> llvmArgs;
  4052. for (auto& arg : args)
  4053. llvmArgs.push_back(arg.mValue);
  4054. auto funcTypeEx = GetTypeMember(func.mTypeEx, 0);
  4055. auto returnTypeEx = GetTypeMember(funcTypeEx, 0);
  4056. BfIRTypedValue result;
  4057. result.mTypeEx = returnTypeEx;
  4058. result.mValue = mIRBuilder->CreateCall(funcType, func.mValue, llvmArgs);
  4059. SetResult(curId, result);
  4060. mLastFuncCalled.mValue = result.mValue;
  4061. mLastFuncCalled.mTypeEx = funcTypeEx;
  4062. }
  4063. break;
  4064. case BfIRCmd_SetCallCallingConv:
  4065. {
  4066. CMD_PARAM(llvm::Value*, callInst);
  4067. BfIRCallingConv callingConv = (BfIRCallingConv)mStream->Read();
  4068. BF_ASSERT(llvm::isa<llvm::CallInst>(callInst));
  4069. ((llvm::CallInst*)callInst)->setCallingConv(GetLLVMCallingConv(callingConv, mTargetTriple));
  4070. }
  4071. break;
  4072. case BfIRCmd_SetFuncCallingConv:
  4073. {
  4074. CMD_PARAM(llvm::Function*, func);
  4075. BfIRCallingConv callingConv = (BfIRCallingConv)mStream->Read();
  4076. ((llvm::Function*)func)->setCallingConv(GetLLVMCallingConv(callingConv, mTargetTriple));
  4077. }
  4078. break;
  4079. case BfIRCmd_SetTailCall:
  4080. {
  4081. CMD_PARAM(llvm::Value*, callInst);
  4082. BF_ASSERT(llvm::isa<llvm::CallInst>(callInst));
  4083. ((llvm::CallInst*)callInst)->setTailCall();
  4084. }
  4085. break;
  4086. case BfIRCmd_SetCallAttribute:
  4087. {
  4088. CMD_PARAM(llvm::Value*, callInst);
  4089. CMD_PARAM(int, paramIdx);
  4090. BfIRAttribute attribute = (BfIRAttribute)mStream->Read();
  4091. BF_ASSERT(llvm::isa<llvm::CallInst>(callInst));
  4092. llvm::Attribute::AttrKind attr = llvm::Attribute::None;
  4093. if (attribute == BfIRAttribute_NoReturn)
  4094. attr = llvm::Attribute::NoReturn;
  4095. ((llvm::CallInst*)callInst)->addParamAttr(paramIdx, attr);
  4096. }
  4097. break;
  4098. case BfIRCmd_CreateRet:
  4099. {
  4100. CMD_PARAM(llvm::Value*, val);
  4101. SetResult(curId, mIRBuilder->CreateRet(val));
  4102. }
  4103. break;
  4104. case BfIRCmd_CreateRetVoid:
  4105. {
  4106. mIRBuilder->CreateRetVoid();
  4107. }
  4108. break;
  4109. case BfIRCmd_CreateUnreachable:
  4110. {
  4111. mIRBuilder->CreateUnreachable();
  4112. }
  4113. break;
  4114. case BfIRCmd_Call_AddAttribute:
  4115. {
  4116. CMD_PARAM(BfIRTypedValue, inst);
  4117. CMD_PARAM(int, argIdx);
  4118. BF_ASSERT(inst.mValue == mLastFuncCalled.mValue);
  4119. BfIRAttribute attribute = (BfIRAttribute)mStream->Read();
  4120. auto attr = LLVMMapAttribute(attribute);
  4121. auto callInst = llvm::dyn_cast<llvm::CallInst>(inst.mValue);
  4122. BfIRTypeEx* funcType = mLastFuncCalled.mTypeEx;
  4123. if (attr == llvm::Attribute::StructRet)
  4124. {
  4125. auto elemPtrType = GetTypeMember(funcType, argIdx);
  4126. auto elemType = GetTypeMember(elemPtrType, 0);
  4127. llvm::Attribute sret = llvm::Attribute::getWithStructRetType(*mLLVMContext, elemType->mLLVMType);
  4128. ((llvm::CallInst*)callInst)->addParamAttr(argIdx - 1, sret);
  4129. }
  4130. else
  4131. {
  4132. if (argIdx == -1)
  4133. ((llvm::CallInst*)callInst)->addFnAttr(attr);
  4134. else if (argIdx == 0)
  4135. ((llvm::CallInst*)callInst)->addRetAttr(attr);
  4136. else
  4137. ((llvm::CallInst*)callInst)->addParamAttr(argIdx - 1, attr);
  4138. }
  4139. }
  4140. break;
  4141. case BfIRCmd_Call_AddAttribute1:
  4142. {
  4143. CMD_PARAM(BfIRTypedValue, inst);
  4144. CMD_PARAM(int, argIdx);
  4145. BfIRAttribute attribute = (BfIRAttribute)mStream->Read();
  4146. CMD_PARAM(int, arg);
  4147. BF_ASSERT(inst.mValue == mLastFuncCalled.mValue);
  4148. auto callInst = llvm::dyn_cast<llvm::CallInst>(inst.mValue);
  4149. if (callInst != NULL)
  4150. {
  4151. BfIRTypeEx* funcType = mLastFuncCalled.mTypeEx;
  4152. if (attribute == BfIRAttribute_Dereferencable)
  4153. {
  4154. ((llvm::CallInst*)callInst)->addDereferenceableParamAttr(argIdx - 1, arg);
  4155. }
  4156. else if (attribute == BfIRAttribute_ByVal)
  4157. {
  4158. auto elemPtrType = GetTypeMember(funcType, argIdx);
  4159. auto elemType = GetTypeMember(elemPtrType, 0);
  4160. llvm::Attribute byValAttr = llvm::Attribute::getWithByValType(*mLLVMContext, elemType->mLLVMType);
  4161. llvm::Attribute alignAttr = llvm::Attribute::getWithAlignment(*mLLVMContext, llvm::Align(arg));
  4162. ((llvm::CallInst*)callInst)->addParamAttr(argIdx - 1, byValAttr);
  4163. ((llvm::CallInst*)callInst)->addParamAttr(argIdx - 1, alignAttr);
  4164. }
  4165. }
  4166. }
  4167. break;
  4168. case BfIRCmd_Func_AddAttribute:
  4169. {
  4170. BfIRTypedValue typedValue;
  4171. ReadFunction(typedValue);
  4172. CMD_PARAM(int, argIdx);
  4173. auto func = llvm::dyn_cast<llvm::Function>(typedValue.mValue);
  4174. auto funcType = GetTypeMember(typedValue.mTypeEx, 0);
  4175. BfIRAttribute attribute = (BfIRAttribute)mStream->Read();
  4176. if (attribute == BFIRAttribute_DllImport)
  4177. {
  4178. func->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
  4179. }
  4180. else if (attribute == BFIRAttribute_DllExport)
  4181. {
  4182. func->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
  4183. mHadDLLExport = true;
  4184. }
  4185. else if (attribute == BFIRAttribute_NoFramePointerElim)
  4186. {
  4187. func->addFnAttr("no-frame-pointer-elim", "true");
  4188. }
  4189. else if ((attribute == BFIRAttribute_Constructor) || (attribute == BFIRAttribute_Destructor))
  4190. {
  4191. CmdParamVec<llvm::Type*> members;
  4192. members.push_back(llvm::Type::getInt32Ty(*mLLVMContext));
  4193. members.push_back(func->getType());
  4194. members.push_back(llvm::PointerType::get(*mLLVMContext, 0));
  4195. llvm::StructType* structType = llvm::StructType::get(*mLLVMContext, members);
  4196. CmdParamVec<llvm::Constant*> structVals;
  4197. structVals.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), 0x7FFFFF00));
  4198. structVals.push_back(func);
  4199. structVals.push_back(llvm::ConstantPointerNull::get(llvm::PointerType::get(*mLLVMContext, 0)));
  4200. auto constStruct = llvm::ConstantStruct::get(structType, structVals);
  4201. CmdParamVec<llvm::Constant*> structArrVals;
  4202. structArrVals.push_back(constStruct);
  4203. auto arrTy = llvm::ArrayType::get(structType, 1);
  4204. auto constArr = llvm::ConstantArray::get(arrTy, structArrVals);
  4205. auto globalVariable = new llvm::GlobalVariable(
  4206. *mLLVMModule,
  4207. arrTy,
  4208. false,
  4209. llvm::GlobalValue::AppendingLinkage,
  4210. constArr,
  4211. (attribute == BFIRAttribute_Constructor) ? "llvm.global_ctors" : "llvm.global_dtors",
  4212. NULL, llvm::GlobalValue::NotThreadLocal);
  4213. }
  4214. else
  4215. {
  4216. auto attr = LLVMMapAttribute(attribute);
  4217. if (attr == llvm::Attribute::StructRet)
  4218. {
  4219. auto elemPtrType = GetTypeMember(funcType, argIdx);
  4220. auto elemType = GetTypeMember(elemPtrType, 0);
  4221. llvm::Attribute sret = llvm::Attribute::getWithStructRetType(*mLLVMContext, elemType->mLLVMType);
  4222. func->addParamAttr(argIdx - 1, sret);
  4223. }
  4224. else if (attr != llvm::Attribute::None)
  4225. {
  4226. if (argIdx < 0)
  4227. {
  4228. switch (attr)
  4229. {
  4230. case llvm::Attribute::UWTable:
  4231. {
  4232. llvm::AttrBuilder attrBuilder(*mLLVMContext);
  4233. attrBuilder.addUWTableAttr(llvm::UWTableKind::Default);
  4234. func->addFnAttrs(attrBuilder);
  4235. }
  4236. break;
  4237. default:
  4238. func->addFnAttr(attr);
  4239. }
  4240. }
  4241. else if (argIdx == 0)
  4242. func->addRetAttr(attr);
  4243. else
  4244. func->addParamAttr(argIdx - 1, attr);
  4245. }
  4246. }
  4247. }
  4248. break;
  4249. case BfIRCmd_Func_AddAttribute1:
  4250. {
  4251. BfIRTypedValue typedValue;
  4252. ReadFunction(typedValue);
  4253. CMD_PARAM(int, argIdx);
  4254. auto func = llvm::dyn_cast<llvm::Function>(typedValue.mValue);
  4255. auto funcType = GetTypeMember(typedValue.mTypeEx, 0);
  4256. BfIRAttribute attribute = (BfIRAttribute)mStream->Read();
  4257. CMD_PARAM(int, arg);
  4258. if (attribute == BfIRAttribute_Dereferencable)
  4259. {
  4260. ((llvm::Function*)func)->addDereferenceableParamAttr(argIdx - 1, arg);
  4261. }
  4262. else if (attribute == BfIRAttribute_ByVal)
  4263. {
  4264. auto elemPtrType = GetTypeMember(funcType, argIdx);
  4265. auto elemType = GetTypeMember(elemPtrType, 0);
  4266. auto funcType = func->getFunctionType();
  4267. llvm::Attribute byValAttr = llvm::Attribute::getWithByValType(*mLLVMContext, elemType->mLLVMType);
  4268. llvm::Attribute alignAttr = llvm::Attribute::getWithAlignment(*mLLVMContext, llvm::Align(arg));
  4269. func->addParamAttr(argIdx - 1, byValAttr);
  4270. func->addParamAttr(argIdx - 1, alignAttr);
  4271. }
  4272. }
  4273. break;
  4274. case BfIRCmd_Func_SetParamName:
  4275. {
  4276. CMD_PARAM(llvm::Function*, func);
  4277. CMD_PARAM(int, argIdx);
  4278. CMD_PARAM(String, name);
  4279. if (argIdx > func->arg_size())
  4280. {
  4281. Fail("BfIRCmd_Func_SetParamName argIdx error");
  4282. break;
  4283. }
  4284. auto argItr = func->arg_begin();
  4285. for (int i = 1; i < argIdx; i++)
  4286. ++argItr;
  4287. argItr->setName(name.c_str());
  4288. }
  4289. break;
  4290. case BfIRCmd_Func_DeleteBody:
  4291. {
  4292. CMD_PARAM(llvm::Function*, func);
  4293. BF_ASSERT(llvm::isa<llvm::Function>(func));
  4294. ((llvm::Function*)func)->deleteBody();
  4295. }
  4296. break;
  4297. case BfIRCmd_Func_SafeRename:
  4298. {
  4299. CMD_PARAM(llvm::Function*, func);
  4300. func->setName(llvm::Twine((Beefy::String(func->getName().data()) + StrFormat("__RENAME%d", curId)).c_str()));
  4301. }
  4302. break;
  4303. case BfIRCmd_Func_SafeRenameFrom:
  4304. {
  4305. CMD_PARAM(llvm::Function*, func);
  4306. CMD_PARAM(String, prevName);
  4307. if (String(func->getName().data()) == prevName)
  4308. func->setName(llvm::Twine((Beefy::String(func->getName().data()) + StrFormat("__RENAME%d", curId)).c_str()));
  4309. }
  4310. break;
  4311. case BfIRCmd_Func_SetLinkage:
  4312. {
  4313. CMD_PARAM(llvm::Function*, func);
  4314. BfIRLinkageType linkageType = (BfIRLinkageType)mStream->Read();
  4315. ((llvm::Function*)func)->setLinkage(LLVMMapLinkageType(linkageType));
  4316. }
  4317. break;
  4318. case BfIRCmd_SaveDebugLocation:
  4319. {
  4320. mSavedDebugLocs.push_back(mIRBuilder->getCurrentDebugLocation());
  4321. }
  4322. break;
  4323. case BfIRCmd_RestoreDebugLocation:
  4324. {
  4325. mDebugLoc = mSavedDebugLocs[mSavedDebugLocs.size() - 1];
  4326. mIRBuilder->SetCurrentDebugLocation(mDebugLoc);
  4327. mSavedDebugLocs.pop_back();
  4328. }
  4329. break;
  4330. case BfIRCmd_DupDebugLocation:
  4331. break;
  4332. case BfIRCmd_ClearDebugLocation:
  4333. {
  4334. mDebugLoc = llvm::DebugLoc();
  4335. mIRBuilder->SetCurrentDebugLocation(llvm::DebugLoc());
  4336. }
  4337. break;
  4338. case BfIRCmd_ClearDebugLocationInst:
  4339. {
  4340. CMD_PARAM_NOTRANS(llvm::Value*, instValue);
  4341. BF_ASSERT(llvm::isa<llvm::Instruction>(instValue));
  4342. if (llvm::dyn_cast<llvm::DbgDeclareInst>(instValue))
  4343. {
  4344. printf("BfIRCmd_ClearDebugLocationInst on DbgDeclareInst in %s\n", mModuleName.c_str());
  4345. }
  4346. else
  4347. {
  4348. ((llvm::Instruction*)instValue)->setDebugLoc(llvm::DebugLoc());
  4349. }
  4350. }
  4351. break;
  4352. case BfIRCmd_ClearDebugLocationInstLast:
  4353. {
  4354. llvm::BasicBlock* bb = mIRBuilder->GetInsertBlock();
  4355. if (bb != NULL)
  4356. {
  4357. if (!bb->empty())
  4358. {
  4359. auto& inst = bb->back();
  4360. if (llvm::dyn_cast<llvm::DbgDeclareInst>(&inst))
  4361. {
  4362. printf("BfIRCmd_ClearDebugLocationInstLast on DbgDeclareInst\n");
  4363. }
  4364. else
  4365. {
  4366. inst.setDebugLoc(llvm::DebugLoc());
  4367. }
  4368. }
  4369. }
  4370. }
  4371. break;
  4372. case BfIRCmd_UpdateDebugLocation:
  4373. {
  4374. CMD_PARAM_NOTRANS(llvm::Value*, instValue);
  4375. BF_ASSERT(llvm::isa<llvm::Instruction>(instValue));
  4376. if ((llvm::dyn_cast<llvm::DbgDeclareInst>(instValue)) && (!mIRBuilder->getCurrentDebugLocation()))
  4377. {
  4378. printf("BfIRCmd_UpdateDebugLocation NULL on DbgDeclareInst\n");
  4379. }
  4380. else
  4381. {
  4382. ((llvm::Instruction*)instValue)->setDebugLoc(mIRBuilder->getCurrentDebugLocation());
  4383. }
  4384. }
  4385. break;
  4386. case BfIRCmd_SetCurrentDebugLocation:
  4387. {
  4388. CMD_PARAM(int, line);
  4389. CMD_PARAM(int, column);
  4390. CMD_PARAM(llvm::MDNode*, diScope);
  4391. CMD_PARAM(llvm::MDNode*, diInlinedAt);
  4392. if (line == 0)
  4393. column = 0;
  4394. mCurLine = line;
  4395. mDebugLoc = llvm::DILocation::get(*mLLVMContext, line, column, diScope, diInlinedAt);
  4396. #ifdef _DEBUG
  4397. llvm::DILocation* DL = mDebugLoc;
  4398. if (DL != NULL)
  4399. {
  4400. llvm::Metadata* Parent = DL->getRawScope();
  4401. llvm::DILocalScope* Scope = DL->getInlinedAtScope();
  4402. llvm::DISubprogram* SP = Scope->getSubprogram();
  4403. if (SP != NULL)
  4404. {
  4405. BF_ASSERT(SP->describes(mActiveFunction));
  4406. }
  4407. }
  4408. #endif
  4409. }
  4410. break;
  4411. case BfIRCmd_Nop:
  4412. case BfIRCmd_EnsureInstructionAt:
  4413. AddNop();
  4414. break;
  4415. case BfIRCmd_StatementStart:
  4416. // We only commit the debug loc for statement starts
  4417. mIRBuilder->SetCurrentDebugLocation(mDebugLoc);
  4418. mHasDebugLoc = true;
  4419. break;
  4420. case BfIRCmd_ObjectAccessCheck:
  4421. {
  4422. CMD_PARAM(llvm::Value*, val);
  4423. CMD_PARAM(bool, useAsm);
  4424. auto curLLVMFunc = mActiveFunction;
  4425. auto irBuilder = mIRBuilder;
  4426. if ((mTargetTriple.GetMachineType() != BfMachineType_x86) && (mTargetTriple.GetMachineType() != BfMachineType_x64))
  4427. useAsm = false;
  4428. if (!useAsm)
  4429. {
  4430. mLockedBlocks.Add(irBuilder->GetInsertBlock());
  4431. // This is generates slower code than the inline asm in debug mode, but can optimize well in release
  4432. auto int8Ty = llvm::Type::getInt8Ty(*mLLVMContext);
  4433. auto int8Ptr = irBuilder->CreateBitCast(val, int8Ty->getPointerTo());
  4434. auto int8Val = irBuilder->CreateLoad(int8Ty, int8Ptr);
  4435. auto cmpResult = irBuilder->CreateICmpUGE(int8Val, llvm::ConstantInt::get(int8Ty, 0x80));
  4436. auto failBB = llvm::BasicBlock::Create(*mLLVMContext, "access.fail");
  4437. auto passBB = llvm::BasicBlock::Create(*mLLVMContext, "access.pass");
  4438. irBuilder->CreateCondBr(cmpResult, failBB, passBB);
  4439. curLLVMFunc->insert(curLLVMFunc->end(), failBB);
  4440. irBuilder->SetInsertPoint(failBB);
  4441. auto trapDecl = llvm::Intrinsic::getDeclaration(mLLVMModule, llvm::Intrinsic::trap);
  4442. auto callInst = irBuilder->CreateCall(trapDecl);
  4443. callInst->addFnAttr(llvm::Attribute::NoReturn);
  4444. irBuilder->CreateBr(passBB);
  4445. curLLVMFunc->insert(curLLVMFunc->end(), passBB);
  4446. irBuilder->SetInsertPoint(passBB);
  4447. SetResult(curId, passBB);
  4448. }
  4449. else
  4450. {
  4451. llvm::Type* voidPtrType = llvm::PointerType::get(*mLLVMContext, 0);
  4452. if (mObjectCheckAsm == NULL)
  4453. {
  4454. std::vector<llvm::Type*> paramTypes;
  4455. paramTypes.push_back(voidPtrType);
  4456. auto funcType = llvm::FunctionType::get(llvm::Type::getVoidTy(*mLLVMContext), paramTypes, false);
  4457. String asmStr =
  4458. "cmpb $$128, ($0)\n"
  4459. "jb 1f\n"
  4460. "int $$3\n"
  4461. "1:";
  4462. mObjectCheckAsm = llvm::InlineAsm::get(funcType,
  4463. asmStr.c_str(), "r,~{dirflag},~{fpsr},~{flags}", true,
  4464. false, llvm::InlineAsm::AD_ATT);
  4465. }
  4466. llvm::SmallVector<llvm::Value*, 1> llvmArgs;
  4467. llvmArgs.push_back(mIRBuilder->CreateBitCast(val, voidPtrType));
  4468. llvm::CallInst* callInst = irBuilder->CreateCall(mObjectCheckAsm, llvmArgs);
  4469. callInst->addFnAttr(llvm::Attribute::NoUnwind);
  4470. SetResult(curId, mIRBuilder->GetInsertBlock());
  4471. }
  4472. }
  4473. break;
  4474. case BfIRCmd_DbgInit:
  4475. {
  4476. mDIBuilder = new llvm::DIBuilder(*mLLVMModule);
  4477. }
  4478. break;
  4479. case BfIRCmd_DbgFinalize:
  4480. {
  4481. for (auto& typeEntryPair : mTypes)
  4482. {
  4483. auto& typeEntry = typeEntryPair.mValue;
  4484. if (typeEntry.mInstDIType != NULL)
  4485. typeEntry.mInstDIType->resolveCycles();
  4486. }
  4487. mDIBuilder->finalize();
  4488. }
  4489. break;
  4490. case BfIRCmd_DbgCreateCompileUnit:
  4491. {
  4492. CMD_PARAM(int, lang);
  4493. CMD_PARAM(String, fileName);
  4494. CMD_PARAM(String, directory);
  4495. CMD_PARAM(String, producer);
  4496. CMD_PARAM(bool, isOptimized);
  4497. CMD_PARAM(String, flags);
  4498. CMD_PARAM(int, runtimeVer);
  4499. CMD_PARAM(bool, linesOnly);
  4500. auto diFile = mDIBuilder->createFile(fileName.c_str(), directory.c_str());
  4501. mDICompileUnit = mDIBuilder->createCompileUnit(lang, diFile, producer.c_str(), isOptimized, flags.c_str(), runtimeVer, "", linesOnly ? llvm::DICompileUnit::LineTablesOnly : llvm::DICompileUnit::FullDebug);
  4502. SetResult(curId, mDICompileUnit);
  4503. }
  4504. break;
  4505. case BfIRCmd_DbgCreateFile:
  4506. {
  4507. CMD_PARAM(String, fileName);
  4508. CMD_PARAM(String, directory);
  4509. CMD_PARAM(Val128, md5Hash);
  4510. char hashStr[64];
  4511. for (int i = 0; i < 16; i++)
  4512. sprintf(&hashStr[i * 2], "%.2x", ((uint8*)&md5Hash)[i]);
  4513. SetResult(curId, mDIBuilder->createFile(fileName.c_str(), directory.c_str(),
  4514. llvm::DIFile::ChecksumInfo<llvm::StringRef>(llvm::DIFile::CSK_MD5, hashStr)));
  4515. }
  4516. break;
  4517. case BfIRCmd_ConstValueI64:
  4518. {
  4519. CMD_PARAM(int64, val);
  4520. SetResult(curId, mDIBuilder->createConstantValueExpression((uint64)val));
  4521. }
  4522. break;
  4523. case BfIRCmd_DbgGetCurrentLocation:
  4524. {
  4525. auto debugLoc = mIRBuilder->getCurrentDebugLocation();
  4526. if (!debugLoc)
  4527. debugLoc = mDebugLoc;
  4528. SetResult(curId, debugLoc);
  4529. }
  4530. break;
  4531. case BfIRCmd_DbgSetType:
  4532. {
  4533. CMD_PARAM(int, typeId);
  4534. CMD_PARAM(llvm::MDNode*, type);
  4535. auto& typeEntry = GetTypeEntry(typeId);
  4536. typeEntry.mDIType = (llvm::DIType*)type;
  4537. if (typeEntry.mInstDIType == NULL)
  4538. typeEntry.mInstDIType = (llvm::DIType*)type;
  4539. }
  4540. break;
  4541. case BfIRCmd_DbgSetInstType:
  4542. {
  4543. CMD_PARAM(int, typeId);
  4544. CMD_PARAM(llvm::MDNode*, type);
  4545. GetTypeEntry(typeId).mInstDIType = (llvm::DIType*)type;
  4546. }
  4547. break;
  4548. case BfIRCmd_DbgGetType:
  4549. {
  4550. CMD_PARAM(int, typeId);
  4551. SetResult(curId, GetTypeEntry(typeId).mDIType);
  4552. }
  4553. break;
  4554. case BfIRCmd_DbgGetTypeInst:
  4555. {
  4556. CMD_PARAM(int, typeId);
  4557. SetResult(curId, GetTypeEntry(typeId).mInstDIType);
  4558. }
  4559. break;
  4560. case BfIRCmd_DbgTrackDITypes:
  4561. {
  4562. CMD_PARAM(int, typeId);
  4563. auto& typeEntry = GetTypeEntry(typeId);
  4564. if (typeEntry.mDIType != NULL)
  4565. llvm::MetadataTracking::track(*(llvm::Metadata**)&typeEntry.mDIType);
  4566. if (typeEntry.mInstDIType != NULL)
  4567. llvm::MetadataTracking::track(*(llvm::Metadata**)&typeEntry.mInstDIType);
  4568. }
  4569. break;
  4570. case BfIRCmd_DbgCreateNamespace:
  4571. {
  4572. CMD_PARAM(llvm::MDNode*, scope);
  4573. CMD_PARAM(String, name);
  4574. CMD_PARAM(llvm::MDNode*, file);
  4575. CMD_PARAM(int, lineNum);
  4576. BF_ASSERT(file != NULL);
  4577. SetResult(curId, mDIBuilder->createNameSpace((llvm::DIScope*)scope, name.c_str(), true));
  4578. }
  4579. break;
  4580. case BfIRCmd_DbgCreateImportedModule:
  4581. {
  4582. CMD_PARAM(llvm::MDNode*, context);
  4583. CMD_PARAM(llvm::MDNode*, namespaceNode);
  4584. CMD_PARAM(int, lineNum);
  4585. //SetResult(curId, mDIBuilder->createImportedModule((llvm::DIScope*)context, (llvm::DINamespace*)namespaceNode, lineNum));
  4586. }
  4587. break;
  4588. case BfIRCmd_DbgCreateBasicType:
  4589. {
  4590. CMD_PARAM(String, name);
  4591. CMD_PARAM(int64, sizeInBits);
  4592. CMD_PARAM(int64, alignInBits);
  4593. CMD_PARAM(int, encoding);
  4594. SetResult(curId, mDIBuilder->createBasicType(name.c_str(), sizeInBits, encoding));
  4595. }
  4596. break;
  4597. case BfIRCmd_DbgCreateStructType:
  4598. {
  4599. CMD_PARAM(llvm::MDNode*, context);
  4600. CMD_PARAM(String, name);
  4601. CMD_PARAM(llvm::MDNode*, file);
  4602. CMD_PARAM(int, lineNum);
  4603. CMD_PARAM(int64, sizeInBits);
  4604. CMD_PARAM(int64, alignInBits);
  4605. CMD_PARAM(int, flags);
  4606. CMD_PARAM(llvm::MDNode*, derivedFrom);
  4607. CMD_PARAM(CmdParamVec<llvm::Metadata*>, members);
  4608. auto diMembersArray = mDIBuilder->getOrCreateArray(members);
  4609. BF_ASSERT(file != NULL);
  4610. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4611. auto mdStruct = mDIBuilder->createStructType((llvm::DIScope*)context, name.c_str(), (llvm::DIFile*)file, lineNum, sizeInBits, (uint32)alignInBits, diFlags, (llvm::DIType*)derivedFrom, diMembersArray);
  4612. SetResult(curId, mdStruct);
  4613. //OutputDebugStrF("BfIRCmd_DbgCreateStructType %p\n", mdStruct);
  4614. }
  4615. break;
  4616. case BfIRCmd_DbgCreateEnumerationType:
  4617. {
  4618. CMD_PARAM(llvm::MDNode*, context);
  4619. CMD_PARAM(String, name);
  4620. CMD_PARAM(llvm::MDNode*, file);
  4621. CMD_PARAM(int, lineNum);
  4622. CMD_PARAM(int64, sizeInBits);
  4623. CMD_PARAM(int64, alignInBits);
  4624. CMD_PARAM(CmdParamVec<llvm::Metadata*>, members);
  4625. CMD_PARAM(llvm::MDNode*, underlyingType);
  4626. auto diMembersArray = mDIBuilder->getOrCreateArray(members);
  4627. /*static int typeIdx = 0;
  4628. if (name == "TypeCode")
  4629. name += StrFormat("_%d", typeIdx);
  4630. typeIdx++;*/
  4631. BF_ASSERT(file != NULL);
  4632. auto enumType = mDIBuilder->createEnumerationType((llvm::DIScope*)context, name.c_str(), (llvm::DIFile*)file, lineNum, sizeInBits, (uint32)alignInBits, diMembersArray, (llvm::DIType*)underlyingType);
  4633. SetResult(curId, enumType);
  4634. //OutputDebugStrF("BfIRCmd_DbgCreateEnumerationType %p\n", enumType);
  4635. }
  4636. break;
  4637. case BfIRCmd_DbgCreatePointerType:
  4638. {
  4639. CMD_PARAM(llvm::MDNode*, diType);
  4640. SetResult(curId, mDIBuilder->createPointerType((llvm::DIType*)diType, mPtrSize*8, (uint32)mPtrSize * 8));
  4641. }
  4642. break;
  4643. case BfIRCmd_DbgCreateReferenceType:
  4644. {
  4645. CMD_PARAM(llvm::MDNode*, diType);
  4646. SetResult(curId, mDIBuilder->createReferenceType(llvm::dwarf::DW_TAG_reference_type, (llvm::DIType*)diType));
  4647. }
  4648. break;
  4649. case BfIRCmd_DbgCreateConstType:
  4650. {
  4651. CMD_PARAM(llvm::MDNode*, diType);
  4652. SetResult(curId, mDIBuilder->createQualifiedType(llvm::dwarf::DW_TAG_const_type, (llvm::DIType*)diType));
  4653. }
  4654. break;
  4655. case BfIRCmd_DbgCreateArtificialType:
  4656. {
  4657. CMD_PARAM(llvm::MDNode*, diType);
  4658. SetResult(curId, mDIBuilder->createArtificialType((llvm::DIType*)diType));
  4659. }
  4660. break;
  4661. case BfIRCmd_DbgCreateArrayType:
  4662. {
  4663. CMD_PARAM(int64, sizeInBits);
  4664. CMD_PARAM(int64, alignInBits);
  4665. CMD_PARAM(llvm::MDNode*, elementType);
  4666. CMD_PARAM(int64, numElements);
  4667. llvm::SmallVector<llvm::Metadata*, 1> diSizeVec;
  4668. diSizeVec.push_back(mDIBuilder->getOrCreateSubrange(0, numElements));
  4669. auto diSizeArray = mDIBuilder->getOrCreateArray(diSizeVec);
  4670. SetResult(curId, mDIBuilder->createArrayType(sizeInBits, (uint32)alignInBits, (llvm::DIType*)elementType, diSizeArray));
  4671. }
  4672. break;
  4673. case BfIRCmd_DbgCreateReplaceableCompositeType:
  4674. {
  4675. CMD_PARAM(int, tag);
  4676. CMD_PARAM(String, name);
  4677. CMD_PARAM(llvm::MDNode*, scope);
  4678. CMD_PARAM(llvm::MDNode*, file);
  4679. CMD_PARAM(int, line);
  4680. CMD_PARAM(int64, sizeInBits);
  4681. CMD_PARAM(int64, alignInBits);
  4682. CMD_PARAM(int, flags);
  4683. BF_ASSERT(file != NULL);
  4684. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4685. SetResult(curId, mDIBuilder->createReplaceableCompositeType(tag, name.c_str(), (llvm::DIScope*)scope, (llvm::DIFile*)file, line, 0, sizeInBits, (uint32)alignInBits, diFlags));
  4686. }
  4687. break;
  4688. case BfIRCmd_DbgCreateForwardDecl:
  4689. {
  4690. CMD_PARAM(int, tag);
  4691. CMD_PARAM(String, name);
  4692. CMD_PARAM(llvm::MDNode*, scope);
  4693. CMD_PARAM(llvm::MDNode*, file);
  4694. CMD_PARAM(int, line);
  4695. BF_ASSERT(file != NULL);
  4696. auto diType = mDIBuilder->createForwardDecl(tag, name.c_str(), (llvm::DIScope*)scope, (llvm::DIFile*)file, line);
  4697. SetResult(curId, diType);
  4698. }
  4699. break;
  4700. case BfIRCmd_DbgCreateSizedForwardDecl:
  4701. {
  4702. CMD_PARAM(int, tag);
  4703. CMD_PARAM(String, name);
  4704. CMD_PARAM(llvm::MDNode*, scope);
  4705. CMD_PARAM(llvm::MDNode*, file);
  4706. CMD_PARAM(int, line);
  4707. CMD_PARAM(int64, sizeInBits);
  4708. CMD_PARAM(int64, alignInBits);
  4709. BF_ASSERT(file != NULL);
  4710. SetResult(curId, mDIBuilder->createForwardDecl(tag, name.c_str(), (llvm::DIScope*)scope, (llvm::DIFile*)file, line, 0, sizeInBits, (uint32)alignInBits));
  4711. }
  4712. break;
  4713. case BeIRCmd_DbgSetTypeSize:
  4714. {
  4715. CMD_PARAM(llvm::MDNode*, mdType);
  4716. CMD_PARAM(int64, sizeInBits);
  4717. CMD_PARAM(int64, alignInBits);
  4718. class DIMutType : public llvm::DIType
  4719. {
  4720. public:
  4721. void Resize(int64 newSize, int32 newAlign)
  4722. {
  4723. init(getLine(), newSize, newAlign, getOffsetInBits(), getFlags());
  4724. }
  4725. };
  4726. auto diType = (DIMutType*)mdType;
  4727. diType->Resize(sizeInBits, (int32)alignInBits);
  4728. }
  4729. break;
  4730. case BfIRCmd_DbgReplaceAllUses:
  4731. {
  4732. CMD_PARAM(llvm::MDNode*, diPrevNode);
  4733. CMD_PARAM(llvm::MDNode*, diNewNode);
  4734. diPrevNode->replaceAllUsesWith(diNewNode);
  4735. }
  4736. break;
  4737. case BfIRCmd_DbgDeleteTemporary:
  4738. {
  4739. CMD_PARAM(llvm::MDNode*, diNode);
  4740. llvm::MDNode::deleteTemporary(diNode);
  4741. }
  4742. break;
  4743. case BfIRCmd_DbgMakePermanent:
  4744. {
  4745. CMD_PARAM(llvm::MDNode*, diNode);
  4746. CMD_PARAM(llvm::MDNode*, diBaseType);
  4747. CMD_PARAM(CmdParamVec<llvm::Metadata*>, members);
  4748. llvm::MDNode* newNode = diNode;
  4749. if (auto diComposite = llvm::dyn_cast<llvm::DICompositeType>(diNode))
  4750. {
  4751. //diComposite->getBaseType()
  4752. if (diBaseType != NULL)
  4753. {
  4754. // It's unfortunate we have to hard-code the '3' here
  4755. diComposite->replaceOperandWith(3, diBaseType);
  4756. BF_ASSERT(diComposite->getBaseType() == diBaseType);
  4757. }
  4758. if (members.size() != 0)
  4759. {
  4760. llvm::DINodeArray elements = mDIBuilder->getOrCreateArray(members);
  4761. mDIBuilder->replaceArrays(diComposite, elements);
  4762. }
  4763. newNode = llvm::MDNode::replaceWithPermanent(llvm::TempDICompositeType(diComposite));
  4764. }
  4765. /*else if (auto diEnumerator = llvm::dyn_cast<llvm::DIEnumerator>(diNode))
  4766. {
  4767. if (members.size() != 0)
  4768. {
  4769. llvm::DINodeArray elements = mDIBuilder->getOrCreateArray(diNode);
  4770. mDIBuilder->set(diComposite, elements);
  4771. }
  4772. newNode = llvm::MDNode::replaceWithPermanent(llvm::TempDIEnumerator(diEnumerator));
  4773. }*/
  4774. SetResult(curId, newNode);
  4775. break;
  4776. }
  4777. case BfIRCmd_CreateEnumerator:
  4778. {
  4779. CMD_PARAM(String, name);
  4780. CMD_PARAM(int64, val);
  4781. SetResult(curId, mDIBuilder->createEnumerator(name.c_str(), val));
  4782. }
  4783. break;
  4784. case BfIRCmd_DbgCreateMemberType:
  4785. {
  4786. CMD_PARAM(llvm::MDNode*, scope);
  4787. CMD_PARAM(String, name);
  4788. CMD_PARAM(llvm::MDNode*, file);
  4789. CMD_PARAM(int, lineNumber);
  4790. CMD_PARAM(int64, sizeInBits);
  4791. CMD_PARAM(int64, alignInBits);
  4792. CMD_PARAM(int64, offsetInBits);
  4793. CMD_PARAM(int, flags);
  4794. CMD_PARAM(llvm::MDNode*, type);
  4795. BF_ASSERT(file != NULL);
  4796. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4797. /*Beefy::debug_ostream os;
  4798. os << "BfIRCmd_DbgCreateMemberType " << name.c_str() << "\n";
  4799. scope->print(os);
  4800. os << "\n";
  4801. type->print(os);
  4802. os << "\n";
  4803. os.flush();*/
  4804. const char* namePtr = name.c_str();
  4805. if (name.IsEmpty())
  4806. namePtr = NULL;
  4807. auto member = mDIBuilder->createMemberType((llvm::DIScope*)scope, namePtr, (llvm::DIFile*)file, lineNumber, sizeInBits, (uint32)alignInBits, offsetInBits, diFlags, (llvm::DIType*)type);
  4808. SetResult(curId, member);
  4809. //OutputDebugStrF("BfIRCmd_DbgCreateMemberType = %p\n", member);
  4810. }
  4811. break;
  4812. case BfIRCmd_DbgStaticCreateMemberType:
  4813. {
  4814. CMD_PARAM(llvm::MDNode*, scope);
  4815. CMD_PARAM(String, name);
  4816. CMD_PARAM(llvm::MDNode*, file);
  4817. CMD_PARAM(int, lineNumber);
  4818. CMD_PARAM(llvm::MDNode*, type);
  4819. CMD_PARAM(int, flags);
  4820. CMD_PARAM(llvm::Constant*, val);
  4821. BF_ASSERT(file != NULL);
  4822. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4823. /*Beefy::debug_ostream os;
  4824. os << "BfIRCmd_DbgStaticCreateMemberType " << name.c_str() << "\n";
  4825. scope->print(os);
  4826. os << "\n";
  4827. type->print(os);
  4828. os << "\n";
  4829. os.flush();*/
  4830. auto member = mDIBuilder->createStaticMemberType((llvm::DIScope*)scope, name.c_str(), (llvm::DIFile*)file, lineNumber, (llvm::DIType*)type, diFlags, val, llvm::dwarf::DW_TAG_member);
  4831. SetResult(curId, member);
  4832. //OutputDebugStrF("BfIRCmd_DbgStaticCreateMemberType = %p\n", member);
  4833. }
  4834. break;
  4835. case BfIRCmd_DbgCreateInheritance:
  4836. {
  4837. CMD_PARAM(llvm::MDNode*, type);
  4838. CMD_PARAM(llvm::MDNode*, baseType);
  4839. CMD_PARAM(int64, baseOffset);
  4840. CMD_PARAM(int, flags);
  4841. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4842. auto member = mDIBuilder->createInheritance((llvm::DIType*)type, (llvm::DIType*)baseType, baseOffset, 0, diFlags);
  4843. SetResult(curId, member);
  4844. //OutputDebugStrF("BfIRCmd_DbgCreateInheritance = %p\n", member);
  4845. }
  4846. break;
  4847. case BfIRCmd_DbgCreateMethod:
  4848. {
  4849. CMD_PARAM(llvm::MDNode*, context);
  4850. CMD_PARAM(String, name);
  4851. CMD_PARAM(String, linkageName);
  4852. CMD_PARAM(llvm::MDNode*, file);
  4853. CMD_PARAM(int, lineNum);
  4854. CMD_PARAM(llvm::MDNode*, type);
  4855. CMD_PARAM(bool, isLocalToUnit);
  4856. CMD_PARAM(bool, isDefinition);
  4857. CMD_PARAM(int, vk);
  4858. CMD_PARAM(int, vIndex);
  4859. CMD_PARAM(llvm::MDNode*, vTableHolder);
  4860. CMD_PARAM(int, flags);
  4861. CMD_PARAM(bool, isOptimized);
  4862. CMD_PARAM(llvm::Value*, fn);
  4863. CMD_PARAM(CmdParamVec<llvm::MDNode*>, genericArgs);
  4864. CMD_PARAM(CmdParamVec<llvm::Constant*>, genericConstValueArgs);
  4865. BF_ASSERT(file != NULL);
  4866. llvm::DITemplateParameterArray templateParamArr = NULL;
  4867. llvm::DINodeArray templateParamNodes;
  4868. if (genericArgs.size() != 0)
  4869. {
  4870. llvm::SmallVector<llvm::Metadata*, 16> templateParams;
  4871. for (int i = 0; i < (int)genericArgs.size(); i++)
  4872. {
  4873. auto genericArg = (llvm::DIType*)genericArgs[i];
  4874. String name = StrFormat("T%d", i);
  4875. llvm::Constant* constant = NULL;
  4876. if (i < genericConstValueArgs.size())
  4877. constant = genericConstValueArgs[i];
  4878. if (constant != NULL)
  4879. templateParams.push_back(mDIBuilder->createTemplateValueParameter(mDICompileUnit, name.c_str(), genericArg, false, constant));
  4880. else
  4881. templateParams.push_back(mDIBuilder->createTemplateTypeParameter(mDICompileUnit, name.c_str(), genericArg, false));
  4882. }
  4883. templateParamNodes = mDIBuilder->getOrCreateArray(templateParams);
  4884. templateParamArr = templateParamNodes.get();
  4885. }
  4886. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4887. llvm::DISubprogram::DISPFlags dispFlags = llvm::DISubprogram::DISPFlags::SPFlagZero;
  4888. if (isLocalToUnit)
  4889. dispFlags = (llvm::DISubprogram::DISPFlags)(dispFlags | llvm::DISubprogram::DISPFlags::SPFlagLocalToUnit);
  4890. if (isDefinition)
  4891. dispFlags = (llvm::DISubprogram::DISPFlags)(dispFlags | llvm::DISubprogram::DISPFlags::SPFlagDefinition);
  4892. if (isOptimized)
  4893. dispFlags = (llvm::DISubprogram::DISPFlags)(dispFlags | llvm::DISubprogram::DISPFlags::SPFlagOptimized);
  4894. if (vk != 0)
  4895. dispFlags = (llvm::DISubprogram::DISPFlags)(dispFlags | llvm::DISubprogram::DISPFlags::SPFlagVirtual);
  4896. auto diSubProgram = mDIBuilder->createMethod((llvm::DIScope*)context, name.c_str(), linkageName.c_str(), (llvm::DIFile*)file, lineNum,
  4897. (llvm::DISubroutineType*)type, vIndex, 0, (llvm::DIType*)vTableHolder, diFlags, dispFlags, templateParamArr);
  4898. if (fn != NULL)
  4899. ((llvm::Function*)fn)->setSubprogram(diSubProgram);
  4900. SetResult(curId, diSubProgram);
  4901. //OutputDebugStrF("BfIRCmd_DbgCreateMethod = %p\n", diSubProgram);
  4902. }
  4903. break;
  4904. case BfIRCmd_DbgCreateFunction:
  4905. {
  4906. CMD_PARAM(llvm::MDNode*, context);
  4907. CMD_PARAM(String, name);
  4908. CMD_PARAM(String, linkageName);
  4909. CMD_PARAM(llvm::MDNode*, file);
  4910. CMD_PARAM(int, lineNum);
  4911. CMD_PARAM(llvm::MDNode*, type);
  4912. CMD_PARAM(bool, isLocalToUnit);
  4913. CMD_PARAM(bool, isDefinition);
  4914. CMD_PARAM(int, scopeLine);
  4915. CMD_PARAM(int, flags);
  4916. CMD_PARAM(bool, isOptimized);
  4917. CMD_PARAM(llvm::Value*, fn);
  4918. BF_ASSERT(file != NULL);
  4919. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4920. llvm::DISubprogram::DISPFlags dispFlags = llvm::DISubprogram::DISPFlags::SPFlagZero;
  4921. if (isLocalToUnit)
  4922. dispFlags = (llvm::DISubprogram::DISPFlags)(dispFlags | llvm::DISubprogram::DISPFlags::SPFlagLocalToUnit);
  4923. if (isDefinition)
  4924. dispFlags = (llvm::DISubprogram::DISPFlags)(dispFlags | llvm::DISubprogram::DISPFlags::SPFlagDefinition);
  4925. if (isOptimized)
  4926. dispFlags = (llvm::DISubprogram::DISPFlags)(dispFlags | llvm::DISubprogram::DISPFlags::SPFlagOptimized);
  4927. auto diSubProgram = mDIBuilder->createFunction((llvm::DIScope*)context, name.c_str(), linkageName.c_str(), (llvm::DIFile*)file, lineNum,
  4928. (llvm::DISubroutineType*)type, scopeLine, diFlags, dispFlags);
  4929. if (fn != NULL)
  4930. ((llvm::Function*)fn)->setSubprogram(diSubProgram);
  4931. SetResult(curId, diSubProgram);
  4932. //OutputDebugStrF("BfIRCmd_DbgCreateFunction = %p\n", diSubProgram);
  4933. }
  4934. break;
  4935. case BfIRCmd_DbgCreateParameterVariable:
  4936. {
  4937. CMD_PARAM(llvm::MDNode*, scope);
  4938. CMD_PARAM(String, name);
  4939. CMD_PARAM(int, argNo);
  4940. CMD_PARAM(llvm::MDNode*, file);
  4941. CMD_PARAM(int, lineNum);
  4942. CMD_PARAM(llvm::MDNode*, type);
  4943. CMD_PARAM(bool, alwaysPreserve);
  4944. CMD_PARAM(int, flags);
  4945. BF_ASSERT(file != NULL);
  4946. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)flags;
  4947. SetResult(curId, mDIBuilder->createParameterVariable((llvm::DIScope*)scope, name.c_str(), argNo, (llvm::DIFile*)file, lineNum, (llvm::DIType*)type,
  4948. alwaysPreserve, diFlags));
  4949. }
  4950. break;
  4951. case BfIRCmd_DbgCreateSubroutineType:
  4952. {
  4953. CMD_PARAM(CmdParamVec<llvm::Metadata*>, elements);
  4954. auto diArray = mDIBuilder->getOrCreateTypeArray(elements);
  4955. SetResult(curId, mDIBuilder->createSubroutineType(diArray));
  4956. }
  4957. break;
  4958. case BfIRCmd_DbgCreateAutoVariable:
  4959. {
  4960. CMD_PARAM(llvm::MDNode*, scope);
  4961. CMD_PARAM(String, name);
  4962. CMD_PARAM(llvm::MDNode*, file);
  4963. CMD_PARAM(int, lineNo);
  4964. CMD_PARAM(llvm::MDNode*, type);
  4965. CMD_PARAM(int, initType);
  4966. BF_ASSERT(file != NULL);
  4967. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)0;
  4968. auto loc = mIRBuilder->getCurrentDebugLocation();
  4969. auto dbgLoc = loc.getAsMDNode();
  4970. SetResult(curId, mDIBuilder->createAutoVariable((llvm::DIScope*)scope, name.c_str(), (llvm::DIFile*)file, lineNo, (llvm::DIType*)type, false, diFlags));
  4971. }
  4972. break;
  4973. case BfIRCmd_DbgInsertValueIntrinsic:
  4974. {
  4975. CMD_PARAM(llvm::Value*, val);
  4976. CMD_PARAM(llvm::MDNode*, varInfo);
  4977. auto diVariable = (llvm::DILocalVariable*)varInfo;
  4978. if (val == NULL)
  4979. {
  4980. val = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), 0);
  4981. }
  4982. else if (mIsCodeView)
  4983. {
  4984. if (auto constant = llvm::dyn_cast<llvm::Constant>(val))
  4985. {
  4986. int64 writeVal = 0;
  4987. if (auto constantInt = llvm::dyn_cast<llvm::ConstantInt>(val))
  4988. {
  4989. writeVal = constantInt->getSExtValue();
  4990. }
  4991. auto nameRef = diVariable->getName();
  4992. if (writeVal < 0)
  4993. diVariable->replaceOperandWith(1, llvm::MDString::get(*mLLVMContext, (String(nameRef.data()) + StrFormat("$_%llu", -writeVal)).c_str()));
  4994. else
  4995. diVariable->replaceOperandWith(1, llvm::MDString::get(*mLLVMContext, (String(nameRef.data()) + StrFormat("$%llu", writeVal)).c_str()));
  4996. }
  4997. }
  4998. mDIBuilder->insertDbgValueIntrinsic(val, diVariable, mDIBuilder->createExpression(),
  4999. mIRBuilder->getCurrentDebugLocation(), (llvm::BasicBlock*)mIRBuilder->GetInsertBlock());
  5000. }
  5001. break;
  5002. case BfIRCmd_DbgInsertDeclare:
  5003. {
  5004. CMD_PARAM(llvm::Value*, val);
  5005. CMD_PARAM(llvm::MDNode*, varInfo);
  5006. CMD_PARAM(llvm::Value*, insertBefore);
  5007. llvm::Instruction* insertBeforeInst = NULL;
  5008. if (insertBefore != NULL)
  5009. insertBeforeInst = llvm::dyn_cast<llvm::Instruction>(insertBefore);
  5010. // Protect against lack of debug location
  5011. if (mIRBuilder->getCurrentDebugLocation())
  5012. {
  5013. if (insertBeforeInst != NULL)
  5014. {
  5015. auto& dbgResult = mDIBuilder->insertDeclare(val, (llvm::DILocalVariable*)varInfo, mDIBuilder->createExpression(),
  5016. mIRBuilder->getCurrentDebugLocation(), insertBeforeInst);
  5017. bool isA = dbgResult.is<llvm::Instruction*>();
  5018. auto inst = dbgResult.get<llvm::Instruction*>();
  5019. SetResult(curId, inst);
  5020. }
  5021. else
  5022. {
  5023. auto& dbgResult = mDIBuilder->insertDeclare(val, (llvm::DILocalVariable*)varInfo, mDIBuilder->createExpression(),
  5024. mIRBuilder->getCurrentDebugLocation(), mIRBuilder->GetInsertBlock());
  5025. bool isA = dbgResult.is<llvm::Instruction*>();
  5026. auto inst = dbgResult.get<llvm::Instruction*>();
  5027. SetResult(curId, inst);
  5028. }
  5029. }
  5030. }
  5031. break;
  5032. case BfIRCmd_DbgLifetimeEnd:
  5033. {
  5034. CMD_PARAM(llvm::MDNode*, varInfo);
  5035. }
  5036. break;
  5037. case BfIRCmd_DbgCreateGlobalVariable:
  5038. {
  5039. CMD_PARAM(llvm::MDNode*, context);
  5040. CMD_PARAM(String, name);
  5041. CMD_PARAM(String, linkageName);
  5042. CMD_PARAM(llvm::MDNode*, file);
  5043. CMD_PARAM(int, lineNum);
  5044. CMD_PARAM(llvm::MDNode*, type);
  5045. CMD_PARAM(bool, isLocalToUnit);
  5046. CMD_PARAM(llvm::Constant*, val);
  5047. CMD_PARAM(llvm::MDNode*, decl);
  5048. //BF_ASSERT(file != NULL);
  5049. llvm::DIExpression* diExpr = NULL;
  5050. auto gve = mDIBuilder->createGlobalVariableExpression((llvm::DIScope*)context, name.c_str(), linkageName.c_str(), (llvm::DIFile*)file, lineNum, (llvm::DIType*)type,
  5051. isLocalToUnit, true, diExpr, decl);
  5052. if (val != NULL)
  5053. {
  5054. if (auto globalVar = llvm::dyn_cast<llvm::GlobalVariable>(val))
  5055. {
  5056. globalVar->addDebugInfo(gve);
  5057. }
  5058. }
  5059. SetResult(curId, diExpr);
  5060. }
  5061. break;
  5062. case BfIRCmd_DbgCreateLexicalBlock:
  5063. {
  5064. CMD_PARAM(llvm::MDNode*, scope);
  5065. CMD_PARAM(llvm::MDNode*, file);
  5066. CMD_PARAM(int, lineNum);
  5067. CMD_PARAM(int, col);
  5068. if (lineNum == 0)
  5069. col = 0;
  5070. BF_ASSERT(file != NULL);
  5071. SetResult(curId, mDIBuilder->createLexicalBlock((llvm::DIScope*)scope, (llvm::DIFile*)file, (unsigned)lineNum, (unsigned)col));
  5072. }
  5073. break;
  5074. case BfIRCmd_DbgCreateAnnotation:
  5075. {
  5076. CMD_PARAM(llvm::MDNode*, scope);
  5077. CMD_PARAM(String, name);
  5078. CMD_PARAM(llvm::Value*, value);
  5079. if (auto dbgFunc = llvm::dyn_cast<llvm::DISubprogram>(scope))
  5080. {
  5081. auto beType = value->getType();
  5082. auto diType = mDIBuilder->createBasicType("int32", 4 * 8, llvm::dwarf::DW_ATE_signed);
  5083. llvm::DINode::DIFlags diFlags = (llvm::DINode::DIFlags)0;
  5084. auto loc = mIRBuilder->getCurrentDebugLocation();
  5085. auto dbgLoc = loc.getAsMDNode();
  5086. auto diScope = (llvm::DIScope*)scope;
  5087. String dbgName = "#" + name;
  5088. int64 writeVal = 0;
  5089. if (auto constant = llvm::dyn_cast<llvm::ConstantInt>(value))
  5090. {
  5091. writeVal = constant->getSExtValue();
  5092. }
  5093. if (writeVal < 0)
  5094. dbgName += StrFormat("$_%llu", -writeVal);
  5095. else
  5096. dbgName += StrFormat("$%llu", writeVal);
  5097. auto dbgVar = mDIBuilder->createAutoVariable((llvm::DIScope*)scope, dbgName.c_str(), (llvm::DIFile*)diScope->getFile(), 0, diType, false, diFlags);
  5098. mDIBuilder->insertDbgValueIntrinsic(value, dbgVar, mDIBuilder->createExpression(),
  5099. mIRBuilder->getCurrentDebugLocation(), (llvm::BasicBlock*)mIRBuilder->GetInsertBlock());
  5100. }
  5101. }
  5102. break;
  5103. default:
  5104. BF_FATAL("Unhandled");
  5105. break;
  5106. }
  5107. }
  5108. void BfIRCodeGen::SetCodeGenOptions(BfCodeGenOptions codeGenOptions)
  5109. {
  5110. mCodeGenOptions = codeGenOptions;
  5111. }
  5112. void BfIRCodeGen::SetConfigConst(int idx, int value)
  5113. {
  5114. auto constVal = llvm::ConstantInt::get(llvm::Type::getInt32Ty(*mLLVMContext), value);
  5115. BF_ASSERT(idx == (int)mConfigConsts32.size());
  5116. mConfigConsts32.Add(constVal);
  5117. constVal = llvm::ConstantInt::get(llvm::Type::getInt64Ty(*mLLVMContext), value);
  5118. BF_ASSERT(idx == (int)mConfigConsts64.size());
  5119. mConfigConsts64.Add(constVal);
  5120. }
  5121. void BfIRCodeGen::SetActiveFunctionSimdType(BfIRSimdType type)
  5122. {
  5123. BfIRSimdType currentType;
  5124. bool contains = mFunctionsUsingSimd.TryGetValue(mActiveFunction, &currentType);
  5125. if (!contains || type > currentType)
  5126. mFunctionsUsingSimd[mActiveFunction] = type;
  5127. }
  5128. String BfIRCodeGen::GetSimdTypeString(BfIRSimdType type)
  5129. {
  5130. switch (type)
  5131. {
  5132. case BfIRSimdType_SSE:
  5133. return "+sse,+mmx";
  5134. case BfIRSimdType_SSE2:
  5135. return "+sse2,+sse,+mmx";
  5136. case BfIRSimdType_AVX:
  5137. return "+avx,+sse4.2,+sse4.1,+sse3,+sse2,+sse,+mmx";
  5138. case BfIRSimdType_AVX2:
  5139. return "+avx2,+avx,+sse4.2,+sse4.1,+sse3,+sse2,+sse,+mmx";
  5140. case BfIRSimdType_AVX512:
  5141. return "+avx512f,+avx2,+avx,+sse4.2,+sse4.1,+sse3,+sse2,+sse,+mmx";
  5142. default:
  5143. return "";
  5144. }
  5145. }
  5146. BfIRSimdType BfIRCodeGen::GetSimdTypeFromFunction(llvm::Function* function)
  5147. {
  5148. if (function->hasFnAttribute("target-features"))
  5149. {
  5150. auto str = function->getFnAttribute("target-features").getValueAsString();
  5151. if (str.contains("+avx512f"))
  5152. return BfIRSimdType_AVX512;
  5153. if (str.contains("+avx2"))
  5154. return BfIRSimdType_AVX2;
  5155. if (str.contains("+avx"))
  5156. return BfIRSimdType_AVX;
  5157. if (str.contains("+sse2"))
  5158. return BfIRSimdType_SSE2;
  5159. if (str.contains("+sse"))
  5160. return BfIRSimdType_SSE;
  5161. }
  5162. return BfIRSimdType_None;
  5163. }
  5164. BfIRTypedValue BfIRCodeGen::GetTypedValue(int id)
  5165. {
  5166. auto& result = mResults[id];
  5167. if (result.mKind == BfIRCodeGenEntryKind_TypedValue)
  5168. return result.mTypedValue;
  5169. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMValue);
  5170. BfIRTypedValue typedValue;
  5171. typedValue.mTypeEx = NULL;
  5172. typedValue.mValue = result.mLLVMValue;
  5173. return typedValue;
  5174. }
  5175. llvm::Value* BfIRCodeGen::GetLLVMValue(int id)
  5176. {
  5177. auto& result = mResults[id];
  5178. if (result.mKind == BfIRCodeGenEntryKind_TypedValue)
  5179. return result.mTypedValue.mValue;
  5180. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMValue);
  5181. return result.mLLVMValue;
  5182. }
  5183. llvm::Type* BfIRCodeGen::GetLLVMType(int id)
  5184. {
  5185. auto& result = mResults[id];
  5186. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMType);
  5187. return result.mLLVMType;
  5188. }
  5189. llvm::BasicBlock * BfIRCodeGen::GetLLVMBlock(int id)
  5190. {
  5191. auto& result = mResults[id];
  5192. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMBasicBlock);
  5193. return result.mLLVMBlock;
  5194. }
  5195. llvm::MDNode* BfIRCodeGen::GetLLVMMetadata(int id)
  5196. {
  5197. auto& result = mResults[id];
  5198. BF_ASSERT(result.mKind == BfIRCodeGenEntryKind_LLVMMetadata);
  5199. return result.mLLVMMetadata;
  5200. }
  5201. llvm::Type* BfIRCodeGen::GetLLVMTypeById(int id)
  5202. {
  5203. return GetTypeEntry(id).mType->mLLVMType;
  5204. }
  5205. // LLVM/Clang 18.1.4
  5206. static void addSanitizers(const llvm::Triple& TargetTriple, BfCodeGenOptions& CodeGenOpts, llvm::PassBuilder& PB)
  5207. {
  5208. #if 0
  5209. auto SanitizersCallback = [&](llvm::ModulePassManager& MPM, llvm::OptimizationLevel Level) {
  5210. if (CodeGenOpts.hasSanitizeCoverage())
  5211. {
  5212. auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts);
  5213. MPM.addPass(SanitizerCoveragePass(
  5214. SancovOpts, CodeGenOpts.SanitizeCoverageAllowlistFiles,
  5215. CodeGenOpts.SanitizeCoverageIgnorelistFiles));
  5216. }
  5217. if (CodeGenOpts.hasSanitizeBinaryMetadata()) {
  5218. MPM.addPass(SanitizerBinaryMetadataPass(
  5219. getSanitizerBinaryMetadataOptions(CodeGenOpts),
  5220. CodeGenOpts.SanitizeMetadataIgnorelistFiles));
  5221. }
  5222. auto MSanPass = [&](SanitizerMask Mask, bool CompileKernel) {
  5223. if (LangOpts.Sanitize.has(Mask)) {
  5224. int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins;
  5225. bool Recover = CodeGenOpts.SanitizeRecover.has(Mask);
  5226. MemorySanitizerOptions options(TrackOrigins, Recover, CompileKernel,
  5227. CodeGenOpts.SanitizeMemoryParamRetval);
  5228. MPM.addPass(MemorySanitizerPass(options));
  5229. if (Level != OptimizationLevel::O0) {
  5230. // MemorySanitizer inserts complex instrumentation that mostly follows
  5231. // the logic of the original code, but operates on "shadow" values. It
  5232. // can benefit from re-running some general purpose optimization
  5233. // passes.
  5234. MPM.addPass(RequireAnalysisPass<GlobalsAA, llvm::Module>());
  5235. FunctionPassManager FPM;
  5236. FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
  5237. FPM.addPass(InstCombinePass());
  5238. FPM.addPass(JumpThreadingPass());
  5239. FPM.addPass(GVNPass());
  5240. FPM.addPass(InstCombinePass());
  5241. MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
  5242. }
  5243. }
  5244. };
  5245. MSanPass(SanitizerKind::Memory, false);
  5246. MSanPass(SanitizerKind::KernelMemory, true);
  5247. if (LangOpts.Sanitize.has(SanitizerKind::Thread)) {
  5248. MPM.addPass(ModuleThreadSanitizerPass());
  5249. MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
  5250. }
  5251. auto ASanPass = [&](SanitizerMask Mask, bool CompileKernel) {
  5252. if (LangOpts.Sanitize.has(Mask)) {
  5253. bool UseGlobalGC = asanUseGlobalsGC(TargetTriple, CodeGenOpts);
  5254. bool UseOdrIndicator = CodeGenOpts.SanitizeAddressUseOdrIndicator;
  5255. llvm::AsanDtorKind DestructorKind =
  5256. CodeGenOpts.getSanitizeAddressDtor();
  5257. AddressSanitizerOptions Opts;
  5258. Opts.CompileKernel = CompileKernel;
  5259. Opts.Recover = CodeGenOpts.SanitizeRecover.has(Mask);
  5260. Opts.UseAfterScope = CodeGenOpts.SanitizeAddressUseAfterScope;
  5261. Opts.UseAfterReturn = CodeGenOpts.getSanitizeAddressUseAfterReturn();
  5262. MPM.addPass(AddressSanitizerPass(Opts, UseGlobalGC, UseOdrIndicator,
  5263. DestructorKind));
  5264. }
  5265. };
  5266. ASanPass(SanitizerKind::Address, false);
  5267. ASanPass(SanitizerKind::KernelAddress, true);
  5268. auto HWASanPass = [&](SanitizerMask Mask, bool CompileKernel) {
  5269. if (LangOpts.Sanitize.has(Mask)) {
  5270. bool Recover = CodeGenOpts.SanitizeRecover.has(Mask);
  5271. MPM.addPass(HWAddressSanitizerPass(
  5272. { CompileKernel, Recover,
  5273. /*DisableOptimization=*/CodeGenOpts.OptimizationLevel == 0 }));
  5274. }
  5275. };
  5276. HWASanPass(SanitizerKind::HWAddress, false);
  5277. HWASanPass(SanitizerKind::KernelHWAddress, true);
  5278. if (LangOpts.Sanitize.has(SanitizerKind::DataFlow)) {
  5279. MPM.addPass(DataFlowSanitizerPass(LangOpts.NoSanitizeFiles));
  5280. }
  5281. };
  5282. if (ClSanitizeOnOptimizerEarlyEP) {
  5283. PB.registerOptimizerEarlyEPCallback(
  5284. [SanitizersCallback](ModulePassManager& MPM, OptimizationLevel Level) {
  5285. ModulePassManager NewMPM;
  5286. SanitizersCallback(NewMPM, Level);
  5287. if (!NewMPM.isEmpty()) {
  5288. // Sanitizers can abandon<GlobalsAA>.
  5289. NewMPM.addPass(RequireAnalysisPass<GlobalsAA, llvm::Module>());
  5290. MPM.addPass(std::move(NewMPM));
  5291. }
  5292. });
  5293. }
  5294. else {
  5295. // LastEP does not need GlobalsAA.
  5296. PB.registerOptimizerLastEPCallback(SanitizersCallback);
  5297. }
  5298. #endif
  5299. }
  5300. // LLVM/Clang 18.1.4
  5301. static void addKCFIPass(const llvm::Triple& TargetTriple, const BfCodeGenOptions& codeGenOpts, llvm::PassBuilder& PB)
  5302. {
  5303. #if 0
  5304. // If the back-end supports KCFI operand bundle lowering, skip KCFIPass.
  5305. if (TargetTriple.getArch() == llvm::Triple::x86_64 ||
  5306. TargetTriple.isAArch64(64) || TargetTriple.isRISCV())
  5307. return;
  5308. // Ensure we lower KCFI operand bundles with -O0.
  5309. PB.registerOptimizerLastEPCallback(
  5310. [&](ModulePassManager& MPM, OptimizationLevel Level) {
  5311. if (Level == OptimizationLevel::O0 &&
  5312. LangOpts.Sanitize.has(SanitizerKind::KCFI))
  5313. MPM.addPass(createModuleToFunctionPassAdaptor(KCFIPass()));
  5314. });
  5315. // When optimizations are requested, run KCIFPass after InstCombine to
  5316. // avoid unnecessary checks.
  5317. PB.registerPeepholeEPCallback(
  5318. [&](FunctionPassManager& FPM, OptimizationLevel Level) {
  5319. if (Level != OptimizationLevel::O0 &&
  5320. LangOpts.Sanitize.has(SanitizerKind::KCFI))
  5321. FPM.addPass(KCFIPass());
  5322. });
  5323. #endif
  5324. }
  5325. /// Check whether we should emit a module summary for regular LTO.
  5326. /// The module summary should be emitted by default for regular LTO
  5327. /// except for ld64 targets.
  5328. ///
  5329. /// \return True if the module summary should be emitted.
  5330. static bool shouldEmitRegularLTOSummary(const llvm::Triple& targetTriple, const BfCodeGenOptions& codeGenOptions, bool PrepareForLTO)
  5331. {
  5332. return PrepareForLTO /*&& !CodeGenOpts.DisableLLVMPasses*/ &&
  5333. targetTriple.getVendor() != llvm::Triple::Apple;
  5334. }
  5335. /// Check whether we should emit a flag for UnifiedLTO.
  5336. /// The UnifiedLTO module flag should be set when UnifiedLTO is enabled for
  5337. /// ThinLTO or Full LTO with module summaries.
  5338. static bool shouldEmitUnifiedLTOModueFlag(const llvm::Triple& targetTriple, const BfCodeGenOptions& codeGenOptions, bool PrepareForLTO)
  5339. {
  5340. return false;
  5341. /*return CodeGenOpts.UnifiedLTO &&
  5342. (CodeGenOpts.PrepareForThinLTO || shouldEmitRegularLTOSummary());*/
  5343. }
  5344. void BfIRCodeGen::RunOptimizationPipeline(const llvm::Triple& targetTriple)
  5345. {
  5346. bool verifyModule = true;
  5347. std::optional<llvm::PGOOptions> pgoOptions;
  5348. mLLVMTargetMachine->setPGOOption(pgoOptions);
  5349. llvm::PipelineTuningOptions pto;
  5350. pto.LoopUnrolling = !mCodeGenOptions.mDisableUnrollLoops;
  5351. // For historical reasons, loop interleaving is set to mirror setting for loop unrolling.
  5352. pto.LoopInterleaving = !mCodeGenOptions.mDisableUnrollLoops;
  5353. pto.LoopVectorization = mCodeGenOptions.mLoopVectorize;
  5354. pto.SLPVectorization = mCodeGenOptions.mSLPVectorize;
  5355. pto.MergeFunctions = mCodeGenOptions.mMergeFunctions;
  5356. //TODO:
  5357. //pto.CallGraphProfile = ???
  5358. //pto.UnifiedLTO = ???
  5359. llvm::LoopAnalysisManager LAM;
  5360. llvm::FunctionAnalysisManager FAM;
  5361. llvm::CGSCCAnalysisManager CGAM;
  5362. llvm::ModuleAnalysisManager MAM;
  5363. llvm::PassInstrumentationCallbacks PIC;
  5364. // PrintPassOptions PrintPassOpts;
  5365. // PrintPassOpts.Indent = DebugPassStructure;
  5366. // PrintPassOpts.SkipAnalyses = DebugPassStructure;
  5367. // StandardInstrumentations SI(
  5368. // TheModule->getContext(),
  5369. // (CodeGenOpts.DebugPassManager || DebugPassStructure),
  5370. // CodeGenOpts.VerifyEach, PrintPassOpts);
  5371. // SI.registerCallbacks(PIC, &MAM);
  5372. llvm::PassBuilder PB(mLLVMTargetMachine, pto, pgoOptions, &PIC);
  5373. // Register all the basic analyses with the managers.
  5374. PB.registerModuleAnalyses(MAM);
  5375. PB.registerCGSCCAnalyses(CGAM);
  5376. PB.registerFunctionAnalyses(FAM);
  5377. PB.registerLoopAnalyses(LAM);
  5378. PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
  5379. //llvm::ModulePassManager MPM;
  5380. // Add a verifier pass, before any other passes, to catch CodeGen issues.
  5381. llvm::ModulePassManager MPM;
  5382. if (verifyModule)
  5383. MPM.addPass(llvm::VerifierPass());
  5384. bool disableLLVMPasses = false;
  5385. if (!disableLLVMPasses)
  5386. {
  5387. llvm::OptimizationLevel Level;
  5388. bool PrepareForLTO = false;
  5389. bool PrepareForThinLTO = mCodeGenOptions.mLTOType == BfLTOType_Thin;
  5390. //bool performThinLTO = false;
  5391. Level = llvm::OptimizationLevel::O0;
  5392. switch (mCodeGenOptions.mOptLevel)
  5393. {
  5394. case BfOptLevel_O0:
  5395. Level = llvm::OptimizationLevel::O0;
  5396. break;
  5397. case BfOptLevel_O1:
  5398. Level = llvm::OptimizationLevel::O1;
  5399. break;
  5400. case BfOptLevel_O2:
  5401. Level = llvm::OptimizationLevel::O2;
  5402. break;
  5403. case BfOptLevel_O3:
  5404. Level = llvm::OptimizationLevel::O3;
  5405. break;
  5406. case BfOptLevel_Og:
  5407. Level = llvm::OptimizationLevel::O1;
  5408. break;
  5409. }
  5410. bool IsThinLTOPostLink = false;
  5411. #if 0
  5412. // If we reached here with a non-empty index file name, then the index
  5413. // file was empty and we are not performing ThinLTO backend compilation
  5414. // (used in testing in a distributed build environment).
  5415. bool IsThinLTOPostLink = !CodeGenOpts.ThinLTOIndexFile.empty();
  5416. // If so drop any the type test assume sequences inserted for whole program
  5417. // vtables so that codegen doesn't complain.
  5418. if (IsThinLTOPostLink)
  5419. PB.registerPipelineStartEPCallback(
  5420. [](ModulePassManager& MPM, OptimizationLevel Level) {
  5421. MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr,
  5422. /*ImportSummary=*/nullptr,
  5423. /*DropTypeTests=*/true));
  5424. });
  5425. // Register callbacks to schedule sanitizer passes at the appropriate part
  5426. // of the pipeline.
  5427. if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds))
  5428. PB.registerScalarOptimizerLateEPCallback(
  5429. [](FunctionPassManager& FPM, OptimizationLevel Level) {
  5430. FPM.addPass(BoundsCheckingPass());
  5431. });
  5432. #endif
  5433. // Don't add sanitizers if we are here from ThinLTO PostLink. That already
  5434. // done on PreLink stage.
  5435. if (!IsThinLTOPostLink) {
  5436. addSanitizers(targetTriple, mCodeGenOptions, PB);
  5437. addKCFIPass(targetTriple, mCodeGenOptions, PB);
  5438. }
  5439. #if 0
  5440. if (std::optional<GCOVOptions> Options =
  5441. getGCOVOptions(CodeGenOpts, LangOpts))
  5442. PB.registerPipelineStartEPCallback(
  5443. [Options](ModulePassManager& MPM, OptimizationLevel Level) {
  5444. MPM.addPass(GCOVProfilerPass(*Options));
  5445. });
  5446. if (std::optional<InstrProfOptions> Options =
  5447. getInstrProfOptions(CodeGenOpts, LangOpts))
  5448. PB.registerPipelineStartEPCallback(
  5449. [Options](ModulePassManager& MPM, OptimizationLevel Level) {
  5450. MPM.addPass(InstrProfilingLoweringPass(*Options, false));
  5451. });
  5452. // TODO: Consider passing the MemoryProfileOutput to the pass builder via
  5453. // the PGOOptions, and set this up there.
  5454. if (!CodeGenOpts.MemoryProfileOutput.empty()) {
  5455. PB.registerOptimizerLastEPCallback(
  5456. [](ModulePassManager& MPM, OptimizationLevel Level) {
  5457. MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
  5458. MPM.addPass(ModuleMemProfilerPass());
  5459. });
  5460. }
  5461. #endif
  5462. if (mCodeGenOptions.mLTOType == BfLTOType_Fat)
  5463. {
  5464. MPM.addPass(PB.buildFatLTODefaultPipeline(
  5465. Level, PrepareForThinLTO,
  5466. PrepareForThinLTO || shouldEmitRegularLTOSummary(targetTriple, mCodeGenOptions, PrepareForLTO)));
  5467. }
  5468. else if (PrepareForThinLTO)
  5469. {
  5470. MPM.addPass(PB.buildThinLTOPreLinkDefaultPipeline(Level));
  5471. }
  5472. else if (PrepareForLTO)
  5473. {
  5474. MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(Level));
  5475. }
  5476. else
  5477. {
  5478. MPM.addPass(PB.buildPerModuleDefaultPipeline(Level));
  5479. }
  5480. }
  5481. // Re-link against any bitcodes supplied via the -mlink-builtin-bitcode option
  5482. // Some optimizations may generate new function calls that would not have
  5483. // been linked pre-optimization (i.e. fused sincos calls generated by
  5484. // AMDGPULibCalls::fold_sincos.)
  5485. //TODO:
  5486. // if (ClRelinkBuiltinBitcodePostop)
  5487. // MPM.addPass(LinkInModulesPass(BC, false));
  5488. // Add a verifier pass if requested. We don't have to do this if the action
  5489. // requires code generation because there will already be a verifier pass in
  5490. // the code-generation pipeline.
  5491. // Since we already added a verifier pass above, this
  5492. // might even not run the analysis, if previous passes caused no changes.
  5493. // if (!actionRequiresCodeGen(Action) && CodeGenOpts.VerifyModule)
  5494. // MPM.addPass(VerifierPass());
  5495. //TODO:
  5496. #if 0
  5497. if (Action == Backend_EmitBC || Action == Backend_EmitLL || CodeGenOpts.FatLTO)
  5498. {
  5499. if (CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.DisableLLVMPasses) {
  5500. if (!TheModule->getModuleFlag("EnableSplitLTOUnit"))
  5501. TheModule->addModuleFlag(llvm::Module::Error, "EnableSplitLTOUnit",
  5502. CodeGenOpts.EnableSplitLTOUnit);
  5503. if (Action == Backend_EmitBC) {
  5504. if (!CodeGenOpts.ThinLinkBitcodeFile.empty()) {
  5505. ThinLinkOS = openOutputFile(CodeGenOpts.ThinLinkBitcodeFile);
  5506. if (!ThinLinkOS)
  5507. return;
  5508. }
  5509. MPM.addPass(ThinLTOBitcodeWriterPass(
  5510. *OS, ThinLinkOS ? &ThinLinkOS->os() : nullptr));
  5511. }
  5512. else if (Action == Backend_EmitLL) {
  5513. MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists,
  5514. /*EmitLTOSummary=*/true));
  5515. }
  5516. }
  5517. else {
  5518. // Emit a module summary by default for Regular LTO except for ld64
  5519. // targets
  5520. bool EmitLTOSummary = shouldEmitRegularLTOSummary();
  5521. if (EmitLTOSummary) {
  5522. if (!TheModule->getModuleFlag("ThinLTO") && !CodeGenOpts.UnifiedLTO)
  5523. TheModule->addModuleFlag(llvm::Module::Error, "ThinLTO", uint32_t(0));
  5524. if (!TheModule->getModuleFlag("EnableSplitLTOUnit"))
  5525. TheModule->addModuleFlag(llvm::Module::Error, "EnableSplitLTOUnit",
  5526. uint32_t(1));
  5527. }
  5528. if (Action == Backend_EmitBC) {
  5529. MPM.addPass(BitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists,
  5530. EmitLTOSummary));
  5531. }
  5532. else if (Action == Backend_EmitLL) {
  5533. MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists,
  5534. EmitLTOSummary));
  5535. }
  5536. }
  5537. if (shouldEmitUnifiedLTOModueFlag())
  5538. TheModule->addModuleFlag(llvm::Module::Error, "UnifiedLTO", uint32_t(1));
  5539. }
  5540. #endif
  5541. // Print a textual, '-passes=' compatible, representation of pipeline if
  5542. // requested.
  5543. // if (PrintPipelinePasses) {
  5544. // MPM.printPipeline(outs(), [&PIC](StringRef ClassName) {
  5545. // auto PassName = PIC.getPassNameForClassName(ClassName);
  5546. // return PassName.empty() ? ClassName : PassName;
  5547. // });
  5548. // outs() << "\n";
  5549. // return;
  5550. // }
  5551. //
  5552. // if (LangOpts.HIPStdPar && !LangOpts.CUDAIsDevice &&
  5553. // LangOpts.HIPStdParInterposeAlloc)
  5554. // MPM.addPass(HipStdParAllocationInterpositionPass());
  5555. // Now that we have all of the passes ready, run them.
  5556. {
  5557. //PrettyStackTraceString CrashInfo("Optimizer");
  5558. //llvm::TimeTraceScope TimeScope("Optimizer");
  5559. MPM.run(*mLLVMModule, MAM);
  5560. }
  5561. }
  5562. bool BfIRCodeGen::WriteObjectFile(const StringImpl& outFileName)
  5563. {
  5564. ApplySimdFeatures();
  5565. // {
  5566. // PassManagerBuilderWrapper pmBuilder;
  5567. //
  5568. //
  5569. // }
  5570. mHasDebugLoc = false; // So fails don't show a line number
  5571. bool enableLTO = mCodeGenOptions.mLTOType != BfLTOType_None;
  5572. if (enableLTO)
  5573. {
  5574. // We have some constructs which trip up ThinLTO, and it's not useful to LTO here anyway
  5575. if (GetFileName(outFileName) == "vdata.obj")
  5576. {
  5577. enableLTO = false;
  5578. }
  5579. if (mHadDLLExport) // LTO bug in LLVM-link?
  5580. enableLTO = false;
  5581. }
  5582. std::error_code EC;
  5583. llvm::sys::fs::OpenFlags OpenFlags = llvm::sys::fs::OF_None;
  5584. llvm::raw_fd_ostream out(outFileName.c_str(), EC, OpenFlags);
  5585. if (EC)
  5586. return false;
  5587. // Build up all of the passes that we want to do to the module.
  5588. //llvm::legacy::PassManager PM;
  5589. llvm::legacy::PassManager PM;
  5590. llvm::Triple theTriple = llvm::Triple(mLLVMModule->getTargetTriple());
  5591. // Add an appropriate TargetLibraryInfo pass for the module's triple.
  5592. llvm::TargetLibraryInfoImpl TLII(theTriple);
  5593. PM.add(new llvm::TargetLibraryInfoWrapperPass(TLII));
  5594. // Add the target data from the target machine, if it exists, or the module.
  5595. //PM.add(new DataLayoutPass());
  5596. RunOptimizationPipeline(theTriple);
  5597. llvm::raw_fd_ostream* outStream = NULL;
  5598. defer ( delete outStream; );
  5599. if ((enableLTO) || (mCodeGenOptions.mWriteBitcode))
  5600. {
  5601. std::error_code ec;
  5602. outStream = new llvm::raw_fd_ostream(outFileName.c_str(), ec, llvm::sys::fs::OF_None);
  5603. if (outStream->has_error())
  5604. {
  5605. return false;
  5606. }
  5607. // if (enableLTO)
  5608. // PM.add(createWriteThinLTOBitcodePass(*outStream, NULL));
  5609. //else
  5610. PM.add(createBitcodeWriterPass(*outStream, false));
  5611. }
  5612. // TargetPassConfig *PassConfig = target->createPassConfig(PM);
  5613. // PM.add(new BfPass());
  5614. // PM.add(sBfPass);
  5615. // Do
  5616. {
  5617. //formatted_raw_ostream FOS(out);
  5618. //raw_pwrite_stream *OS = &out->os();
  5619. //TODO:
  5620. llvm::AnalysisID StartAfterID = nullptr;
  5621. llvm::AnalysisID StopAfterID = nullptr;
  5622. const llvm::PassRegistry *PR = llvm::PassRegistry::getPassRegistry();
  5623. //WriteBitcode
  5624. bool noVerify = false; // Option
  5625. if ((!enableLTO) && (!mCodeGenOptions.mWriteBitcode))
  5626. {
  5627. // Ask the target to add backend passes as necessary.
  5628. if (mLLVMTargetMachine->addPassesToEmitFile(PM, out, NULL,
  5629. (mCodeGenOptions.mAsmKind != BfAsmKind_None) ? llvm::CodeGenFileType::AssemblyFile : llvm::CodeGenFileType::ObjectFile,
  5630. //TargetMachine::CGFT_AssemblyFile,
  5631. noVerify /*, StartAfterID, StopAfterID*/))
  5632. {
  5633. Fail("Target does not support generation of this file type");
  5634. /*errs() << argv[0] << ": target does not support generation of this"
  5635. << " file type!\n";*/
  5636. return false;
  5637. }
  5638. }
  5639. bool success = PM.run(*mLLVMModule);
  5640. if ((mCodeGenOptions.mOptLevel > BfOptLevel_O0) && (mCodeGenOptions.mWriteLLVMIR))
  5641. {
  5642. BP_ZONE("BfCodeGen::RunLoop.LLVM.IR");
  5643. String fileName = outFileName;
  5644. int dotPos = (int)fileName.LastIndexOf('.');
  5645. if (dotPos != -1)
  5646. fileName.RemoveToEnd(dotPos);
  5647. fileName += "_OPT.ll";
  5648. String irError;
  5649. WriteIR(fileName, irError);
  5650. }
  5651. }
  5652. return true;
  5653. }
  5654. bool BfIRCodeGen::WriteIR(const StringImpl& outFileName, StringImpl& error)
  5655. {
  5656. std::error_code ec;
  5657. llvm::raw_fd_ostream outStream(outFileName.c_str(), ec, llvm::sys::fs::OpenFlags::OF_Text);
  5658. if (ec)
  5659. {
  5660. error = ec.message();
  5661. return false;
  5662. }
  5663. mLLVMModule->print(outStream, NULL);
  5664. return true;
  5665. }
  5666. void BfIRCodeGen::ApplySimdFeatures()
  5667. {
  5668. Array<std::tuple<llvm::Function*, BfIRSimdType>> functionsToProcess;
  5669. for (auto pair : mFunctionsUsingSimd)
  5670. functionsToProcess.Add({ pair.mKey, pair.mValue });
  5671. while (functionsToProcess.Count() > 0)
  5672. {
  5673. auto tuple = functionsToProcess.front();
  5674. functionsToProcess.RemoveAt(0);
  5675. auto function = std::get<0>(tuple);
  5676. auto simdType = std::get<1>(tuple);
  5677. auto currentSimdType = GetSimdTypeFromFunction(function);
  5678. simdType = simdType > currentSimdType ? simdType : currentSimdType;
  5679. function->addFnAttr("target-features", GetSimdTypeString(simdType).c_str());
  5680. if (function->hasFnAttribute(llvm::Attribute::AlwaysInline))
  5681. {
  5682. for (auto user : function->users())
  5683. {
  5684. if (auto call = llvm::dyn_cast<llvm::CallInst>(user))
  5685. {
  5686. auto func = call->getFunction();
  5687. functionsToProcess.Add({ func, simdType });
  5688. }
  5689. }
  5690. }
  5691. }
  5692. }
  5693. int BfIRCodeGen::GetIntrinsicId(const StringImpl& name)
  5694. {
  5695. auto itr = std::lower_bound(std::begin(gIntrinEntries), std::end(gIntrinEntries), name);
  5696. if (itr != std::end(gIntrinEntries) && strcmp(itr->mName, name.c_str()) == 0)
  5697. {
  5698. int id = (int)(itr - gIntrinEntries);
  5699. return id;
  5700. }
  5701. if (name.StartsWith("shuffle"))
  5702. return BfIRIntrinsic_Shuffle;
  5703. if (name.Contains(':'))
  5704. return BfIRIntrinsic__PLATFORM;
  5705. return -1;
  5706. }
  5707. const char* BfIRCodeGen::GetIntrinsicName(int intrinId)
  5708. {
  5709. return gIntrinEntries[intrinId].mName;
  5710. }
  5711. void BfIRCodeGen::SetAsmKind(BfAsmKind asmKind)
  5712. {
  5713. const char* args[] = {"", (asmKind == BfAsmKind_ATT) ? "-x86-asm-syntax=att" : "-x86-asm-syntax=intel" };
  5714. llvm::cl::ParseCommandLineOptions(2, args);
  5715. }
  5716. #ifdef BF_PLATFORM_LINUX
  5717. //HACK: I don't know why this is needed, but we get link errors if we don't have it.
  5718. int BF_LinuxFixLinkage()
  5719. {
  5720. llvm::MCContext* ctx = NULL;
  5721. llvm::raw_pwrite_stream* stream = NULL;
  5722. createWasmStreamer(*ctx, NULL, NULL, NULL, false);
  5723. createMachOStreamer(*ctx, NULL, NULL, NULL, false, false, false);
  5724. createAsmStreamer(*ctx, NULL, false, false, NULL, NULL, NULL, false);
  5725. createELFStreamer(*ctx, NULL, NULL, NULL, false);
  5726. return 0;
  5727. }
  5728. #endif
  5729. //#include "aarch64/Disassembler/X86DisassemblerDecoder.h"
  5730. //#include "X86/MCTargetDesc/X86MCTargetDesc.h"
  5731. //#include "X86/MCTargetDesc/X86BaseInfo.h"
  5732. //#include "X86InstrInfo.h"
  5733. #ifdef BF_PLATFORM_MACOS
  5734. #include "AArch64/MCTargetDesc/AArch64MCTargetDesc.h"
  5735. //#include "AArch64/MCTargetDesc/AArch64BaseInfo.h"
  5736. //#include "../X86InstrInfo.h"
  5737. int BF_AARC64_Linkage()
  5738. {
  5739. LLVMInitializeAArch64TargetInfo();
  5740. LLVMInitializeAArch64Target();
  5741. LLVMInitializeAArch64TargetMC();
  5742. return 0;
  5743. }
  5744. #endif
  5745. void BfIRCodeGen::StaticInit()
  5746. {
  5747. LLVMInitializeX86TargetInfo();
  5748. LLVMInitializeX86Target();
  5749. LLVMInitializeX86TargetMC();
  5750. LLVMInitializeX86AsmPrinter();
  5751. LLVMInitializeX86AsmParser();
  5752. LLVMInitializeX86Disassembler();
  5753. LLVMInitializeARMTargetInfo();
  5754. LLVMInitializeARMTarget();
  5755. LLVMInitializeARMTargetMC();
  5756. LLVMInitializeARMAsmPrinter();
  5757. LLVMInitializeAArch64TargetInfo();
  5758. LLVMInitializeAArch64Target();
  5759. LLVMInitializeAArch64TargetMC();
  5760. LLVMInitializeAArch64AsmPrinter();
  5761. //LLVMInitializeAArch64Parser();
  5762. //LLVMInitializeX86Disassembler();
  5763. LLVMInitializeWebAssemblyTargetInfo();
  5764. LLVMInitializeWebAssemblyTarget();
  5765. LLVMInitializeWebAssemblyTargetMC();
  5766. LLVMInitializeWebAssemblyAsmPrinter();
  5767. //LLVMInitializeWebAssemblyAsmParser();
  5768. LLVMInitializeWebAssemblyDisassembler();
  5769. }