123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462 |
- # HG changeset patch
- # Parent 8464bb6dfd0210f5e1a72a938f3dd16b1aad78ca
- [Clean-up] Remove most of the rust meat
- diff --git a/Kbuild b/Kbuild
- --- a/Kbuild
- +++ b/Kbuild
- @@ -89,7 +89,6 @@ obj-y += security/
- obj-y += crypto/
- obj-$(CONFIG_BLOCK) += block/
- obj-$(CONFIG_IO_URING) += io_uring/
- -obj-$(CONFIG_RUST) += rust/
- obj-y += $(ARCH_LIB)
- obj-y += drivers/
- obj-y += sound/
- diff --git a/rust/.gitignore b/rust/.gitignore
- deleted file mode 100644
- --- a/rust/.gitignore
- +++ /dev/null
- @@ -1,10 +0,0 @@
- -# SPDX-License-Identifier: GPL-2.0
- -
- -bindings_generated.rs
- -bindings_helpers_generated.rs
- -doctests_kernel_generated.rs
- -doctests_kernel_generated_kunit.c
- -uapi_generated.rs
- -exports_*_generated.h
- -doc/
- -test/
- diff --git a/rust/Makefile b/rust/Makefile
- deleted file mode 100644
- --- a/rust/Makefile
- +++ /dev/null
- @@ -1,463 +0,0 @@
- -# SPDX-License-Identifier: GPL-2.0
- -
- -# Where to place rustdoc generated documentation
- -rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc
- -
- -obj-$(CONFIG_RUST) += core.o compiler_builtins.o
- -always-$(CONFIG_RUST) += exports_core_generated.h
- -
- -# Missing prototypes are expected in the helpers since these are exported
- -# for Rust only, thus there is no header nor prototypes.
- -obj-$(CONFIG_RUST) += helpers.o
- -CFLAGS_REMOVE_helpers.o = -Wmissing-prototypes -Wmissing-declarations
- -
- -always-$(CONFIG_RUST) += libmacros.so
- -no-clean-files += libmacros.so
- -
- -always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs
- -obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o
- -always-$(CONFIG_RUST) += exports_alloc_generated.h exports_bindings_generated.h \
- - exports_kernel_generated.h
- -
- -always-$(CONFIG_RUST) += uapi/uapi_generated.rs
- -obj-$(CONFIG_RUST) += uapi.o
- -
- -ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW
- -obj-$(CONFIG_RUST) += build_error.o
- -else
- -always-$(CONFIG_RUST) += build_error.o
- -endif
- -
- -obj-$(CONFIG_RUST) += exports.o
- -
- -always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs
- -always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c
- -
- -obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o
- -obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o
- -
- -# Avoids running `$(RUSTC)` for the sysroot when it may not be available.
- -ifdef CONFIG_RUST
- -
- -# `$(rust_flags)` is passed in case the user added `--sysroot`.
- -rustc_sysroot := $(shell MAKEFLAGS= $(RUSTC) $(rust_flags) --print sysroot)
- -rustc_host_target := $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2)
- -RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library
- -
- -ifeq ($(quiet),silent_)
- -cargo_quiet=-q
- -rust_test_quiet=-q
- -rustdoc_test_quiet=--test-args -q
- -rustdoc_test_kernel_quiet=>/dev/null
- -else ifeq ($(quiet),quiet_)
- -rust_test_quiet=-q
- -rustdoc_test_quiet=--test-args -q
- -rustdoc_test_kernel_quiet=>/dev/null
- -else
- -cargo_quiet=--verbose
- -endif
- -
- -core-cfgs = \
- - --cfg no_fp_fmt_parse
- -
- -alloc-cfgs = \
- - --cfg no_global_oom_handling \
- - --cfg no_rc \
- - --cfg no_sync
- -
- -quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $<
- - cmd_rustdoc = \
- - OBJTREE=$(abspath $(objtree)) \
- - $(RUSTDOC) $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags)) \
- - $(rustc_target_flags) -L$(objtree)/$(obj) \
- - --output $(rustdoc_output) \
- - --crate-name $(subst rustdoc-,,$@) \
- - $(if $(rustdoc_host),,--sysroot=/dev/null) \
- - @$(objtree)/include/generated/rustc_cfg $<
- -
- -# The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute
- -# can be used to specify a custom logo. However:
- -# - The given value is used as-is, thus it cannot be relative or a local file
- -# (unlike the non-custom case) since the generated docs have subfolders.
- -# - It requires adding it to every crate.
- -# - It requires changing `core` which comes from the sysroot.
- -#
- -# Using `-Zcrate-attr` would solve the last two points, but not the first.
- -# The https://github.com/rust-lang/rfcs/pull/3226 RFC suggests two new
- -# command-like flags to solve the issue. Meanwhile, we use the non-custom case
- -# and then retouch the generated files.
- -rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \
- - rustdoc-alloc rustdoc-kernel
- - $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/
- - $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/
- - $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \
- - -e 's:rust-logo-[0-9a-f]+\.svg:logo.svg:g' \
- - -e 's:favicon-[0-9a-f]+\.svg:logo.svg:g' \
- - -e 's:<link rel="alternate icon" type="image/png" href="[/.]+/static\.files/favicon-(16x16|32x32)-[0-9a-f]+\.png">::g' \
- - -e 's:<a href="srctree/([^"]+)">:<a href="$(realpath $(srctree))/\1">:g'
- - $(Q)for f in $(rustdoc_output)/static.files/rustdoc-*.css; do \
- - echo ".logo-container > img { object-fit: contain; }" >> $$f; done
- -
- -rustdoc-macros: private rustdoc_host = yes
- -rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \
- - --extern proc_macro
- -rustdoc-macros: $(src)/macros/lib.rs FORCE
- - +$(call if_changed,rustdoc)
- -
- -rustdoc-core: private rustc_target_flags = $(core-cfgs)
- -rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE
- - +$(call if_changed,rustdoc)
- -
- -rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE
- - +$(call if_changed,rustdoc)
- -
- -# We need to allow `rustdoc::broken_intra_doc_links` because some
- -# `no_global_oom_handling` functions refer to non-`no_global_oom_handling`
- -# functions. Ideally `rustdoc` would have a way to distinguish broken links
- -# due to things that are "configured out" vs. entirely non-existing ones.
- -rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \
- - -Arustdoc::broken_intra_doc_links
- -rustdoc-alloc: $(RUST_LIB_SRC)/alloc/src/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE
- - +$(call if_changed,rustdoc)
- -
- -rustdoc-kernel: private rustc_target_flags = --extern alloc \
- - --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \
- - --extern bindings --extern uapi
- -rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \
- - rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \
- - $(obj)/bindings.o FORCE
- - +$(call if_changed,rustdoc)
- -
- -quiet_cmd_rustc_test_library = RUSTC TL $<
- - cmd_rustc_test_library = \
- - OBJTREE=$(abspath $(objtree)) \
- - $(RUSTC) $(rust_common_flags) \
- - @$(objtree)/include/generated/rustc_cfg $(rustc_target_flags) \
- - --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \
- - --out-dir $(objtree)/$(obj)/test --cfg testlib \
- - --sysroot $(objtree)/$(obj)/test/sysroot \
- - -L$(objtree)/$(obj)/test \
- - --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $<
- -
- -rusttestlib-build_error: $(src)/build_error.rs rusttest-prepare FORCE
- - +$(call if_changed,rustc_test_library)
- -
- -rusttestlib-macros: private rustc_target_flags = --extern proc_macro
- -rusttestlib-macros: private rustc_test_library_proc = yes
- -rusttestlib-macros: $(src)/macros/lib.rs rusttest-prepare FORCE
- - +$(call if_changed,rustc_test_library)
- -
- -rusttestlib-bindings: $(src)/bindings/lib.rs rusttest-prepare FORCE
- - +$(call if_changed,rustc_test_library)
- -
- -rusttestlib-uapi: $(src)/uapi/lib.rs rusttest-prepare FORCE
- - +$(call if_changed,rustc_test_library)
- -
- -quiet_cmd_rustdoc_test = RUSTDOC T $<
- - cmd_rustdoc_test = \
- - OBJTREE=$(abspath $(objtree)) \
- - $(RUSTDOC) --test $(rust_common_flags) \
- - @$(objtree)/include/generated/rustc_cfg \
- - $(rustc_target_flags) $(rustdoc_test_target_flags) \
- - --sysroot $(objtree)/$(obj)/test/sysroot $(rustdoc_test_quiet) \
- - -L$(objtree)/$(obj)/test --output $(rustdoc_output) \
- - --crate-name $(subst rusttest-,,$@) $<
- -
- -quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $<
- - cmd_rustdoc_test_kernel = \
- - rm -rf $(objtree)/$(obj)/test/doctests/kernel; \
- - mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \
- - OBJTREE=$(abspath $(objtree)) \
- - $(RUSTDOC) --test $(rust_flags) \
- - -L$(objtree)/$(obj) --extern alloc --extern kernel \
- - --extern build_error --extern macros \
- - --extern bindings --extern uapi \
- - --no-run --crate-name kernel -Zunstable-options \
- - --sysroot=/dev/null \
- - --test-builder $(objtree)/scripts/rustdoc_test_builder \
- - $< $(rustdoc_test_kernel_quiet); \
- - $(objtree)/scripts/rustdoc_test_gen
- -
- -%/doctests_kernel_generated.rs %/doctests_kernel_generated_kunit.c: \
- - $(src)/kernel/lib.rs $(obj)/kernel.o \
- - $(objtree)/scripts/rustdoc_test_builder \
- - $(objtree)/scripts/rustdoc_test_gen FORCE
- - +$(call if_changed,rustdoc_test_kernel)
- -
- -# We cannot use `-Zpanic-abort-tests` because some tests are dynamic,
- -# so for the moment we skip `-Cpanic=abort`.
- -quiet_cmd_rustc_test = RUSTC T $<
- - cmd_rustc_test = \
- - OBJTREE=$(abspath $(objtree)) \
- - $(RUSTC) --test $(rust_common_flags) \
- - @$(objtree)/include/generated/rustc_cfg \
- - $(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \
- - --sysroot $(objtree)/$(obj)/test/sysroot \
- - -L$(objtree)/$(obj)/test \
- - --crate-name $(subst rusttest-,,$@) $<; \
- - $(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \
- - $(rustc_test_run_flags)
- -
- -rusttest: rusttest-macros rusttest-kernel
- -
- -# This prepares a custom sysroot with our custom `alloc` instead of
- -# the standard one.
- -#
- -# This requires several hacks:
- -# - Unlike `core` and `alloc`, `std` depends on more than a dozen crates,
- -# including third-party crates that need to be downloaded, plus custom
- -# `build.rs` steps. Thus hardcoding things here is not maintainable.
- -# - `cargo` knows how to build the standard library, but it is an unstable
- -# feature so far (`-Zbuild-std`).
- -# - `cargo` only considers the use case of building the standard library
- -# to use it in a given package. Thus we need to create a dummy package
- -# and pick the generated libraries from there.
- -# - The usual ways of modifying the dependency graph in `cargo` do not seem
- -# to apply for the `-Zbuild-std` steps, thus we have to mislead it
- -# by modifying the sources in the sysroot.
- -# - To avoid messing with the user's Rust installation, we create a clone
- -# of the sysroot. However, `cargo` ignores `RUSTFLAGS` in the `-Zbuild-std`
- -# steps, thus we use a wrapper binary passed via `RUSTC` to pass the flag.
- -#
- -# In the future, we hope to avoid the whole ordeal by either:
- -# - Making the `test` crate not depend on `std` (either improving upstream
- -# or having our own custom crate).
- -# - Making the tests run in kernel space (requires the previous point).
- -# - Making `std` and friends be more like a "normal" crate, so that
- -# `-Zbuild-std` and related hacks are not needed.
- -quiet_cmd_rustsysroot = RUSTSYSROOT
- - cmd_rustsysroot = \
- - rm -rf $(objtree)/$(obj)/test; \
- - mkdir -p $(objtree)/$(obj)/test; \
- - cp -a $(rustc_sysroot) $(objtree)/$(obj)/test/sysroot; \
- - echo '\#!/bin/sh' > $(objtree)/$(obj)/test/rustc_sysroot; \
- - echo "$(RUSTC) --sysroot=$(abspath $(objtree)/$(obj)/test/sysroot) \"\$$@\"" \
- - >> $(objtree)/$(obj)/test/rustc_sysroot; \
- - chmod u+x $(objtree)/$(obj)/test/rustc_sysroot; \
- - $(CARGO) -q new $(objtree)/$(obj)/test/dummy; \
- - RUSTC=$(objtree)/$(obj)/test/rustc_sysroot $(CARGO) $(cargo_quiet) \
- - test -Zbuild-std --target $(rustc_host_target) \
- - --manifest-path $(objtree)/$(obj)/test/dummy/Cargo.toml; \
- - rm $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib/*; \
- - cp $(objtree)/$(obj)/test/dummy/target/$(rustc_host_target)/debug/deps/* \
- - $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib
- -
- -rusttest-prepare: FORCE
- - +$(call if_changed,rustsysroot)
- -
- -rusttest-macros: private rustc_target_flags = --extern proc_macro
- -rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro
- -rusttest-macros: $(src)/macros/lib.rs rusttest-prepare FORCE
- - +$(call if_changed,rustc_test)
- - +$(call if_changed,rustdoc_test)
- -
- -rusttest-kernel: private rustc_target_flags = --extern alloc \
- - --extern build_error --extern macros --extern bindings --extern uapi
- -rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \
- - rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \
- - rusttestlib-uapi FORCE
- - +$(call if_changed,rustc_test)
- - +$(call if_changed,rustc_test_library)
- -
- -ifdef CONFIG_CC_IS_CLANG
- -bindgen_c_flags = $(c_flags)
- -else
- -# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC
- -# plugin backend and/or the Clang driver would be perfectly compatible with GCC.
- -#
- -# For the moment, here we are tweaking the flags on the fly. This is a hack,
- -# and some kernel configurations may not work (e.g. `GCC_PLUGIN_RANDSTRUCT`
- -# if we end up using one of those structs).
- -bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \
- - -mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \
- - -mindirect-branch=thunk-extern -mindirect-branch-register \
- - -mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \
- - -mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \
- - -mno-pointers-to-nested-functions -mno-string \
- - -mno-strict-align -mstrict-align \
- - -fconserve-stack -falign-jumps=% -falign-loops=% \
- - -femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \
- - -fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \
- - -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \
- - -fzero-call-used-regs=% -fno-stack-clash-protection \
- - -fno-inline-functions-called-once -fsanitize=bounds-strict \
- - -fstrict-flex-arrays=% \
- - --param=% --param asan-%
- -
- -# Derived from `scripts/Makefile.clang`.
- -BINDGEN_TARGET_x86 := x86_64-linux-gnu
- -BINDGEN_TARGET_arm64 := aarch64-linux-gnu
- -BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH))
- -
- -# All warnings are inhibited since GCC builds are very experimental,
- -# many GCC warnings are not supported by Clang, they may only appear in
- -# some configurations, with new GCC versions, etc.
- -bindgen_extra_c_flags = -w --target=$(BINDGEN_TARGET)
- -
- -# Auto variable zero-initialization requires an additional special option with
- -# clang that is going to be removed sometime in the future (likely in
- -# clang-18), so make sure to pass this option only if clang supports it
- -# (libclang major version < 16).
- -#
- -# https://github.com/llvm/llvm-project/issues/44842
- -# https://github.com/llvm/llvm-project/blob/llvmorg-16.0.0-rc2/clang/docs/ReleaseNotes.rst#deprecated-compiler-flags
- -ifdef CONFIG_INIT_STACK_ALL_ZERO
- -libclang_maj_ver=$(shell $(BINDGEN) $(srctree)/scripts/rust_is_available_bindgen_libclang.h 2>&1 | sed -ne 's/.*clang version \([0-9]*\).*/\1/p')
- -ifeq ($(shell expr $(libclang_maj_ver) \< 16), 1)
- -bindgen_extra_c_flags += -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang
- -endif
- -endif
- -
- -bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \
- - $(bindgen_extra_c_flags)
- -endif
- -
- -ifdef CONFIG_LTO
- -bindgen_c_flags_lto = $(filter-out $(CC_FLAGS_LTO), $(bindgen_c_flags))
- -else
- -bindgen_c_flags_lto = $(bindgen_c_flags)
- -endif
- -
- -bindgen_c_flags_final = $(bindgen_c_flags_lto) -D__BINDGEN__
- -
- -quiet_cmd_bindgen = BINDGEN $@
- - cmd_bindgen = \
- - $(BINDGEN) $< $(bindgen_target_flags) \
- - --use-core --with-derive-default --ctypes-prefix core::ffi --no-layout-tests \
- - --no-debug '.*' \
- - -o $@ -- $(bindgen_c_flags_final) -DMODULE \
- - $(bindgen_target_cflags) $(bindgen_target_extra)
- -
- -$(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \
- - $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters)
- -$(obj)/bindings/bindings_generated.rs: private bindgen_target_extra = ; \
- - sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \1/g' $@
- -$(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \
- - $(src)/bindgen_parameters FORCE
- - $(call if_changed_dep,bindgen)
- -
- -$(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \
- - $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters)
- -$(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \
- - $(src)/bindgen_parameters FORCE
- - $(call if_changed_dep,bindgen)
- -
- -# See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn
- -# with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here
- -# given it is `libclang`; but for consistency, future Clang changes and/or
- -# a potential future GCC backend for `bindgen`, we disable it too.
- -$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \
- - --blocklist-type '.*' --allowlist-var '' \
- - --allowlist-function 'rust_helper_.*'
- -$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \
- - -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations
- -$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \
- - sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n pub fn \1/g' $@
- -$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers.c FORCE
- - $(call if_changed_dep,bindgen)
- -
- -quiet_cmd_exports = EXPORTS $@
- - cmd_exports = \
- - $(NM) -p --defined-only $< \
- - | awk '/ (T|R|D|B) / {printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3}' > $@
- -
- -$(obj)/exports_core_generated.h: $(obj)/core.o FORCE
- - $(call if_changed,exports)
- -
- -$(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE
- - $(call if_changed,exports)
- -
- -$(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE
- - $(call if_changed,exports)
- -
- -$(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE
- - $(call if_changed,exports)
- -
- -quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@
- - cmd_rustc_procmacro = \
- - $(RUSTC_OR_CLIPPY) $(rust_common_flags) \
- - -Clinker-flavor=gcc -Clinker=$(HOSTCC) \
- - -Clink-args='$(call escsq,$(KBUILD_HOSTLDFLAGS))' \
- - --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \
- - --crate-type proc-macro \
- - --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<
- -
- -# Procedural macros can only be used with the `rustc` that compiled it.
- -# Therefore, to get `libmacros.so` automatically recompiled when the compiler
- -# version changes, we add `core.o` as a dependency (even if it is not needed).
- -$(obj)/libmacros.so: $(src)/macros/lib.rs $(obj)/core.o FORCE
- - +$(call if_changed_dep,rustc_procmacro)
- -
- -quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@
- - cmd_rustc_library = \
- - OBJTREE=$(abspath $(objtree)) \
- - $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \
- - $(filter-out $(skip_flags),$(rust_flags) $(rustc_target_flags)) \
- - --emit=dep-info=$(depfile) --emit=obj=$@ \
- - --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \
- - --crate-type rlib -L$(objtree)/$(obj) \
- - --crate-name $(patsubst %.o,%,$(notdir $@)) $< \
- - --sysroot=/dev/null \
- - $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
- -
- -rust-analyzer:
- - $(Q)$(srctree)/scripts/generate_rust_analyzer.py \
- - --cfgs='core=$(core-cfgs)' --cfgs='alloc=$(alloc-cfgs)' \
- - $(realpath $(srctree)) $(realpath $(objtree)) \
- - $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \
- - $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json
- -
- -redirect-intrinsics = \
- - __addsf3 __eqsf2 __gesf2 __lesf2 __ltsf2 __mulsf3 __nesf2 __unordsf2 \
- - __adddf3 __ledf2 __ltdf2 __muldf3 __unorddf2 \
- - __muloti4 __multi3 \
- - __udivmodti4 __udivti3 __umodti3
- -
- -ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),)
- - # These intrinsics are defined for ARM64 and RISCV64
- - redirect-intrinsics += \
- - __ashrti3 \
- - __ashlti3 __lshrti3
- -endif
- -
- -$(obj)/core.o: private skip_clippy = 1
- -$(obj)/core.o: private skip_flags = -Dunreachable_pub
- -$(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym))
- -$(obj)/core.o: private rustc_target_flags = $(core-cfgs)
- -$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs FORCE
- - +$(call if_changed_dep,rustc_library)
- -ifdef CONFIG_X86_64
- -$(obj)/core.o: scripts/target.json
- -endif
- -
- -$(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*'
- -$(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE
- - +$(call if_changed_dep,rustc_library)
- -
- -$(obj)/alloc.o: private skip_clippy = 1
- -$(obj)/alloc.o: private skip_flags = -Dunreachable_pub
- -$(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs)
- -$(obj)/alloc.o: $(RUST_LIB_SRC)/alloc/src/lib.rs $(obj)/compiler_builtins.o FORCE
- - +$(call if_changed_dep,rustc_library)
- -
- -$(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE
- - +$(call if_changed_dep,rustc_library)
- -
- -$(obj)/bindings.o: $(src)/bindings/lib.rs \
- - $(obj)/compiler_builtins.o \
- - $(obj)/bindings/bindings_generated.rs \
- - $(obj)/bindings/bindings_helpers_generated.rs FORCE
- - +$(call if_changed_dep,rustc_library)
- -
- -$(obj)/uapi.o: $(src)/uapi/lib.rs \
- - $(obj)/compiler_builtins.o \
- - $(obj)/uapi/uapi_generated.rs FORCE
- - +$(call if_changed_dep,rustc_library)
- -
- -$(obj)/kernel.o: private rustc_target_flags = --extern alloc \
- - --extern build_error --extern macros --extern bindings --extern uapi
- -$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \
- - $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE
- - +$(call if_changed_dep,rustc_library)
- -
- -endif # CONFIG_RUST
- diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters
- deleted file mode 100644
- --- a/rust/bindgen_parameters
- +++ /dev/null
- @@ -1,26 +0,0 @@
- -# SPDX-License-Identifier: GPL-2.0
- -
- ---opaque-type xregs_state
- ---opaque-type desc_struct
- ---opaque-type arch_lbr_state
- ---opaque-type local_apic
- -
- -# Packed type cannot transitively contain a `#[repr(align)]` type.
- ---opaque-type alt_instr
- ---opaque-type x86_msi_data
- ---opaque-type x86_msi_addr_lo
- -
- -# `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2,
- -# commit 2aed6b021680 ("context: Escape the try keyword properly").
- ---opaque-type kunit_try_catch
- -
- -# If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust
- -# warning. We don't need to peek into it anyway.
- ---opaque-type spinlock
- -
- -# `seccomp`'s comment gets understood as a doctest
- ---no-doc-comments
- -
- -# These functions use the `__preserve_most` calling convention, which neither bindgen
- -# nor Rust currently understand, and which Clang currently declares to be unstable.
- ---blocklist-function __list_.*_report
- diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
- deleted file mode 100644
- --- a/rust/bindings/bindings_helper.h
- +++ /dev/null
- @@ -1,27 +0,0 @@
- -/* SPDX-License-Identifier: GPL-2.0 */
- -/*
- - * Header that contains the code (mostly headers) for which Rust bindings
- - * will be automatically generated by `bindgen`.
- - *
- - * Sorted alphabetically.
- - */
- -
- -#include <kunit/test.h>
- -#include <linux/errname.h>
- -#include <linux/ethtool.h>
- -#include <linux/jiffies.h>
- -#include <linux/mdio.h>
- -#include <linux/phy.h>
- -#include <linux/refcount.h>
- -#include <linux/sched.h>
- -#include <linux/slab.h>
- -#include <linux/wait.h>
- -#include <linux/workqueue.h>
- -
- -/* `bindgen` gets confused at certain things. */
- -const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN;
- -const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC;
- -const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL;
- -const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT;
- -const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT;
- -const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO;
- diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs
- deleted file mode 100644
- --- a/rust/bindings/lib.rs
- +++ /dev/null
- @@ -1,50 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Bindings.
- -//!
- -//! Imports the generated bindings by `bindgen`.
- -//!
- -//! This crate may not be directly used. If you need a kernel C API that is
- -//! not ported or wrapped in the `kernel` crate, then do so first instead of
- -//! using this crate.
- -
- -#![no_std]
- -// See <https://github.com/rust-lang/rust-bindgen/issues/1651>.
- -#![cfg_attr(test, allow(deref_nullptr))]
- -#![cfg_attr(test, allow(unaligned_references))]
- -#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))]
- -#![allow(
- - clippy::all,
- - missing_docs,
- - non_camel_case_types,
- - non_upper_case_globals,
- - non_snake_case,
- - improper_ctypes,
- - unreachable_pub,
- - unsafe_op_in_unsafe_fn
- -)]
- -
- -mod bindings_raw {
- - // Use glob import here to expose all helpers.
- - // Symbols defined within the module will take precedence to the glob import.
- - pub use super::bindings_helper::*;
- - include!(concat!(
- - env!("OBJTREE"),
- - "/rust/bindings/bindings_generated.rs"
- - ));
- -}
- -
- -// When both a directly exposed symbol and a helper exists for the same function,
- -// the directly exposed symbol is preferred and the helper becomes dead code, so
- -// ignore the warning here.
- -#[allow(dead_code)]
- -mod bindings_helper {
- - // Import the generated bindings for types.
- - use super::bindings_raw::*;
- - include!(concat!(
- - env!("OBJTREE"),
- - "/rust/bindings/bindings_helpers_generated.rs"
- - ));
- -}
- -
- -pub use bindings_raw::*;
- diff --git a/rust/build_error.rs b/rust/build_error.rs
- deleted file mode 100644
- --- a/rust/build_error.rs
- +++ /dev/null
- @@ -1,31 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Build-time error.
- -//!
- -//! This crate provides a [const function][const-functions] `build_error`, which will panic in
- -//! compile-time if executed in [const context][const-context], and will cause a build error
- -//! if not executed at compile time and the optimizer does not optimise away the call.
- -//!
- -//! It is used by `build_assert!` in the kernel crate, allowing checking of
- -//! conditions that could be checked statically, but could not be enforced in
- -//! Rust yet (e.g. perform some checks in [const functions][const-functions], but those
- -//! functions could still be called in the runtime).
- -//!
- -//! For details on constant evaluation in Rust, please see the [Reference][const-eval].
- -//!
- -//! [const-eval]: https://doc.rust-lang.org/reference/const_eval.html
- -//! [const-functions]: https://doc.rust-lang.org/reference/const_eval.html#const-functions
- -//! [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context
- -
- -#![no_std]
- -
- -/// Panics if executed in [const context][const-context], or triggers a build error if not.
- -///
- -/// [const-context]: https://doc.rust-lang.org/reference/const_eval.html#const-context
- -#[inline(never)]
- -#[cold]
- -#[export_name = "rust_build_error"]
- -#[track_caller]
- -pub const fn build_error(msg: &'static str) -> ! {
- - panic!("{}", msg);
- -}
- diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs
- deleted file mode 100644
- --- a/rust/compiler_builtins.rs
- +++ /dev/null
- @@ -1,74 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Our own `compiler_builtins`.
- -//!
- -//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`].
- -//! Since we do not need the vast majority of them, we avoid the dependency
- -//! by providing this file.
- -//!
- -//! At the moment, some builtins are required that should not be. For instance,
- -//! [`core`] has 128-bit integers functionality which we should not be compiling
- -//! in. We will work with upstream [`core`] to provide feature flags to disable
- -//! the parts we do not need. For the moment, we define them to [`panic!`] at
- -//! runtime for simplicity to catch mistakes, instead of performing surgery
- -//! on `core.o`.
- -//!
- -//! In any case, all these symbols are weakened to ensure we do not override
- -//! those that may be provided by the rest of the kernel.
- -//!
- -//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins
- -//! [`compiler-rt`]: https://compiler-rt.llvm.org/
- -
- -#![allow(internal_features)]
- -#![feature(compiler_builtins)]
- -#![compiler_builtins]
- -#![no_builtins]
- -#![no_std]
- -
- -macro_rules! define_panicking_intrinsics(
- - ($reason: tt, { $($ident: ident, )* }) => {
- - $(
- - #[doc(hidden)]
- - #[export_name = concat!("__rust", stringify!($ident))]
- - pub extern "C" fn $ident() {
- - panic!($reason);
- - }
- - )*
- - }
- -);
- -
- -define_panicking_intrinsics!("`f32` should not be used", {
- - __addsf3,
- - __eqsf2,
- - __gesf2,
- - __lesf2,
- - __ltsf2,
- - __mulsf3,
- - __nesf2,
- - __unordsf2,
- -});
- -
- -define_panicking_intrinsics!("`f64` should not be used", {
- - __adddf3,
- - __ledf2,
- - __ltdf2,
- - __muldf3,
- - __unorddf2,
- -});
- -
- -define_panicking_intrinsics!("`i128` should not be used", {
- - __ashrti3,
- - __muloti4,
- - __multi3,
- -});
- -
- -define_panicking_intrinsics!("`u128` should not be used", {
- - __ashlti3,
- - __lshrti3,
- - __udivmodti4,
- - __udivti3,
- - __umodti3,
- -});
- -
- -// NOTE: if you are adding a new intrinsic here, you should also add it to
- -// `redirect-intrinsics` in `rust/Makefile`.
- diff --git a/rust/exports.c b/rust/exports.c
- deleted file mode 100644
- --- a/rust/exports.c
- +++ /dev/null
- @@ -1,26 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -/*
- - * A hack to export Rust symbols for loadable modules without having to redo
- - * the entire `include/linux/export.h` logic in Rust.
- - *
- - * This requires the Rust's new/future `v0` mangling scheme because the default
- - * one ("legacy") uses invalid characters for C identifiers (thus we cannot use
- - * the `EXPORT_SYMBOL_*` macros).
- - *
- - * All symbols are exported as GPL-only to guarantee no GPL-only feature is
- - * accidentally exposed.
- - */
- -
- -#include <linux/export.h>
- -
- -#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym)
- -
- -#include "exports_core_generated.h"
- -#include "exports_alloc_generated.h"
- -#include "exports_bindings_generated.h"
- -#include "exports_kernel_generated.h"
- -
- -// For modules using `rust/build_error.rs`.
- -#ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW
- -EXPORT_SYMBOL_RUST_GPL(rust_build_error);
- -#endif
- diff --git a/rust/helpers.c b/rust/helpers.c
- deleted file mode 100644
- --- a/rust/helpers.c
- +++ /dev/null
- @@ -1,188 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -/*
- - * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions
- - * cannot be called either. This file explicitly creates functions ("helpers")
- - * that wrap those so that they can be called from Rust.
- - *
- - * Even though Rust kernel modules should never use the bindings directly, some
- - * of these helpers need to be exported because Rust generics and inlined
- - * functions may not get their code generated in the crate where they are
- - * defined. Other helpers, called from non-inline functions, may not be
- - * exported, in principle. However, in general, the Rust compiler does not
- - * guarantee codegen will be performed for a non-inline function either.
- - * Therefore, this file exports all the helpers. In the future, this may be
- - * revisited to reduce the number of exports after the compiler is informed
- - * about the places codegen is required.
- - *
- - * All symbols are exported as GPL-only to guarantee no GPL-only feature is
- - * accidentally exposed.
- - *
- - * Sorted alphabetically.
- - */
- -
- -#include <kunit/test-bug.h>
- -#include <linux/bug.h>
- -#include <linux/build_bug.h>
- -#include <linux/err.h>
- -#include <linux/errname.h>
- -#include <linux/mutex.h>
- -#include <linux/refcount.h>
- -#include <linux/sched/signal.h>
- -#include <linux/slab.h>
- -#include <linux/spinlock.h>
- -#include <linux/wait.h>
- -#include <linux/workqueue.h>
- -
- -__noreturn void rust_helper_BUG(void)
- -{
- - BUG();
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_BUG);
- -
- -void rust_helper_mutex_lock(struct mutex *lock)
- -{
- - mutex_lock(lock);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_mutex_lock);
- -
- -void rust_helper___spin_lock_init(spinlock_t *lock, const char *name,
- - struct lock_class_key *key)
- -{
- -#ifdef CONFIG_DEBUG_SPINLOCK
- - __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG);
- -#else
- - spin_lock_init(lock);
- -#endif
- -}
- -EXPORT_SYMBOL_GPL(rust_helper___spin_lock_init);
- -
- -void rust_helper_spin_lock(spinlock_t *lock)
- -{
- - spin_lock(lock);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_spin_lock);
- -
- -void rust_helper_spin_unlock(spinlock_t *lock)
- -{
- - spin_unlock(lock);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_spin_unlock);
- -
- -void rust_helper_init_wait(struct wait_queue_entry *wq_entry)
- -{
- - init_wait(wq_entry);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_init_wait);
- -
- -int rust_helper_signal_pending(struct task_struct *t)
- -{
- - return signal_pending(t);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_signal_pending);
- -
- -refcount_t rust_helper_REFCOUNT_INIT(int n)
- -{
- - return (refcount_t)REFCOUNT_INIT(n);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_REFCOUNT_INIT);
- -
- -void rust_helper_refcount_inc(refcount_t *r)
- -{
- - refcount_inc(r);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_refcount_inc);
- -
- -bool rust_helper_refcount_dec_and_test(refcount_t *r)
- -{
- - return refcount_dec_and_test(r);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test);
- -
- -__force void *rust_helper_ERR_PTR(long err)
- -{
- - return ERR_PTR(err);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR);
- -
- -bool rust_helper_IS_ERR(__force const void *ptr)
- -{
- - return IS_ERR(ptr);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_IS_ERR);
- -
- -long rust_helper_PTR_ERR(__force const void *ptr)
- -{
- - return PTR_ERR(ptr);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR);
- -
- -const char *rust_helper_errname(int err)
- -{
- - return errname(err);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_errname);
- -
- -struct task_struct *rust_helper_get_current(void)
- -{
- - return current;
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_get_current);
- -
- -void rust_helper_get_task_struct(struct task_struct *t)
- -{
- - get_task_struct(t);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_get_task_struct);
- -
- -void rust_helper_put_task_struct(struct task_struct *t)
- -{
- - put_task_struct(t);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_put_task_struct);
- -
- -struct kunit *rust_helper_kunit_get_current_test(void)
- -{
- - return kunit_get_current_test();
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_kunit_get_current_test);
- -
- -void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func,
- - bool onstack, const char *name,
- - struct lock_class_key *key)
- -{
- - __init_work(work, onstack);
- - work->data = (atomic_long_t)WORK_DATA_INIT();
- - lockdep_init_map(&work->lockdep_map, name, key, 0);
- - INIT_LIST_HEAD(&work->entry);
- - work->func = func;
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_init_work_with_key);
- -
- -void * __must_check __realloc_size(2)
- -rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags)
- -{
- - return krealloc(objp, new_size, flags);
- -}
- -EXPORT_SYMBOL_GPL(rust_helper_krealloc);
- -
- -/*
- - * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can
- - * use it in contexts where Rust expects a `usize` like slice (array) indices.
- - * `usize` is defined to be the same as C's `uintptr_t` type (can hold any
- - * pointer) but not necessarily the same as `size_t` (can hold the size of any
- - * single object). Most modern platforms use the same concrete integer type for
- - * both of them, but in case we find ourselves on a platform where
- - * that's not true, fail early instead of risking ABI or
- - * integer-overflow issues.
- - *
- - * If your platform fails this assertion, it means that you are in
- - * danger of integer-overflow bugs (even if you attempt to add
- - * `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on
- - * your platform such that `size_t` matches `uintptr_t` (i.e., to increase
- - * `size_t`, because `uintptr_t` has to be at least as big as `size_t`).
- - */
- -static_assert(
- - sizeof(size_t) == sizeof(uintptr_t) &&
- - __alignof__(size_t) == __alignof__(uintptr_t),
- - "Rust code expects C `size_t` to match Rust `usize`"
- -);
- diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs
- deleted file mode 100644
- --- a/rust/kernel/alloc.rs
- +++ /dev/null
- @@ -1,73 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Extensions to the [`alloc`] crate.
- -
- -#[cfg(not(test))]
- -#[cfg(not(testlib))]
- -mod allocator;
- -pub mod box_ext;
- -pub mod vec_ext;
- -
- -/// Indicates an allocation error.
- -#[derive(Copy, Clone, PartialEq, Eq, Debug)]
- -pub struct AllocError;
- -
- -/// Flags to be used when allocating memory.
- -///
- -/// They can be combined with the operators `|`, `&`, and `!`.
- -///
- -/// Values can be used from the [`flags`] module.
- -#[derive(Clone, Copy)]
- -pub struct Flags(u32);
- -
- -impl core::ops::BitOr for Flags {
- - type Output = Self;
- - fn bitor(self, rhs: Self) -> Self::Output {
- - Self(self.0 | rhs.0)
- - }
- -}
- -
- -impl core::ops::BitAnd for Flags {
- - type Output = Self;
- - fn bitand(self, rhs: Self) -> Self::Output {
- - Self(self.0 & rhs.0)
- - }
- -}
- -
- -impl core::ops::Not for Flags {
- - type Output = Self;
- - fn not(self) -> Self::Output {
- - Self(!self.0)
- - }
- -}
- -
- -/// Allocation flags.
- -///
- -/// These are meant to be used in functions that can allocate memory.
- -pub mod flags {
- - use super::Flags;
- -
- - /// Zeroes out the allocated memory.
- - ///
- - /// This is normally or'd with other flags.
- - pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO);
- -
- - /// Users can not sleep and need the allocation to succeed.
- - ///
- - /// A lower watermark is applied to allow access to "atomic reserves". The current
- - /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g.
- - /// raw_spin_lock). The same applies to [`GFP_NOWAIT`].
- - pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC);
- -
- - /// Typical for kernel-internal allocations. The caller requires ZONE_NORMAL or a lower zone
- - /// for direct access but can direct reclaim.
- - pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL);
- -
- - /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg.
- - pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT);
- -
- - /// Ror kernel allocations that should not stall for direct reclaim, start physical IO or
- - /// use any filesystem callback. It is very likely to fail to allocate memory, even for very
- - /// small allocations.
- - pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT);
- -}
- diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs
- deleted file mode 100644
- --- a/rust/kernel/alloc/allocator.rs
- +++ /dev/null
- @@ -1,81 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Allocator support.
- -
- -use super::{flags::*, Flags};
- -use core::alloc::{GlobalAlloc, Layout};
- -use core::ptr;
- -
- -struct KernelAllocator;
- -
- -/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment.
- -///
- -/// # Safety
- -///
- -/// - `ptr` can be either null or a pointer which has been allocated by this allocator.
- -/// - `new_layout` must have a non-zero size.
- -pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 {
- - // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first.
- - let layout = new_layout.pad_to_align();
- -
- - let mut size = layout.size();
- -
- - if layout.align() > bindings::ARCH_SLAB_MINALIGN {
- - // The alignment requirement exceeds the slab guarantee, thus try to enlarge the size
- - // to use the "power-of-two" size/alignment guarantee (see comments in `kmalloc()` for
- - // more information).
- - //
- - // Note that `layout.size()` (after padding) is guaranteed to be a multiple of
- - // `layout.align()`, so `next_power_of_two` gives enough alignment guarantee.
- - size = size.next_power_of_two();
- - }
- -
- - // SAFETY:
- - // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the
- - // function safety requirement.
- - // - `size` is greater than 0 since it's either a `layout.size()` (which cannot be zero
- - // according to the function safety requirement) or a result from `next_power_of_two()`.
- - unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 }
- -}
- -
- -unsafe impl GlobalAlloc for KernelAllocator {
- - unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
- - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety
- - // requirement.
- - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) }
- - }
- -
- - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
- - unsafe {
- - bindings::kfree(ptr as *const core::ffi::c_void);
- - }
- - }
- -
- - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
- - // SAFETY:
- - // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not
- - // overflow `isize` by the function safety requirement.
- - // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two).
- - let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
- -
- - // SAFETY:
- - // - `ptr` is either null or a pointer allocated by this allocator by the function safety
- - // requirement.
- - // - the size of `layout` is not zero because `new_size` is not zero by the function safety
- - // requirement.
- - unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) }
- - }
- -
- - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
- - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety
- - // requirement.
- - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) }
- - }
- -}
- -
- -#[global_allocator]
- -static ALLOCATOR: KernelAllocator = KernelAllocator;
- -
- -// See <https://github.com/rust-lang/rust/pull/86844>.
- -#[no_mangle]
- -static __rust_no_alloc_shim_is_unstable: u8 = 0;
- diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs
- deleted file mode 100644
- --- a/rust/kernel/alloc/box_ext.rs
- +++ /dev/null
- @@ -1,56 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Extensions to [`Box`] for fallible allocations.
- -
- -use super::{AllocError, Flags};
- -use alloc::boxed::Box;
- -use core::mem::MaybeUninit;
- -
- -/// Extensions to [`Box`].
- -pub trait BoxExt<T>: Sized {
- - /// Allocates a new box.
- - ///
- - /// The allocation may fail, in which case an error is returned.
- - fn new(x: T, flags: Flags) -> Result<Self, AllocError>;
- -
- - /// Allocates a new uninitialised box.
- - ///
- - /// The allocation may fail, in which case an error is returned.
- - fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError>;
- -}
- -
- -impl<T> BoxExt<T> for Box<T> {
- - fn new(x: T, flags: Flags) -> Result<Self, AllocError> {
- - let b = <Self as BoxExt<_>>::new_uninit(flags)?;
- - Ok(Box::write(b, x))
- - }
- -
- - #[cfg(any(test, testlib))]
- - fn new_uninit(_flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> {
- - Ok(Box::new_uninit())
- - }
- -
- - #[cfg(not(any(test, testlib)))]
- - fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> {
- - let ptr = if core::mem::size_of::<MaybeUninit<T>>() == 0 {
- - core::ptr::NonNull::<_>::dangling().as_ptr()
- - } else {
- - let layout = core::alloc::Layout::new::<MaybeUninit<T>>();
- -
- - // SAFETY: Memory is being allocated (first arg is null). The only other source of
- - // safety issues is sleeping on atomic context, which is addressed by klint. Lastly,
- - // the type is not a SZT (checked above).
- - let ptr =
- - unsafe { super::allocator::krealloc_aligned(core::ptr::null_mut(), layout, flags) };
- - if ptr.is_null() {
- - return Err(AllocError);
- - }
- -
- - ptr.cast::<MaybeUninit<T>>()
- - };
- -
- - // SAFETY: For non-zero-sized types, we allocate above using the global allocator. For
- - // zero-sized types, we use `NonNull::dangling`.
- - Ok(unsafe { Box::from_raw(ptr) })
- - }
- -}
- diff --git a/rust/kernel/alloc/vec_ext.rs b/rust/kernel/alloc/vec_ext.rs
- deleted file mode 100644
- --- a/rust/kernel/alloc/vec_ext.rs
- +++ /dev/null
- @@ -1,185 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Extensions to [`Vec`] for fallible allocations.
- -
- -use super::{AllocError, Flags};
- -use alloc::vec::Vec;
- -
- -/// Extensions to [`Vec`].
- -pub trait VecExt<T>: Sized {
- - /// Creates a new [`Vec`] instance with at least the given capacity.
- - ///
- - /// # Examples
- - ///
- - /// ```
- - /// let v = Vec::<u32>::with_capacity(20, GFP_KERNEL)?;
- - ///
- - /// assert!(v.capacity() >= 20);
- - /// # Ok::<(), Error>(())
- - /// ```
- - fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError>;
- -
- - /// Appends an element to the back of the [`Vec`] instance.
- - ///
- - /// # Examples
- - ///
- - /// ```
- - /// let mut v = Vec::new();
- - /// v.push(1, GFP_KERNEL)?;
- - /// assert_eq!(&v, &[1]);
- - ///
- - /// v.push(2, GFP_KERNEL)?;
- - /// assert_eq!(&v, &[1, 2]);
- - /// # Ok::<(), Error>(())
- - /// ```
- - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError>;
- -
- - /// Pushes clones of the elements of slice into the [`Vec`] instance.
- - ///
- - /// # Examples
- - ///
- - /// ```
- - /// let mut v = Vec::new();
- - /// v.push(1, GFP_KERNEL)?;
- - ///
- - /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?;
- - /// assert_eq!(&v, &[1, 20, 30, 40]);
- - ///
- - /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?;
- - /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]);
- - /// # Ok::<(), Error>(())
- - /// ```
- - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError>
- - where
- - T: Clone;
- -
- - /// Ensures that the capacity exceeds the length by at least `additional` elements.
- - ///
- - /// # Examples
- - ///
- - /// ```
- - /// let mut v = Vec::new();
- - /// v.push(1, GFP_KERNEL)?;
- - ///
- - /// v.reserve(10, GFP_KERNEL)?;
- - /// let cap = v.capacity();
- - /// assert!(cap >= 10);
- - ///
- - /// v.reserve(10, GFP_KERNEL)?;
- - /// let new_cap = v.capacity();
- - /// assert_eq!(new_cap, cap);
- - ///
- - /// # Ok::<(), Error>(())
- - /// ```
- - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError>;
- -}
- -
- -impl<T> VecExt<T> for Vec<T> {
- - fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError> {
- - let mut v = Vec::new();
- - <Self as VecExt<_>>::reserve(&mut v, capacity, flags)?;
- - Ok(v)
- - }
- -
- - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> {
- - <Self as VecExt<_>>::reserve(self, 1, flags)?;
- - let s = self.spare_capacity_mut();
- - s[0].write(v);
- -
- - // SAFETY: We just initialised the first spare entry, so it is safe to increase the length
- - // by 1. We also know that the new length is <= capacity because of the previous call to
- - // `reserve` above.
- - unsafe { self.set_len(self.len() + 1) };
- - Ok(())
- - }
- -
- - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError>
- - where
- - T: Clone,
- - {
- - <Self as VecExt<_>>::reserve(self, other.len(), flags)?;
- - for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) {
- - slot.write(item.clone());
- - }
- -
- - // SAFETY: We just initialised the `other.len()` spare entries, so it is safe to increase
- - // the length by the same amount. We also know that the new length is <= capacity because
- - // of the previous call to `reserve` above.
- - unsafe { self.set_len(self.len() + other.len()) };
- - Ok(())
- - }
- -
- - #[cfg(any(test, testlib))]
- - fn reserve(&mut self, additional: usize, _flags: Flags) -> Result<(), AllocError> {
- - Vec::reserve(self, additional);
- - Ok(())
- - }
- -
- - #[cfg(not(any(test, testlib)))]
- - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> {
- - let len = self.len();
- - let cap = self.capacity();
- -
- - if cap - len >= additional {
- - return Ok(());
- - }
- -
- - if core::mem::size_of::<T>() == 0 {
- - // The capacity is already `usize::MAX` for SZTs, we can't go higher.
- - return Err(AllocError);
- - }
- -
- - // We know cap is <= `isize::MAX` because `Layout::array` fails if the resulting byte size
- - // is greater than `isize::MAX`. So the multiplication by two won't overflow.
- - let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?);
- - let layout = core::alloc::Layout::array::<T>(new_cap).map_err(|_| AllocError)?;
- -
- - let (old_ptr, len, cap) = destructure(self);
- -
- - // We need to make sure that `ptr` is either NULL or comes from a previous call to
- - // `krealloc_aligned`. A `Vec<T>`'s `ptr` value is not guaranteed to be NULL and might be
- - // dangling after being created with `Vec::new`. Instead, we can rely on `Vec<T>`'s capacity
- - // to be zero if no memory has been allocated yet.
- - let ptr = if cap == 0 {
- - core::ptr::null_mut()
- - } else {
- - old_ptr
- - };
- -
- - // SAFETY: `ptr` is valid because it's either NULL or comes from a previous call to
- - // `krealloc_aligned`. We also verified that the type is not a ZST.
- - let new_ptr = unsafe { super::allocator::krealloc_aligned(ptr.cast(), layout, flags) };
- - if new_ptr.is_null() {
- - // SAFETY: We are just rebuilding the existing `Vec` with no changes.
- - unsafe { rebuild(self, old_ptr, len, cap) };
- - Err(AllocError)
- - } else {
- - // SAFETY: `ptr` has been reallocated with the layout for `new_cap` elements. New cap
- - // is greater than `cap`, so it continues to be >= `len`.
- - unsafe { rebuild(self, new_ptr.cast::<T>(), len, new_cap) };
- - Ok(())
- - }
- - }
- -}
- -
- -#[cfg(not(any(test, testlib)))]
- -fn destructure<T>(v: &mut Vec<T>) -> (*mut T, usize, usize) {
- - let mut tmp = Vec::new();
- - core::mem::swap(&mut tmp, v);
- - let mut tmp = core::mem::ManuallyDrop::new(tmp);
- - let len = tmp.len();
- - let cap = tmp.capacity();
- - (tmp.as_mut_ptr(), len, cap)
- -}
- -
- -/// Rebuilds a `Vec` from a pointer, length, and capacity.
- -///
- -/// # Safety
- -///
- -/// The same as [`Vec::from_raw_parts`].
- -#[cfg(not(any(test, testlib)))]
- -unsafe fn rebuild<T>(v: &mut Vec<T>, ptr: *mut T, len: usize, cap: usize) {
- - // SAFETY: The safety requirements from this function satisfy those of `from_raw_parts`.
- - let mut tmp = unsafe { Vec::from_raw_parts(ptr, len, cap) };
- - core::mem::swap(&mut tmp, v);
- -}
- diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs
- deleted file mode 100644
- --- a/rust/kernel/build_assert.rs
- +++ /dev/null
- @@ -1,84 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Build-time assert.
- -
- -/// Fails the build if the code path calling `build_error!` can possibly be executed.
- -///
- -/// If the macro is executed in const context, `build_error!` will panic.
- -/// If the compiler or optimizer cannot guarantee that `build_error!` can never
- -/// be called, a build error will be triggered.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// # use kernel::build_error;
- -/// #[inline]
- -/// fn foo(a: usize) -> usize {
- -/// a.checked_add(1).unwrap_or_else(|| build_error!("overflow"))
- -/// }
- -///
- -/// assert_eq!(foo(usize::MAX - 1), usize::MAX); // OK.
- -/// // foo(usize::MAX); // Fails to compile.
- -/// ```
- -#[macro_export]
- -macro_rules! build_error {
- - () => {{
- - $crate::build_error("")
- - }};
- - ($msg:expr) => {{
- - $crate::build_error($msg)
- - }};
- -}
- -
- -/// Asserts that a boolean expression is `true` at compile time.
- -///
- -/// If the condition is evaluated to `false` in const context, `build_assert!`
- -/// will panic. If the compiler or optimizer cannot guarantee the condition will
- -/// be evaluated to `true`, a build error will be triggered.
- -///
- -/// [`static_assert!`] should be preferred to `build_assert!` whenever possible.
- -///
- -/// # Examples
- -///
- -/// These examples show that different types of [`assert!`] will trigger errors
- -/// at different stage of compilation. It is preferred to err as early as
- -/// possible, so [`static_assert!`] should be used whenever possible.
- -/// ```ignore
- -/// fn foo() {
- -/// static_assert!(1 > 1); // Compile-time error
- -/// build_assert!(1 > 1); // Build-time error
- -/// assert!(1 > 1); // Run-time error
- -/// }
- -/// ```
- -///
- -/// When the condition refers to generic parameters or parameters of an inline function,
- -/// [`static_assert!`] cannot be used. Use `build_assert!` in this scenario.
- -/// ```
- -/// fn foo<const N: usize>() {
- -/// // `static_assert!(N > 1);` is not allowed
- -/// build_assert!(N > 1); // Build-time check
- -/// assert!(N > 1); // Run-time check
- -/// }
- -///
- -/// #[inline]
- -/// fn bar(n: usize) {
- -/// // `static_assert!(n > 1);` is not allowed
- -/// build_assert!(n > 1); // Build-time check
- -/// assert!(n > 1); // Run-time check
- -/// }
- -/// ```
- -///
- -/// [`static_assert!`]: crate::static_assert!
- -#[macro_export]
- -macro_rules! build_assert {
- - ($cond:expr $(,)?) => {{
- - if !$cond {
- - $crate::build_error(concat!("assertion failed: ", stringify!($cond)));
- - }
- - }};
- - ($cond:expr, $msg:expr) => {{
- - if !$cond {
- - $crate::build_error($msg);
- - }
- - }};
- -}
- diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs
- deleted file mode 100644
- --- a/rust/kernel/error.rs
- +++ /dev/null
- @@ -1,327 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Kernel errors.
- -//!
- -//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h)
- -
- -use crate::{alloc::AllocError, str::CStr};
- -
- -use alloc::alloc::LayoutError;
- -
- -use core::fmt;
- -use core::num::TryFromIntError;
- -use core::str::Utf8Error;
- -
- -/// Contains the C-compatible error codes.
- -#[rustfmt::skip]
- -pub mod code {
- - macro_rules! declare_err {
- - ($err:tt $(,)? $($doc:expr),+) => {
- - $(
- - #[doc = $doc]
- - )*
- - pub const $err: super::Error = super::Error(-(crate::bindings::$err as i32));
- - };
- - }
- -
- - declare_err!(EPERM, "Operation not permitted.");
- - declare_err!(ENOENT, "No such file or directory.");
- - declare_err!(ESRCH, "No such process.");
- - declare_err!(EINTR, "Interrupted system call.");
- - declare_err!(EIO, "I/O error.");
- - declare_err!(ENXIO, "No such device or address.");
- - declare_err!(E2BIG, "Argument list too long.");
- - declare_err!(ENOEXEC, "Exec format error.");
- - declare_err!(EBADF, "Bad file number.");
- - declare_err!(ECHILD, "No child processes.");
- - declare_err!(EAGAIN, "Try again.");
- - declare_err!(ENOMEM, "Out of memory.");
- - declare_err!(EACCES, "Permission denied.");
- - declare_err!(EFAULT, "Bad address.");
- - declare_err!(ENOTBLK, "Block device required.");
- - declare_err!(EBUSY, "Device or resource busy.");
- - declare_err!(EEXIST, "File exists.");
- - declare_err!(EXDEV, "Cross-device link.");
- - declare_err!(ENODEV, "No such device.");
- - declare_err!(ENOTDIR, "Not a directory.");
- - declare_err!(EISDIR, "Is a directory.");
- - declare_err!(EINVAL, "Invalid argument.");
- - declare_err!(ENFILE, "File table overflow.");
- - declare_err!(EMFILE, "Too many open files.");
- - declare_err!(ENOTTY, "Not a typewriter.");
- - declare_err!(ETXTBSY, "Text file busy.");
- - declare_err!(EFBIG, "File too large.");
- - declare_err!(ENOSPC, "No space left on device.");
- - declare_err!(ESPIPE, "Illegal seek.");
- - declare_err!(EROFS, "Read-only file system.");
- - declare_err!(EMLINK, "Too many links.");
- - declare_err!(EPIPE, "Broken pipe.");
- - declare_err!(EDOM, "Math argument out of domain of func.");
- - declare_err!(ERANGE, "Math result not representable.");
- - declare_err!(ERESTARTSYS, "Restart the system call.");
- - declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted.");
- - declare_err!(ERESTARTNOHAND, "Restart if no handler.");
- - declare_err!(ENOIOCTLCMD, "No ioctl command.");
- - declare_err!(ERESTART_RESTARTBLOCK, "Restart by calling sys_restart_syscall.");
- - declare_err!(EPROBE_DEFER, "Driver requests probe retry.");
- - declare_err!(EOPENSTALE, "Open found a stale dentry.");
- - declare_err!(ENOPARAM, "Parameter not supported.");
- - declare_err!(EBADHANDLE, "Illegal NFS file handle.");
- - declare_err!(ENOTSYNC, "Update synchronization mismatch.");
- - declare_err!(EBADCOOKIE, "Cookie is stale.");
- - declare_err!(ENOTSUPP, "Operation is not supported.");
- - declare_err!(ETOOSMALL, "Buffer or request is too small.");
- - declare_err!(ESERVERFAULT, "An untranslatable error occurred.");
- - declare_err!(EBADTYPE, "Type not supported by server.");
- - declare_err!(EJUKEBOX, "Request initiated, but will not complete before timeout.");
- - declare_err!(EIOCBQUEUED, "iocb queued, will get completion event.");
- - declare_err!(ERECALLCONFLICT, "Conflict with recalled state.");
- - declare_err!(ENOGRACE, "NFS file lock reclaim refused.");
- -}
- -
- -/// Generic integer kernel error.
- -///
- -/// The kernel defines a set of integer generic error codes based on C and
- -/// POSIX ones. These codes may have a more specific meaning in some contexts.
- -///
- -/// # Invariants
- -///
- -/// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`).
- -#[derive(Clone, Copy, PartialEq, Eq)]
- -pub struct Error(core::ffi::c_int);
- -
- -impl Error {
- - /// Creates an [`Error`] from a kernel error code.
- - ///
- - /// It is a bug to pass an out-of-range `errno`. `EINVAL` would
- - /// be returned in such a case.
- - pub(crate) fn from_errno(errno: core::ffi::c_int) -> Error {
- - if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 {
- - // TODO: Make it a `WARN_ONCE` once available.
- - crate::pr_warn!(
- - "attempted to create `Error` with out of range `errno`: {}",
- - errno
- - );
- - return code::EINVAL;
- - }
- -
- - // INVARIANT: The check above ensures the type invariant
- - // will hold.
- - Error(errno)
- - }
- -
- - /// Creates an [`Error`] from a kernel error code.
- - ///
- - /// # Safety
- - ///
- - /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`).
- - unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error {
- - // INVARIANT: The contract ensures the type invariant
- - // will hold.
- - Error(errno)
- - }
- -
- - /// Returns the kernel error code.
- - pub fn to_errno(self) -> core::ffi::c_int {
- - self.0
- - }
- -
- - /// Returns the error encoded as a pointer.
- - #[allow(dead_code)]
- - pub(crate) fn to_ptr<T>(self) -> *mut T {
- - // SAFETY: `self.0` is a valid error due to its invariant.
- - unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ }
- - }
- -
- - /// Returns a string representing the error, if one exists.
- - #[cfg(not(testlib))]
- - pub fn name(&self) -> Option<&'static CStr> {
- - // SAFETY: Just an FFI call, there are no extra safety requirements.
- - let ptr = unsafe { bindings::errname(-self.0) };
- - if ptr.is_null() {
- - None
- - } else {
- - // SAFETY: The string returned by `errname` is static and `NUL`-terminated.
- - Some(unsafe { CStr::from_char_ptr(ptr) })
- - }
- - }
- -
- - /// Returns a string representing the error, if one exists.
- - ///
- - /// When `testlib` is configured, this always returns `None` to avoid the dependency on a
- - /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still
- - /// run in userspace.
- - #[cfg(testlib)]
- - pub fn name(&self) -> Option<&'static CStr> {
- - None
- - }
- -}
- -
- -impl fmt::Debug for Error {
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - match self.name() {
- - // Print out number if no name can be found.
- - None => f.debug_tuple("Error").field(&-self.0).finish(),
- - // SAFETY: These strings are ASCII-only.
- - Some(name) => f
- - .debug_tuple(unsafe { core::str::from_utf8_unchecked(name) })
- - .finish(),
- - }
- - }
- -}
- -
- -impl From<AllocError> for Error {
- - fn from(_: AllocError) -> Error {
- - code::ENOMEM
- - }
- -}
- -
- -impl From<TryFromIntError> for Error {
- - fn from(_: TryFromIntError) -> Error {
- - code::EINVAL
- - }
- -}
- -
- -impl From<Utf8Error> for Error {
- - fn from(_: Utf8Error) -> Error {
- - code::EINVAL
- - }
- -}
- -
- -impl From<LayoutError> for Error {
- - fn from(_: LayoutError) -> Error {
- - code::ENOMEM
- - }
- -}
- -
- -impl From<core::fmt::Error> for Error {
- - fn from(_: core::fmt::Error) -> Error {
- - code::EINVAL
- - }
- -}
- -
- -impl From<core::convert::Infallible> for Error {
- - fn from(e: core::convert::Infallible) -> Error {
- - match e {}
- - }
- -}
- -
- -/// A [`Result`] with an [`Error`] error type.
- -///
- -/// To be used as the return type for functions that may fail.
- -///
- -/// # Error codes in C and Rust
- -///
- -/// In C, it is common that functions indicate success or failure through
- -/// their return value; modifying or returning extra data through non-`const`
- -/// pointer parameters. In particular, in the kernel, functions that may fail
- -/// typically return an `int` that represents a generic error code. We model
- -/// those as [`Error`].
- -///
- -/// In Rust, it is idiomatic to model functions that may fail as returning
- -/// a [`Result`]. Since in the kernel many functions return an error code,
- -/// [`Result`] is a type alias for a [`core::result::Result`] that uses
- -/// [`Error`] as its error type.
- -///
- -/// Note that even if a function does not return anything when it succeeds,
- -/// it should still be modeled as returning a `Result` rather than
- -/// just an [`Error`].
- -pub type Result<T = (), E = Error> = core::result::Result<T, E>;
- -
- -/// Converts an integer as returned by a C kernel function to an error if it's negative, and
- -/// `Ok(())` otherwise.
- -pub fn to_result(err: core::ffi::c_int) -> Result {
- - if err < 0 {
- - Err(Error::from_errno(err))
- - } else {
- - Ok(())
- - }
- -}
- -
- -/// Transform a kernel "error pointer" to a normal pointer.
- -///
- -/// Some kernel C API functions return an "error pointer" which optionally
- -/// embeds an `errno`. Callers are supposed to check the returned pointer
- -/// for errors. This function performs the check and converts the "error pointer"
- -/// to a normal pointer in an idiomatic fashion.
- -///
- -/// # Examples
- -///
- -/// ```ignore
- -/// # use kernel::from_err_ptr;
- -/// # use kernel::bindings;
- -/// fn devm_platform_ioremap_resource(
- -/// pdev: &mut PlatformDevice,
- -/// index: u32,
- -/// ) -> Result<*mut core::ffi::c_void> {
- -/// // SAFETY: `pdev` points to a valid platform device. There are no safety requirements
- -/// // on `index`.
- -/// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) })
- -/// }
- -/// ```
- -// TODO: Remove `dead_code` marker once an in-kernel client is available.
- -#[allow(dead_code)]
- -pub(crate) fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> {
- - // CAST: Casting a pointer to `*const core::ffi::c_void` is always valid.
- - let const_ptr: *const core::ffi::c_void = ptr.cast();
- - // SAFETY: The FFI function does not deref the pointer.
- - if unsafe { bindings::IS_ERR(const_ptr) } {
- - // SAFETY: The FFI function does not deref the pointer.
- - let err = unsafe { bindings::PTR_ERR(const_ptr) };
- - // CAST: If `IS_ERR()` returns `true`,
- - // then `PTR_ERR()` is guaranteed to return a
- - // negative value greater-or-equal to `-bindings::MAX_ERRNO`,
- - // which always fits in an `i16`, as per the invariant above.
- - // And an `i16` always fits in an `i32`. So casting `err` to
- - // an `i32` can never overflow, and is always valid.
- - //
- - // SAFETY: `IS_ERR()` ensures `err` is a
- - // negative value greater-or-equal to `-bindings::MAX_ERRNO`.
- - #[allow(clippy::unnecessary_cast)]
- - return Err(unsafe { Error::from_errno_unchecked(err as core::ffi::c_int) });
- - }
- - Ok(ptr)
- -}
- -
- -/// Calls a closure returning a [`crate::error::Result<T>`] and converts the result to
- -/// a C integer result.
- -///
- -/// This is useful when calling Rust functions that return [`crate::error::Result<T>`]
- -/// from inside `extern "C"` functions that need to return an integer error result.
- -///
- -/// `T` should be convertible from an `i16` via `From<i16>`.
- -///
- -/// # Examples
- -///
- -/// ```ignore
- -/// # use kernel::from_result;
- -/// # use kernel::bindings;
- -/// unsafe extern "C" fn probe_callback(
- -/// pdev: *mut bindings::platform_device,
- -/// ) -> core::ffi::c_int {
- -/// from_result(|| {
- -/// let ptr = devm_alloc(pdev)?;
- -/// bindings::platform_set_drvdata(pdev, ptr);
- -/// Ok(0)
- -/// })
- -/// }
- -/// ```
- -// TODO: Remove `dead_code` marker once an in-kernel client is available.
- -#[allow(dead_code)]
- -pub(crate) fn from_result<T, F>(f: F) -> T
- -where
- - T: From<i16>,
- - F: FnOnce() -> Result<T>,
- -{
- - match f() {
- - Ok(v) => v,
- - // NO-OVERFLOW: negative `errno`s are no smaller than `-bindings::MAX_ERRNO`,
- - // `-bindings::MAX_ERRNO` fits in an `i16` as per invariant above,
- - // therefore a negative `errno` always fits in an `i16` and will not overflow.
- - Err(e) => T::from(e.to_errno() as i16),
- - }
- -}
- -
- -/// Error message for calling a default function of a [`#[vtable]`](macros::vtable) trait.
- -pub const VTABLE_DEFAULT_ERROR: &str =
- - "This function must not be called, see the #[vtable] documentation.";
- diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs
- deleted file mode 100644
- --- a/rust/kernel/init.rs
- +++ /dev/null
- @@ -1,1352 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
- -//!
- -//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
- -//! overflow.
- -//!
- -//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
- -//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
- -//!
- -//! # Overview
- -//!
- -//! To initialize a `struct` with an in-place constructor you will need two things:
- -//! - an in-place constructor,
- -//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
- -//! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]).
- -//!
- -//! To get an in-place constructor there are generally three options:
- -//! - directly creating an in-place constructor using the [`pin_init!`] macro,
- -//! - a custom function/macro returning an in-place constructor provided by someone else,
- -//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
- -//!
- -//! Aside from pinned initialization, this API also supports in-place construction without pinning,
- -//! the macros/types/functions are generally named like the pinned variants without the `pin`
- -//! prefix.
- -//!
- -//! # Examples
- -//!
- -//! ## Using the [`pin_init!`] macro
- -//!
- -//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
- -//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
- -//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
- -//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
- -//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
- -//!
- -//! ```rust
- -//! # #![allow(clippy::disallowed_names)]
- -//! use kernel::sync::{new_mutex, Mutex};
- -//! # use core::pin::Pin;
- -//! #[pin_data]
- -//! struct Foo {
- -//! #[pin]
- -//! a: Mutex<usize>,
- -//! b: u32,
- -//! }
- -//!
- -//! let foo = pin_init!(Foo {
- -//! a <- new_mutex!(42, "Foo::a"),
- -//! b: 24,
- -//! });
- -//! ```
- -//!
- -//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
- -//! (or just the stack) to actually initialize a `Foo`:
- -//!
- -//! ```rust
- -//! # #![allow(clippy::disallowed_names)]
- -//! # use kernel::sync::{new_mutex, Mutex};
- -//! # use core::pin::Pin;
- -//! # #[pin_data]
- -//! # struct Foo {
- -//! # #[pin]
- -//! # a: Mutex<usize>,
- -//! # b: u32,
- -//! # }
- -//! # let foo = pin_init!(Foo {
- -//! # a <- new_mutex!(42, "Foo::a"),
- -//! # b: 24,
- -//! # });
- -//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo, GFP_KERNEL);
- -//! ```
- -//!
- -//! For more information see the [`pin_init!`] macro.
- -//!
- -//! ## Using a custom function/macro that returns an initializer
- -//!
- -//! Many types from the kernel supply a function/macro that returns an initializer, because the
- -//! above method only works for types where you can access the fields.
- -//!
- -//! ```rust
- -//! # use kernel::sync::{new_mutex, Arc, Mutex};
- -//! let mtx: Result<Arc<Mutex<usize>>> =
- -//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
- -//! ```
- -//!
- -//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
- -//!
- -//! ```rust
- -//! # #![allow(clippy::disallowed_names)]
- -//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
- -//! #[pin_data]
- -//! struct DriverData {
- -//! #[pin]
- -//! status: Mutex<i32>,
- -//! buffer: Box<[u8; 1_000_000]>,
- -//! }
- -//!
- -//! impl DriverData {
- -//! fn new() -> impl PinInit<Self, Error> {
- -//! try_pin_init!(Self {
- -//! status <- new_mutex!(0, "DriverData::status"),
- -//! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?,
- -//! })
- -//! }
- -//! }
- -//! ```
- -//!
- -//! ## Manual creation of an initializer
- -//!
- -//! Often when working with primitives the previous approaches are not sufficient. That is where
- -//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
- -//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
- -//! actually does the initialization in the correct way. Here are the things to look out for
- -//! (we are calling the parameter to the closure `slot`):
- -//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
- -//! `slot` now contains a valid bit pattern for the type `T`,
- -//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
- -//! you need to take care to clean up anything if your initialization fails mid-way,
- -//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
- -//! `slot` gets called.
- -//!
- -//! ```rust
- -//! # #![allow(unreachable_pub, clippy::disallowed_names)]
- -//! use kernel::{init, types::Opaque};
- -//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
- -//! # mod bindings {
- -//! # #![allow(non_camel_case_types)]
- -//! # pub struct foo;
- -//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
- -//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
- -//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
- -//! # }
- -//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
- -//! # trait FromErrno {
- -//! # fn from_errno(errno: core::ffi::c_int) -> Error {
- -//! # // Dummy error that can be constructed outside the `kernel` crate.
- -//! # Error::from(core::fmt::Error)
- -//! # }
- -//! # }
- -//! # impl FromErrno for Error {}
- -//! /// # Invariants
- -//! ///
- -//! /// `foo` is always initialized
- -//! #[pin_data(PinnedDrop)]
- -//! pub struct RawFoo {
- -//! #[pin]
- -//! foo: Opaque<bindings::foo>,
- -//! #[pin]
- -//! _p: PhantomPinned,
- -//! }
- -//!
- -//! impl RawFoo {
- -//! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
- -//! // SAFETY:
- -//! // - when the closure returns `Ok(())`, then it has successfully initialized and
- -//! // enabled `foo`,
- -//! // - when it returns `Err(e)`, then it has cleaned up before
- -//! unsafe {
- -//! init::pin_init_from_closure(move |slot: *mut Self| {
- -//! // `slot` contains uninit memory, avoid creating a reference.
- -//! let foo = addr_of_mut!((*slot).foo);
- -//!
- -//! // Initialize the `foo`
- -//! bindings::init_foo(Opaque::raw_get(foo));
- -//!
- -//! // Try to enable it.
- -//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
- -//! if err != 0 {
- -//! // Enabling has failed, first clean up the foo and then return the error.
- -//! bindings::destroy_foo(Opaque::raw_get(foo));
- -//! return Err(Error::from_errno(err));
- -//! }
- -//!
- -//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
- -//! Ok(())
- -//! })
- -//! }
- -//! }
- -//! }
- -//!
- -//! #[pinned_drop]
- -//! impl PinnedDrop for RawFoo {
- -//! fn drop(self: Pin<&mut Self>) {
- -//! // SAFETY: Since `foo` is initialized, destroying is safe.
- -//! unsafe { bindings::destroy_foo(self.foo.get()) };
- -//! }
- -//! }
- -//! ```
- -//!
- -//! For the special case where initializing a field is a single FFI-function call that cannot fail,
- -//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
- -//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
- -//! with [`pin_init!`].
- -//!
- -//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
- -//! the `kernel` crate. The [`sync`] module is a good starting point.
- -//!
- -//! [`sync`]: kernel::sync
- -//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
- -//! [structurally pinned fields]:
- -//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
- -//! [stack]: crate::stack_pin_init
- -//! [`Arc<T>`]: crate::sync::Arc
- -//! [`impl PinInit<Foo>`]: PinInit
- -//! [`impl PinInit<T, E>`]: PinInit
- -//! [`impl Init<T, E>`]: Init
- -//! [`Opaque`]: kernel::types::Opaque
- -//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
- -//! [`pin_data`]: ::macros::pin_data
- -//! [`pin_init!`]: crate::pin_init!
- -
- -use crate::{
- - alloc::{box_ext::BoxExt, AllocError, Flags},
- - error::{self, Error},
- - sync::UniqueArc,
- - types::{Opaque, ScopeGuard},
- -};
- -use alloc::boxed::Box;
- -use core::{
- - cell::UnsafeCell,
- - convert::Infallible,
- - marker::PhantomData,
- - mem::MaybeUninit,
- - num::*,
- - pin::Pin,
- - ptr::{self, NonNull},
- -};
- -
- -#[doc(hidden)]
- -pub mod __internal;
- -#[doc(hidden)]
- -pub mod macros;
- -
- -/// Initialize and pin a type directly on the stack.
- -///
- -/// # Examples
- -///
- -/// ```rust
- -/// # #![allow(clippy::disallowed_names)]
- -/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
- -/// # use core::pin::Pin;
- -/// #[pin_data]
- -/// struct Foo {
- -/// #[pin]
- -/// a: Mutex<usize>,
- -/// b: Bar,
- -/// }
- -///
- -/// #[pin_data]
- -/// struct Bar {
- -/// x: u32,
- -/// }
- -///
- -/// stack_pin_init!(let foo = pin_init!(Foo {
- -/// a <- new_mutex!(42),
- -/// b: Bar {
- -/// x: 64,
- -/// },
- -/// }));
- -/// let foo: Pin<&mut Foo> = foo;
- -/// pr_info!("a: {}", &*foo.a.lock());
- -/// ```
- -///
- -/// # Syntax
- -///
- -/// A normal `let` binding with optional type annotation. The expression is expected to implement
- -/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
- -/// type, then use [`stack_try_pin_init!`].
- -///
- -/// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
- -#[macro_export]
- -macro_rules! stack_pin_init {
- - (let $var:ident $(: $t:ty)? = $val:expr) => {
- - let val = $val;
- - let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- - let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
- - Ok(res) => res,
- - Err(x) => {
- - let x: ::core::convert::Infallible = x;
- - match x {}
- - }
- - };
- - };
- -}
- -
- -/// Initialize and pin a type directly on the stack.
- -///
- -/// # Examples
- -///
- -/// ```rust,ignore
- -/// # #![allow(clippy::disallowed_names)]
- -/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
- -/// # use macros::pin_data;
- -/// # use core::{alloc::AllocError, pin::Pin};
- -/// #[pin_data]
- -/// struct Foo {
- -/// #[pin]
- -/// a: Mutex<usize>,
- -/// b: Box<Bar>,
- -/// }
- -///
- -/// struct Bar {
- -/// x: u32,
- -/// }
- -///
- -/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
- -/// a <- new_mutex!(42),
- -/// b: Box::new(Bar {
- -/// x: 64,
- -/// }, GFP_KERNEL)?,
- -/// }));
- -/// let foo = foo.unwrap();
- -/// pr_info!("a: {}", &*foo.a.lock());
- -/// ```
- -///
- -/// ```rust,ignore
- -/// # #![allow(clippy::disallowed_names)]
- -/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
- -/// # use macros::pin_data;
- -/// # use core::{alloc::AllocError, pin::Pin};
- -/// #[pin_data]
- -/// struct Foo {
- -/// #[pin]
- -/// a: Mutex<usize>,
- -/// b: Box<Bar>,
- -/// }
- -///
- -/// struct Bar {
- -/// x: u32,
- -/// }
- -///
- -/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
- -/// a <- new_mutex!(42),
- -/// b: Box::new(Bar {
- -/// x: 64,
- -/// }, GFP_KERNEL)?,
- -/// }));
- -/// pr_info!("a: {}", &*foo.a.lock());
- -/// # Ok::<_, AllocError>(())
- -/// ```
- -///
- -/// # Syntax
- -///
- -/// A normal `let` binding with optional type annotation. The expression is expected to implement
- -/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
- -/// `=` will propagate this error.
- -#[macro_export]
- -macro_rules! stack_try_pin_init {
- - (let $var:ident $(: $t:ty)? = $val:expr) => {
- - let val = $val;
- - let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- - let mut $var = $crate::init::__internal::StackInit::init($var, val);
- - };
- - (let $var:ident $(: $t:ty)? =? $val:expr) => {
- - let val = $val;
- - let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- - let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
- - };
- -}
- -
- -/// Construct an in-place, pinned initializer for `struct`s.
- -///
- -/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
- -/// [`try_pin_init!`].
- -///
- -/// The syntax is almost identical to that of a normal `struct` initializer:
- -///
- -/// ```rust
- -/// # #![allow(clippy::disallowed_names)]
- -/// # use kernel::{init, pin_init, macros::pin_data, init::*};
- -/// # use core::pin::Pin;
- -/// #[pin_data]
- -/// struct Foo {
- -/// a: usize,
- -/// b: Bar,
- -/// }
- -///
- -/// #[pin_data]
- -/// struct Bar {
- -/// x: u32,
- -/// }
- -///
- -/// # fn demo() -> impl PinInit<Foo> {
- -/// let a = 42;
- -///
- -/// let initializer = pin_init!(Foo {
- -/// a,
- -/// b: Bar {
- -/// x: 64,
- -/// },
- -/// });
- -/// # initializer }
- -/// # Box::pin_init(demo(), GFP_KERNEL).unwrap();
- -/// ```
- -///
- -/// Arbitrary Rust expressions can be used to set the value of a variable.
- -///
- -/// The fields are initialized in the order that they appear in the initializer. So it is possible
- -/// to read already initialized fields using raw pointers.
- -///
- -/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
- -/// initializer.
- -///
- -/// # Init-functions
- -///
- -/// When working with this API it is often desired to let others construct your types without
- -/// giving access to all fields. This is where you would normally write a plain function `new`
- -/// that would return a new instance of your type. With this API that is also possible.
- -/// However, there are a few extra things to keep in mind.
- -///
- -/// To create an initializer function, simply declare it like this:
- -///
- -/// ```rust
- -/// # #![allow(clippy::disallowed_names)]
- -/// # use kernel::{init, pin_init, init::*};
- -/// # use core::pin::Pin;
- -/// # #[pin_data]
- -/// # struct Foo {
- -/// # a: usize,
- -/// # b: Bar,
- -/// # }
- -/// # #[pin_data]
- -/// # struct Bar {
- -/// # x: u32,
- -/// # }
- -/// impl Foo {
- -/// fn new() -> impl PinInit<Self> {
- -/// pin_init!(Self {
- -/// a: 42,
- -/// b: Bar {
- -/// x: 64,
- -/// },
- -/// })
- -/// }
- -/// }
- -/// ```
- -///
- -/// Users of `Foo` can now create it like this:
- -///
- -/// ```rust
- -/// # #![allow(clippy::disallowed_names)]
- -/// # use kernel::{init, pin_init, macros::pin_data, init::*};
- -/// # use core::pin::Pin;
- -/// # #[pin_data]
- -/// # struct Foo {
- -/// # a: usize,
- -/// # b: Bar,
- -/// # }
- -/// # #[pin_data]
- -/// # struct Bar {
- -/// # x: u32,
- -/// # }
- -/// # impl Foo {
- -/// # fn new() -> impl PinInit<Self> {
- -/// # pin_init!(Self {
- -/// # a: 42,
- -/// # b: Bar {
- -/// # x: 64,
- -/// # },
- -/// # })
- -/// # }
- -/// # }
- -/// let foo = Box::pin_init(Foo::new(), GFP_KERNEL);
- -/// ```
- -///
- -/// They can also easily embed it into their own `struct`s:
- -///
- -/// ```rust
- -/// # #![allow(clippy::disallowed_names)]
- -/// # use kernel::{init, pin_init, macros::pin_data, init::*};
- -/// # use core::pin::Pin;
- -/// # #[pin_data]
- -/// # struct Foo {
- -/// # a: usize,
- -/// # b: Bar,
- -/// # }
- -/// # #[pin_data]
- -/// # struct Bar {
- -/// # x: u32,
- -/// # }
- -/// # impl Foo {
- -/// # fn new() -> impl PinInit<Self> {
- -/// # pin_init!(Self {
- -/// # a: 42,
- -/// # b: Bar {
- -/// # x: 64,
- -/// # },
- -/// # })
- -/// # }
- -/// # }
- -/// #[pin_data]
- -/// struct FooContainer {
- -/// #[pin]
- -/// foo1: Foo,
- -/// #[pin]
- -/// foo2: Foo,
- -/// other: u32,
- -/// }
- -///
- -/// impl FooContainer {
- -/// fn new(other: u32) -> impl PinInit<Self> {
- -/// pin_init!(Self {
- -/// foo1 <- Foo::new(),
- -/// foo2 <- Foo::new(),
- -/// other,
- -/// })
- -/// }
- -/// }
- -/// ```
- -///
- -/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
- -/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
- -/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
- -///
- -/// # Syntax
- -///
- -/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
- -/// the following modifications is expected:
- -/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
- -/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
- -/// pointer named `this` inside of the initializer.
- -/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
- -/// struct, this initializes every field with 0 and then runs all initializers specified in the
- -/// body. This can only be done if [`Zeroable`] is implemented for the struct.
- -///
- -/// For instance:
- -///
- -/// ```rust
- -/// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
- -/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
- -/// #[pin_data]
- -/// #[derive(Zeroable)]
- -/// struct Buf {
- -/// // `ptr` points into `buf`.
- -/// ptr: *mut u8,
- -/// buf: [u8; 64],
- -/// #[pin]
- -/// pin: PhantomPinned,
- -/// }
- -/// pin_init!(&this in Buf {
- -/// buf: [0; 64],
- -/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
- -/// pin: PhantomPinned,
- -/// });
- -/// pin_init!(Buf {
- -/// buf: [1; 64],
- -/// ..Zeroable::zeroed()
- -/// });
- -/// ```
- -///
- -/// [`try_pin_init!`]: kernel::try_pin_init
- -/// [`NonNull<Self>`]: core::ptr::NonNull
- -// For a detailed example of how this macro works, see the module documentation of the hidden
- -// module `__internal` inside of `init/__internal.rs`.
- -#[macro_export]
- -macro_rules! pin_init {
- - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- - $($fields:tt)*
- - }) => {
- - $crate::__init_internal!(
- - @this($($this)?),
- - @typ($t $(::<$($generics),*>)?),
- - @fields($($fields)*),
- - @error(::core::convert::Infallible),
- - @data(PinData, use_data),
- - @has_data(HasPinData, __pin_data),
- - @construct_closure(pin_init_from_closure),
- - @munch_fields($($fields)*),
- - )
- - };
- -}
- -
- -/// Construct an in-place, fallible pinned initializer for `struct`s.
- -///
- -/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
- -///
- -/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
- -/// initialization and return the error.
- -///
- -/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
- -/// initialization fails, the memory can be safely deallocated without any further modifications.
- -///
- -/// This macro defaults the error to [`Error`].
- -///
- -/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
- -/// after the `struct` initializer to specify the error type you want to use.
- -///
- -/// # Examples
- -///
- -/// ```rust
- -/// # #![feature(new_uninit)]
- -/// use kernel::{init::{self, PinInit}, error::Error};
- -/// #[pin_data]
- -/// struct BigBuf {
- -/// big: Box<[u8; 1024 * 1024 * 1024]>,
- -/// small: [u8; 1024 * 1024],
- -/// ptr: *mut u8,
- -/// }
- -///
- -/// impl BigBuf {
- -/// fn new() -> impl PinInit<Self, Error> {
- -/// try_pin_init!(Self {
- -/// big: Box::init(init::zeroed(), GFP_KERNEL)?,
- -/// small: [0; 1024 * 1024],
- -/// ptr: core::ptr::null_mut(),
- -/// }? Error)
- -/// }
- -/// }
- -/// ```
- -// For a detailed example of how this macro works, see the module documentation of the hidden
- -// module `__internal` inside of `init/__internal.rs`.
- -#[macro_export]
- -macro_rules! try_pin_init {
- - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- - $($fields:tt)*
- - }) => {
- - $crate::__init_internal!(
- - @this($($this)?),
- - @typ($t $(::<$($generics),*>)? ),
- - @fields($($fields)*),
- - @error($crate::error::Error),
- - @data(PinData, use_data),
- - @has_data(HasPinData, __pin_data),
- - @construct_closure(pin_init_from_closure),
- - @munch_fields($($fields)*),
- - )
- - };
- - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- - $($fields:tt)*
- - }? $err:ty) => {
- - $crate::__init_internal!(
- - @this($($this)?),
- - @typ($t $(::<$($generics),*>)? ),
- - @fields($($fields)*),
- - @error($err),
- - @data(PinData, use_data),
- - @has_data(HasPinData, __pin_data),
- - @construct_closure(pin_init_from_closure),
- - @munch_fields($($fields)*),
- - )
- - };
- -}
- -
- -/// Construct an in-place initializer for `struct`s.
- -///
- -/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
- -/// [`try_init!`].
- -///
- -/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
- -/// - `unsafe` code must guarantee either full initialization or return an error and allow
- -/// deallocation of the memory.
- -/// - the fields are initialized in the order given in the initializer.
- -/// - no references to fields are allowed to be created inside of the initializer.
- -///
- -/// This initializer is for initializing data in-place that might later be moved. If you want to
- -/// pin-initialize, use [`pin_init!`].
- -///
- -/// [`try_init!`]: crate::try_init!
- -// For a detailed example of how this macro works, see the module documentation of the hidden
- -// module `__internal` inside of `init/__internal.rs`.
- -#[macro_export]
- -macro_rules! init {
- - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- - $($fields:tt)*
- - }) => {
- - $crate::__init_internal!(
- - @this($($this)?),
- - @typ($t $(::<$($generics),*>)?),
- - @fields($($fields)*),
- - @error(::core::convert::Infallible),
- - @data(InitData, /*no use_data*/),
- - @has_data(HasInitData, __init_data),
- - @construct_closure(init_from_closure),
- - @munch_fields($($fields)*),
- - )
- - }
- -}
- -
- -/// Construct an in-place fallible initializer for `struct`s.
- -///
- -/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
- -/// [`init!`].
- -///
- -/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
- -/// append `? $type` after the `struct` initializer.
- -/// The safety caveats from [`try_pin_init!`] also apply:
- -/// - `unsafe` code must guarantee either full initialization or return an error and allow
- -/// deallocation of the memory.
- -/// - the fields are initialized in the order given in the initializer.
- -/// - no references to fields are allowed to be created inside of the initializer.
- -///
- -/// # Examples
- -///
- -/// ```rust
- -/// use kernel::{init::{PinInit, zeroed}, error::Error};
- -/// struct BigBuf {
- -/// big: Box<[u8; 1024 * 1024 * 1024]>,
- -/// small: [u8; 1024 * 1024],
- -/// }
- -///
- -/// impl BigBuf {
- -/// fn new() -> impl Init<Self, Error> {
- -/// try_init!(Self {
- -/// big: Box::init(zeroed(), GFP_KERNEL)?,
- -/// small: [0; 1024 * 1024],
- -/// }? Error)
- -/// }
- -/// }
- -/// ```
- -// For a detailed example of how this macro works, see the module documentation of the hidden
- -// module `__internal` inside of `init/__internal.rs`.
- -#[macro_export]
- -macro_rules! try_init {
- - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- - $($fields:tt)*
- - }) => {
- - $crate::__init_internal!(
- - @this($($this)?),
- - @typ($t $(::<$($generics),*>)?),
- - @fields($($fields)*),
- - @error($crate::error::Error),
- - @data(InitData, /*no use_data*/),
- - @has_data(HasInitData, __init_data),
- - @construct_closure(init_from_closure),
- - @munch_fields($($fields)*),
- - )
- - };
- - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- - $($fields:tt)*
- - }? $err:ty) => {
- - $crate::__init_internal!(
- - @this($($this)?),
- - @typ($t $(::<$($generics),*>)?),
- - @fields($($fields)*),
- - @error($err),
- - @data(InitData, /*no use_data*/),
- - @has_data(HasInitData, __init_data),
- - @construct_closure(init_from_closure),
- - @munch_fields($($fields)*),
- - )
- - };
- -}
- -
- -/// A pin-initializer for the type `T`.
- -///
- -/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
- -/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
- -/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
- -///
- -/// Also see the [module description](self).
- -///
- -/// # Safety
- -///
- -/// When implementing this trait you will need to take great care. Also there are probably very few
- -/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
- -///
- -/// The [`PinInit::__pinned_init`] function:
- -/// - returns `Ok(())` if it initialized every field of `slot`,
- -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
- -/// - `slot` can be deallocated without UB occurring,
- -/// - `slot` does not need to be dropped,
- -/// - `slot` is not partially initialized.
- -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
- -///
- -/// [`Arc<T>`]: crate::sync::Arc
- -/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
- -#[must_use = "An initializer must be used in order to create its value."]
- -pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
- - /// Initializes `slot`.
- - ///
- - /// # Safety
- - ///
- - /// - `slot` is a valid pointer to uninitialized memory.
- - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- - /// deallocate.
- - /// - `slot` will not move until it is dropped, i.e. it will be pinned.
- - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
- -
- - /// First initializes the value using `self` then calls the function `f` with the initialized
- - /// value.
- - ///
- - /// If `f` returns an error the value is dropped and the initializer will forward the error.
- - ///
- - /// # Examples
- - ///
- - /// ```rust
- - /// # #![allow(clippy::disallowed_names)]
- - /// use kernel::{types::Opaque, init::pin_init_from_closure};
- - /// #[repr(C)]
- - /// struct RawFoo([u8; 16]);
- - /// extern {
- - /// fn init_foo(_: *mut RawFoo);
- - /// }
- - ///
- - /// #[pin_data]
- - /// struct Foo {
- - /// #[pin]
- - /// raw: Opaque<RawFoo>,
- - /// }
- - ///
- - /// impl Foo {
- - /// fn setup(self: Pin<&mut Self>) {
- - /// pr_info!("Setting up foo");
- - /// }
- - /// }
- - ///
- - /// let foo = pin_init!(Foo {
- - /// raw <- unsafe {
- - /// Opaque::ffi_init(|s| {
- - /// init_foo(s);
- - /// })
- - /// },
- - /// }).pin_chain(|foo| {
- - /// foo.setup();
- - /// Ok(())
- - /// });
- - /// ```
- - fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
- - where
- - F: FnOnce(Pin<&mut T>) -> Result<(), E>,
- - {
- - ChainPinInit(self, f, PhantomData)
- - }
- -}
- -
- -/// An initializer returned by [`PinInit::pin_chain`].
- -pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>);
- -
- -// SAFETY: The `__pinned_init` function is implemented such that it
- -// - returns `Ok(())` on successful initialization,
- -// - returns `Err(err)` on error and in this case `slot` will be dropped.
- -// - considers `slot` pinned.
- -unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
- -where
- - I: PinInit<T, E>,
- - F: FnOnce(Pin<&mut T>) -> Result<(), E>,
- -{
- - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- - // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
- - unsafe { self.0.__pinned_init(slot)? };
- - // SAFETY: The above call initialized `slot` and we still have unique access.
- - let val = unsafe { &mut *slot };
- - // SAFETY: `slot` is considered pinned.
- - let val = unsafe { Pin::new_unchecked(val) };
- - (self.1)(val).map_err(|e| {
- - // SAFETY: `slot` was initialized above.
- - unsafe { core::ptr::drop_in_place(slot) };
- - e
- - })
- - }
- -}
- -
- -/// An initializer for `T`.
- -///
- -/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
- -/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
- -/// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
- -/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
- -///
- -/// Also see the [module description](self).
- -///
- -/// # Safety
- -///
- -/// When implementing this trait you will need to take great care. Also there are probably very few
- -/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
- -///
- -/// The [`Init::__init`] function:
- -/// - returns `Ok(())` if it initialized every field of `slot`,
- -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
- -/// - `slot` can be deallocated without UB occurring,
- -/// - `slot` does not need to be dropped,
- -/// - `slot` is not partially initialized.
- -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
- -///
- -/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
- -/// code as `__init`.
- -///
- -/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
- -/// move the pointee after initialization.
- -///
- -/// [`Arc<T>`]: crate::sync::Arc
- -#[must_use = "An initializer must be used in order to create its value."]
- -pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
- - /// Initializes `slot`.
- - ///
- - /// # Safety
- - ///
- - /// - `slot` is a valid pointer to uninitialized memory.
- - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- - /// deallocate.
- - unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
- -
- - /// First initializes the value using `self` then calls the function `f` with the initialized
- - /// value.
- - ///
- - /// If `f` returns an error the value is dropped and the initializer will forward the error.
- - ///
- - /// # Examples
- - ///
- - /// ```rust
- - /// # #![allow(clippy::disallowed_names)]
- - /// use kernel::{types::Opaque, init::{self, init_from_closure}};
- - /// struct Foo {
- - /// buf: [u8; 1_000_000],
- - /// }
- - ///
- - /// impl Foo {
- - /// fn setup(&mut self) {
- - /// pr_info!("Setting up foo");
- - /// }
- - /// }
- - ///
- - /// let foo = init!(Foo {
- - /// buf <- init::zeroed()
- - /// }).chain(|foo| {
- - /// foo.setup();
- - /// Ok(())
- - /// });
- - /// ```
- - fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
- - where
- - F: FnOnce(&mut T) -> Result<(), E>,
- - {
- - ChainInit(self, f, PhantomData)
- - }
- -}
- -
- -/// An initializer returned by [`Init::chain`].
- -pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>);
- -
- -// SAFETY: The `__init` function is implemented such that it
- -// - returns `Ok(())` on successful initialization,
- -// - returns `Err(err)` on error and in this case `slot` will be dropped.
- -unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
- -where
- - I: Init<T, E>,
- - F: FnOnce(&mut T) -> Result<(), E>,
- -{
- - unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- - // SAFETY: All requirements fulfilled since this function is `__init`.
- - unsafe { self.0.__pinned_init(slot)? };
- - // SAFETY: The above call initialized `slot` and we still have unique access.
- - (self.1)(unsafe { &mut *slot }).map_err(|e| {
- - // SAFETY: `slot` was initialized above.
- - unsafe { core::ptr::drop_in_place(slot) };
- - e
- - })
- - }
- -}
- -
- -// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
- -unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
- -where
- - I: Init<T, E>,
- - F: FnOnce(&mut T) -> Result<(), E>,
- -{
- - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- - // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
- - unsafe { self.__init(slot) }
- - }
- -}
- -
- -/// Creates a new [`PinInit<T, E>`] from the given closure.
- -///
- -/// # Safety
- -///
- -/// The closure:
- -/// - returns `Ok(())` if it initialized every field of `slot`,
- -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
- -/// - `slot` can be deallocated without UB occurring,
- -/// - `slot` does not need to be dropped,
- -/// - `slot` is not partially initialized.
- -/// - may assume that the `slot` does not move if `T: !Unpin`,
- -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
- -#[inline]
- -pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
- - f: impl FnOnce(*mut T) -> Result<(), E>,
- -) -> impl PinInit<T, E> {
- - __internal::InitClosure(f, PhantomData)
- -}
- -
- -/// Creates a new [`Init<T, E>`] from the given closure.
- -///
- -/// # Safety
- -///
- -/// The closure:
- -/// - returns `Ok(())` if it initialized every field of `slot`,
- -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
- -/// - `slot` can be deallocated without UB occurring,
- -/// - `slot` does not need to be dropped,
- -/// - `slot` is not partially initialized.
- -/// - the `slot` may move after initialization.
- -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
- -#[inline]
- -pub const unsafe fn init_from_closure<T: ?Sized, E>(
- - f: impl FnOnce(*mut T) -> Result<(), E>,
- -) -> impl Init<T, E> {
- - __internal::InitClosure(f, PhantomData)
- -}
- -
- -/// An initializer that leaves the memory uninitialized.
- -///
- -/// The initializer is a no-op. The `slot` memory is not changed.
- -#[inline]
- -pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
- - // SAFETY: The memory is allowed to be uninitialized.
- - unsafe { init_from_closure(|_| Ok(())) }
- -}
- -
- -/// Initializes an array by initializing each element via the provided initializer.
- -///
- -/// # Examples
- -///
- -/// ```rust
- -/// use kernel::{error::Error, init::init_array_from_fn};
- -/// let array: Box<[usize; 1_000]> = Box::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap();
- -/// assert_eq!(array.len(), 1_000);
- -/// ```
- -pub fn init_array_from_fn<I, const N: usize, T, E>(
- - mut make_init: impl FnMut(usize) -> I,
- -) -> impl Init<[T; N], E>
- -where
- - I: Init<T, E>,
- -{
- - let init = move |slot: *mut [T; N]| {
- - let slot = slot.cast::<T>();
- - // Counts the number of initialized elements and when dropped drops that many elements from
- - // `slot`.
- - let mut init_count = ScopeGuard::new_with_data(0, |i| {
- - // We now free every element that has been initialized before.
- - // SAFETY: The loop initialized exactly the values from 0..i and since we
- - // return `Err` below, the caller will consider the memory at `slot` as
- - // uninitialized.
- - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- - });
- - for i in 0..N {
- - let init = make_init(i);
- - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- - let ptr = unsafe { slot.add(i) };
- - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- - // requirements.
- - unsafe { init.__init(ptr) }?;
- - *init_count += 1;
- - }
- - init_count.dismiss();
- - Ok(())
- - };
- - // SAFETY: The initializer above initializes every element of the array. On failure it drops
- - // any initialized elements and returns `Err`.
- - unsafe { init_from_closure(init) }
- -}
- -
- -/// Initializes an array by initializing each element via the provided initializer.
- -///
- -/// # Examples
- -///
- -/// ```rust
- -/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
- -/// let array: Arc<[Mutex<usize>; 1_000]> =
- -/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap();
- -/// assert_eq!(array.len(), 1_000);
- -/// ```
- -pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
- - mut make_init: impl FnMut(usize) -> I,
- -) -> impl PinInit<[T; N], E>
- -where
- - I: PinInit<T, E>,
- -{
- - let init = move |slot: *mut [T; N]| {
- - let slot = slot.cast::<T>();
- - // Counts the number of initialized elements and when dropped drops that many elements from
- - // `slot`.
- - let mut init_count = ScopeGuard::new_with_data(0, |i| {
- - // We now free every element that has been initialized before.
- - // SAFETY: The loop initialized exactly the values from 0..i and since we
- - // return `Err` below, the caller will consider the memory at `slot` as
- - // uninitialized.
- - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- - });
- - for i in 0..N {
- - let init = make_init(i);
- - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- - let ptr = unsafe { slot.add(i) };
- - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- - // requirements.
- - unsafe { init.__pinned_init(ptr) }?;
- - *init_count += 1;
- - }
- - init_count.dismiss();
- - Ok(())
- - };
- - // SAFETY: The initializer above initializes every element of the array. On failure it drops
- - // any initialized elements and returns `Err`.
- - unsafe { pin_init_from_closure(init) }
- -}
- -
- -// SAFETY: Every type can be initialized by-value.
- -unsafe impl<T, E> Init<T, E> for T {
- - unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- - unsafe { slot.write(self) };
- - Ok(())
- - }
- -}
- -
- -// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
- -unsafe impl<T, E> PinInit<T, E> for T {
- - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- - unsafe { self.__init(slot) }
- - }
- -}
- -
- -/// Smart pointer that can initialize memory in-place.
- -pub trait InPlaceInit<T>: Sized {
- - /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- - /// type.
- - ///
- - /// If `T: !Unpin` it will not be able to move afterwards.
- - fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E>
- - where
- - E: From<AllocError>;
- -
- - /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- - /// type.
- - ///
- - /// If `T: !Unpin` it will not be able to move afterwards.
- - fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Pin<Self>>
- - where
- - Error: From<E>,
- - {
- - // SAFETY: We delegate to `init` and only change the error type.
- - let init = unsafe {
- - pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- - };
- - Self::try_pin_init(init, flags)
- - }
- -
- - /// Use the given initializer to in-place initialize a `T`.
- - fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- - where
- - E: From<AllocError>;
- -
- - /// Use the given initializer to in-place initialize a `T`.
- - fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
- - where
- - Error: From<E>,
- - {
- - // SAFETY: We delegate to `init` and only change the error type.
- - let init = unsafe {
- - init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- - };
- - Self::try_init(init, flags)
- - }
- -}
- -
- -impl<T> InPlaceInit<T> for Box<T> {
- - #[inline]
- - fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E>
- - where
- - E: From<AllocError>,
- - {
- - let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?;
- - let slot = this.as_mut_ptr();
- - // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- - // slot is valid and will not be moved, because we pin it later.
- - unsafe { init.__pinned_init(slot)? };
- - // SAFETY: All fields have been initialized.
- - Ok(unsafe { this.assume_init() }.into())
- - }
- -
- - #[inline]
- - fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- - where
- - E: From<AllocError>,
- - {
- - let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?;
- - let slot = this.as_mut_ptr();
- - // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- - // slot is valid.
- - unsafe { init.__init(slot)? };
- - // SAFETY: All fields have been initialized.
- - Ok(unsafe { this.assume_init() })
- - }
- -}
- -
- -impl<T> InPlaceInit<T> for UniqueArc<T> {
- - #[inline]
- - fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E>
- - where
- - E: From<AllocError>,
- - {
- - let mut this = UniqueArc::new_uninit(flags)?;
- - let slot = this.as_mut_ptr();
- - // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- - // slot is valid and will not be moved, because we pin it later.
- - unsafe { init.__pinned_init(slot)? };
- - // SAFETY: All fields have been initialized.
- - Ok(unsafe { this.assume_init() }.into())
- - }
- -
- - #[inline]
- - fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- - where
- - E: From<AllocError>,
- - {
- - let mut this = UniqueArc::new_uninit(flags)?;
- - let slot = this.as_mut_ptr();
- - // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- - // slot is valid.
- - unsafe { init.__init(slot)? };
- - // SAFETY: All fields have been initialized.
- - Ok(unsafe { this.assume_init() })
- - }
- -}
- -
- -/// Trait facilitating pinned destruction.
- -///
- -/// Use [`pinned_drop`] to implement this trait safely:
- -///
- -/// ```rust
- -/// # use kernel::sync::Mutex;
- -/// use kernel::macros::pinned_drop;
- -/// use core::pin::Pin;
- -/// #[pin_data(PinnedDrop)]
- -/// struct Foo {
- -/// #[pin]
- -/// mtx: Mutex<usize>,
- -/// }
- -///
- -/// #[pinned_drop]
- -/// impl PinnedDrop for Foo {
- -/// fn drop(self: Pin<&mut Self>) {
- -/// pr_info!("Foo is being dropped!");
- -/// }
- -/// }
- -/// ```
- -///
- -/// # Safety
- -///
- -/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
- -///
- -/// [`pinned_drop`]: kernel::macros::pinned_drop
- -pub unsafe trait PinnedDrop: __internal::HasPinData {
- - /// Executes the pinned destructor of this type.
- - ///
- - /// While this function is marked safe, it is actually unsafe to call it manually. For this
- - /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
- - /// and thus prevents this function from being called where it should not.
- - ///
- - /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
- - /// automatically.
- - fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
- -}
- -
- -/// Marker trait for types that can be initialized by writing just zeroes.
- -///
- -/// # Safety
- -///
- -/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
- -/// this is not UB:
- -///
- -/// ```rust,ignore
- -/// let val: Self = unsafe { core::mem::zeroed() };
- -/// ```
- -pub unsafe trait Zeroable {}
- -
- -/// Create a new zeroed T.
- -///
- -/// The returned initializer will write `0x00` to every byte of the given `slot`.
- -#[inline]
- -pub fn zeroed<T: Zeroable>() -> impl Init<T> {
- - // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
- - // and because we write all zeroes, the memory is initialized.
- - unsafe {
- - init_from_closure(|slot: *mut T| {
- - slot.write_bytes(0, 1);
- - Ok(())
- - })
- - }
- -}
- -
- -macro_rules! impl_zeroable {
- - ($($({$($generics:tt)*})? $t:ty, )*) => {
- - $(unsafe impl$($($generics)*)? Zeroable for $t {})*
- - };
- -}
- -
- -impl_zeroable! {
- - // SAFETY: All primitives that are allowed to be zero.
- - bool,
- - char,
- - u8, u16, u32, u64, u128, usize,
- - i8, i16, i32, i64, i128, isize,
- - f32, f64,
- -
- - // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
- - // creating an instance of an uninhabited type is immediate undefined behavior. For more on
- - // uninhabited/empty types, consult The Rustonomicon:
- - // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
- - // also has information on undefined behavior:
- - // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
- - //
- - // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
- - {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
- -
- - // SAFETY: Type is allowed to take any value, including all zeros.
- - {<T>} MaybeUninit<T>,
- - // SAFETY: Type is allowed to take any value, including all zeros.
- - {<T>} Opaque<T>,
- -
- - // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
- - {<T: ?Sized + Zeroable>} UnsafeCell<T>,
- -
- - // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- - Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
- - Option<NonZeroU128>, Option<NonZeroUsize>,
- - Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
- - Option<NonZeroI128>, Option<NonZeroIsize>,
- -
- - // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- - //
- - // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
- - {<T: ?Sized>} Option<NonNull<T>>,
- - {<T: ?Sized>} Option<Box<T>>,
- -
- - // SAFETY: `null` pointer is valid.
- - //
- - // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
- - // null.
- - //
- - // When `Pointee` gets stabilized, we could use
- - // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
- - {<T>} *mut T, {<T>} *const T,
- -
- - // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
- - // zero.
- - {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
- -
- - // SAFETY: `T` is `Zeroable`.
- - {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
- -}
- -
- -macro_rules! impl_tuple_zeroable {
- - ($(,)?) => {};
- - ($first:ident, $($t:ident),* $(,)?) => {
- - // SAFETY: All elements are zeroable and padding can be zero.
- - unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
- - impl_tuple_zeroable!($($t),* ,);
- - }
- -}
- -
- -impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
- diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs
- deleted file mode 100644
- --- a/rust/kernel/init/__internal.rs
- +++ /dev/null
- @@ -1,230 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -//! This module contains API-internal items for pin-init.
- -//!
- -//! These items must not be used outside of
- -//! - `kernel/init.rs`
- -//! - `macros/pin_data.rs`
- -//! - `macros/pinned_drop.rs`
- -
- -use super::*;
- -
- -/// See the [nomicon] for what subtyping is. See also [this table].
- -///
- -/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
- -/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
- -pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>;
- -
- -/// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this
- -/// type, since the closure needs to fulfill the same safety requirement as the
- -/// `__pinned_init`/`__init` functions.
- -pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>);
- -
- -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
- -// `__init` invariants.
- -unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E>
- -where
- - F: FnOnce(*mut T) -> Result<(), E>,
- -{
- - #[inline]
- - unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- - (self.0)(slot)
- - }
- -}
- -
- -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
- -// `__pinned_init` invariants.
- -unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E>
- -where
- - F: FnOnce(*mut T) -> Result<(), E>,
- -{
- - #[inline]
- - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- - (self.0)(slot)
- - }
- -}
- -
- -/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate
- -/// the pin projections within the initializers.
- -///
- -/// # Safety
- -///
- -/// Only the `init` module is allowed to use this trait.
- -pub unsafe trait HasPinData {
- - type PinData: PinData;
- -
- - unsafe fn __pin_data() -> Self::PinData;
- -}
- -
- -/// Marker trait for pinning data of structs.
- -///
- -/// # Safety
- -///
- -/// Only the `init` module is allowed to use this trait.
- -pub unsafe trait PinData: Copy {
- - type Datee: ?Sized + HasPinData;
- -
- - /// Type inference helper function.
- - fn make_closure<F, O, E>(self, f: F) -> F
- - where
- - F: FnOnce(*mut Self::Datee) -> Result<O, E>,
- - {
- - f
- - }
- -}
- -
- -/// This trait is automatically implemented for every type. It aims to provide the same type
- -/// inference help as `HasPinData`.
- -///
- -/// # Safety
- -///
- -/// Only the `init` module is allowed to use this trait.
- -pub unsafe trait HasInitData {
- - type InitData: InitData;
- -
- - unsafe fn __init_data() -> Self::InitData;
- -}
- -
- -/// Same function as `PinData`, but for arbitrary data.
- -///
- -/// # Safety
- -///
- -/// Only the `init` module is allowed to use this trait.
- -pub unsafe trait InitData: Copy {
- - type Datee: ?Sized + HasInitData;
- -
- - /// Type inference helper function.
- - fn make_closure<F, O, E>(self, f: F) -> F
- - where
- - F: FnOnce(*mut Self::Datee) -> Result<O, E>,
- - {
- - f
- - }
- -}
- -
- -pub struct AllData<T: ?Sized>(PhantomData<fn(Box<T>) -> Box<T>>);
- -
- -impl<T: ?Sized> Clone for AllData<T> {
- - fn clone(&self) -> Self {
- - *self
- - }
- -}
- -
- -impl<T: ?Sized> Copy for AllData<T> {}
- -
- -unsafe impl<T: ?Sized> InitData for AllData<T> {
- - type Datee = T;
- -}
- -
- -unsafe impl<T: ?Sized> HasInitData for T {
- - type InitData = AllData<T>;
- -
- - unsafe fn __init_data() -> Self::InitData {
- - AllData(PhantomData)
- - }
- -}
- -
- -/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive.
- -///
- -/// # Invariants
- -///
- -/// If `self.is_init` is true, then `self.value` is initialized.
- -///
- -/// [`stack_pin_init`]: kernel::stack_pin_init
- -pub struct StackInit<T> {
- - value: MaybeUninit<T>,
- - is_init: bool,
- -}
- -
- -impl<T> Drop for StackInit<T> {
- - #[inline]
- - fn drop(&mut self) {
- - if self.is_init {
- - // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is
- - // true, `self.value` is initialized.
- - unsafe { self.value.assume_init_drop() };
- - }
- - }
- -}
- -
- -impl<T> StackInit<T> {
- - /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this
- - /// primitive.
- - ///
- - /// [`stack_pin_init`]: kernel::stack_pin_init
- - #[inline]
- - pub fn uninit() -> Self {
- - Self {
- - value: MaybeUninit::uninit(),
- - is_init: false,
- - }
- - }
- -
- - /// Initializes the contents and returns the result.
- - #[inline]
- - pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> {
- - // SAFETY: We never move out of `this`.
- - let this = unsafe { Pin::into_inner_unchecked(self) };
- - // The value is currently initialized, so it needs to be dropped before we can reuse
- - // the memory (this is a safety guarantee of `Pin`).
- - if this.is_init {
- - this.is_init = false;
- - // SAFETY: `this.is_init` was true and therefore `this.value` is initialized.
- - unsafe { this.value.assume_init_drop() };
- - }
- - // SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
- - unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
- - // INVARIANT: `this.value` is initialized above.
- - this.is_init = true;
- - // SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
- - Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) })
- - }
- -}
- -
- -/// When a value of this type is dropped, it drops a `T`.
- -///
- -/// Can be forgotten to prevent the drop.
- -pub struct DropGuard<T: ?Sized> {
- - ptr: *mut T,
- -}
- -
- -impl<T: ?Sized> DropGuard<T> {
- - /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped.
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must be a valid pointer.
- - ///
- - /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`:
- - /// - has not been dropped,
- - /// - is not accessible by any other means,
- - /// - will not be dropped by any other means.
- - #[inline]
- - pub unsafe fn new(ptr: *mut T) -> Self {
- - Self { ptr }
- - }
- -}
- -
- -impl<T: ?Sized> Drop for DropGuard<T> {
- - #[inline]
- - fn drop(&mut self) {
- - // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function
- - // ensuring that this operation is safe.
- - unsafe { ptr::drop_in_place(self.ptr) }
- - }
- -}
- -
- -/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely
- -/// created struct. This is needed, because the `drop` function is safe, but should not be called
- -/// manually.
- -pub struct OnlyCallFromDrop(());
- -
- -impl OnlyCallFromDrop {
- - /// # Safety
- - ///
- - /// This function should only be called from the [`Drop::drop`] function and only be used to
- - /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type.
- - pub unsafe fn new() -> Self {
- - Self(())
- - }
- -}
- diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs
- deleted file mode 100644
- --- a/rust/kernel/init/macros.rs
- +++ /dev/null
- @@ -1,1400 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -//! This module provides the macros that actually implement the proc-macros `pin_data` and
- -//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
- -//! macros.
- -//!
- -//! These macros should never be called directly, since they expect their input to be
- -//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
- -//! safe code! Use the public facing macros instead.
- -//!
- -//! This architecture has been chosen because the kernel does not yet have access to `syn` which
- -//! would make matters a lot easier for implementing these as proc-macros.
- -//!
- -//! # Macro expansion example
- -//!
- -//! This section is intended for readers trying to understand the macros in this module and the
- -//! `pin_init!` macros from `init.rs`.
- -//!
- -//! We will look at the following example:
- -//!
- -//! ```rust,ignore
- -//! # use kernel::init::*;
- -//! # use core::pin::Pin;
- -//! #[pin_data]
- -//! #[repr(C)]
- -//! struct Bar<T> {
- -//! #[pin]
- -//! t: T,
- -//! pub x: usize,
- -//! }
- -//!
- -//! impl<T> Bar<T> {
- -//! fn new(t: T) -> impl PinInit<Self> {
- -//! pin_init!(Self { t, x: 0 })
- -//! }
- -//! }
- -//!
- -//! #[pin_data(PinnedDrop)]
- -//! struct Foo {
- -//! a: usize,
- -//! #[pin]
- -//! b: Bar<u32>,
- -//! }
- -//!
- -//! #[pinned_drop]
- -//! impl PinnedDrop for Foo {
- -//! fn drop(self: Pin<&mut Self>) {
- -//! pr_info!("{self:p} is getting dropped.");
- -//! }
- -//! }
- -//!
- -//! let a = 42;
- -//! let initializer = pin_init!(Foo {
- -//! a,
- -//! b <- Bar::new(36),
- -//! });
- -//! ```
- -//!
- -//! This example includes the most common and important features of the pin-init API.
- -//!
- -//! Below you can find individual section about the different macro invocations. Here are some
- -//! general things we need to take into account when designing macros:
- -//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
- -//! this ensures that the correct item is used, since users could define their own `mod core {}`
- -//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
- -//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
- -//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
- -//! `unsafe` operations without an associated `unsafe` block.
- -//!
- -//! ## `#[pin_data]` on `Bar`
- -//!
- -//! This macro is used to specify which fields are structurally pinned and which fields are not. It
- -//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
- -//!
- -//! Here is the definition of `Bar` from our example:
- -//!
- -//! ```rust,ignore
- -//! # use kernel::init::*;
- -//! #[pin_data]
- -//! #[repr(C)]
- -//! struct Bar<T> {
- -//! #[pin]
- -//! t: T,
- -//! pub x: usize,
- -//! }
- -//! ```
- -//!
- -//! This expands to the following code:
- -//!
- -//! ```rust,ignore
- -//! // Firstly the normal definition of the struct, attributes are preserved:
- -//! #[repr(C)]
- -//! struct Bar<T> {
- -//! t: T,
- -//! pub x: usize,
- -//! }
- -//! // Then an anonymous constant is defined, this is because we do not want any code to access the
- -//! // types that we define inside:
- -//! const _: () = {
- -//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
- -//! // since we need to implement access functions for each field and thus need to know its
- -//! // type.
- -//! struct __ThePinData<T> {
- -//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
- -//! }
- -//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
- -//! // `self` by value.
- -//! impl<T> ::core::clone::Clone for __ThePinData<T> {
- -//! fn clone(&self) -> Self {
- -//! *self
- -//! }
- -//! }
- -//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
- -//! // For every field of `Bar`, the pin-data struct will define a function with the same name
- -//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
- -//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
- -//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
- -//! #[allow(dead_code)]
- -//! impl<T> __ThePinData<T> {
- -//! unsafe fn t<E>(
- -//! self,
- -//! slot: *mut T,
- -//! // Since `t` is `#[pin]`, this is `PinInit`.
- -//! init: impl ::kernel::init::PinInit<T, E>,
- -//! ) -> ::core::result::Result<(), E> {
- -//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
- -//! }
- -//! pub unsafe fn x<E>(
- -//! self,
- -//! slot: *mut usize,
- -//! // Since `x` is not `#[pin]`, this is `Init`.
- -//! init: impl ::kernel::init::Init<usize, E>,
- -//! ) -> ::core::result::Result<(), E> {
- -//! unsafe { ::kernel::init::Init::__init(init, slot) }
- -//! }
- -//! }
- -//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
- -//! // that we constructed above.
- -//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
- -//! type PinData = __ThePinData<T>;
- -//! unsafe fn __pin_data() -> Self::PinData {
- -//! __ThePinData {
- -//! __phantom: ::core::marker::PhantomData,
- -//! }
- -//! }
- -//! }
- -//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
- -//! // struct. This is important to ensure that no user can implement a rouge `__pin_data`
- -//! // function without using `unsafe`.
- -//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
- -//! type Datee = Bar<T>;
- -//! }
- -//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
- -//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
- -//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
- -//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
- -//! // for two reasons:
- -//! // - `__phantom`: every generic must be used, since we cannot really know which generics
- -//! // are used, we declere all and then use everything here once.
- -//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
- -//! // over it. The lifetime is needed to work around the limitation that trait bounds must
- -//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
- -//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
- -//! // into accepting these bounds regardless.
- -//! #[allow(dead_code)]
- -//! struct __Unpin<'__pin, T> {
- -//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
- -//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
- -//! // Our only `#[pin]` field is `t`.
- -//! t: T,
- -//! }
- -//! #[doc(hidden)]
- -//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
- -//! where
- -//! __Unpin<'__pin, T>: ::core::marker::Unpin,
- -//! {}
- -//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
- -//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
- -//! // UB with only safe code, so we disallow this by giving a trait implementation error using
- -//! // a direct impl and a blanket implementation.
- -//! trait MustNotImplDrop {}
- -//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
- -//! // (normally people want to know if a type has any kind of drop glue at all, here we want
- -//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
- -//! #[allow(drop_bounds)]
- -//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
- -//! impl<T> MustNotImplDrop for Bar<T> {}
- -//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
- -//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
- -//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
- -//! #[allow(non_camel_case_types)]
- -//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
- -//! impl<
- -//! T: ::kernel::init::PinnedDrop,
- -//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
- -//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
- -//! };
- -//! ```
- -//!
- -//! ## `pin_init!` in `impl Bar`
- -//!
- -//! This macro creates an pin-initializer for the given struct. It requires that the struct is
- -//! annotated by `#[pin_data]`.
- -//!
- -//! Here is the impl on `Bar` defining the new function:
- -//!
- -//! ```rust,ignore
- -//! impl<T> Bar<T> {
- -//! fn new(t: T) -> impl PinInit<Self> {
- -//! pin_init!(Self { t, x: 0 })
- -//! }
- -//! }
- -//! ```
- -//!
- -//! This expands to the following code:
- -//!
- -//! ```rust,ignore
- -//! impl<T> Bar<T> {
- -//! fn new(t: T) -> impl PinInit<Self> {
- -//! {
- -//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
- -//! // return type and shadow it later when we insert the arbitrary user code. That way
- -//! // there will be no possibility of returning without `unsafe`.
- -//! struct __InitOk;
- -//! // Get the data about fields from the supplied type.
- -//! // - the function is unsafe, hence the unsafe block
- -//! // - we `use` the `HasPinData` trait in the block, it is only available in that
- -//! // scope.
- -//! let data = unsafe {
- -//! use ::kernel::init::__internal::HasPinData;
- -//! Self::__pin_data()
- -//! };
- -//! // Ensure that `data` really is of type `PinData` and help with type inference:
- -//! let init = ::kernel::init::__internal::PinData::make_closure::<
- -//! _,
- -//! __InitOk,
- -//! ::core::convert::Infallible,
- -//! >(data, move |slot| {
- -//! {
- -//! // Shadow the structure so it cannot be used to return early. If a user
- -//! // tries to write `return Ok(__InitOk)`, then they get a type error,
- -//! // since that will refer to this struct instead of the one defined
- -//! // above.
- -//! struct __InitOk;
- -//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
- -//! {
- -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
- -//! }
- -//! // Since initialization could fail later (not in this case, since the
- -//! // error type is `Infallible`) we will need to drop this field if there
- -//! // is an error later. This `DropGuard` will drop the field when it gets
- -//! // dropped and has not yet been forgotten.
- -//! let __t_guard = unsafe {
- -//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
- -//! };
- -//! // Expansion of `x: 0,`:
- -//! // Since this can be an arbitrary expression we cannot place it inside
- -//! // of the `unsafe` block, so we bind it here.
- -//! {
- -//! let x = 0;
- -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
- -//! }
- -//! // We again create a `DropGuard`.
- -//! let __x_guard = unsafe {
- -//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
- -//! };
- -//! // Since initialization has successfully completed, we can now forget
- -//! // the guards. This is not `mem::forget`, since we only have
- -//! // `&DropGuard`.
- -//! ::core::mem::forget(__x_guard);
- -//! ::core::mem::forget(__t_guard);
- -//! // Here we use the type checker to ensure that every field has been
- -//! // initialized exactly once, since this is `if false` it will never get
- -//! // executed, but still type-checked.
- -//! // Additionally we abuse `slot` to automatically infer the correct type
- -//! // for the struct. This is also another check that every field is
- -//! // accessible from this scope.
- -//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
- -//! let _ = || {
- -//! unsafe {
- -//! ::core::ptr::write(
- -//! slot,
- -//! Self {
- -//! // We only care about typecheck finding every field
- -//! // here, the expression does not matter, just conjure
- -//! // one using `panic!()`:
- -//! t: ::core::panic!(),
- -//! x: ::core::panic!(),
- -//! },
- -//! );
- -//! };
- -//! };
- -//! }
- -//! // We leave the scope above and gain access to the previously shadowed
- -//! // `__InitOk` that we need to return.
- -//! Ok(__InitOk)
- -//! });
- -//! // Change the return type from `__InitOk` to `()`.
- -//! let init = move |
- -//! slot,
- -//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
- -//! init(slot).map(|__InitOk| ())
- -//! };
- -//! // Construct the initializer.
- -//! let init = unsafe {
- -//! ::kernel::init::pin_init_from_closure::<
- -//! _,
- -//! ::core::convert::Infallible,
- -//! >(init)
- -//! };
- -//! init
- -//! }
- -//! }
- -//! }
- -//! ```
- -//!
- -//! ## `#[pin_data]` on `Foo`
- -//!
- -//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
- -//! differences/new things in the expansion of the `Foo` definition:
- -//!
- -//! ```rust,ignore
- -//! #[pin_data(PinnedDrop)]
- -//! struct Foo {
- -//! a: usize,
- -//! #[pin]
- -//! b: Bar<u32>,
- -//! }
- -//! ```
- -//!
- -//! This expands to the following code:
- -//!
- -//! ```rust,ignore
- -//! struct Foo {
- -//! a: usize,
- -//! b: Bar<u32>,
- -//! }
- -//! const _: () = {
- -//! struct __ThePinData {
- -//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
- -//! }
- -//! impl ::core::clone::Clone for __ThePinData {
- -//! fn clone(&self) -> Self {
- -//! *self
- -//! }
- -//! }
- -//! impl ::core::marker::Copy for __ThePinData {}
- -//! #[allow(dead_code)]
- -//! impl __ThePinData {
- -//! unsafe fn b<E>(
- -//! self,
- -//! slot: *mut Bar<u32>,
- -//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,
- -//! ) -> ::core::result::Result<(), E> {
- -//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
- -//! }
- -//! unsafe fn a<E>(
- -//! self,
- -//! slot: *mut usize,
- -//! init: impl ::kernel::init::Init<usize, E>,
- -//! ) -> ::core::result::Result<(), E> {
- -//! unsafe { ::kernel::init::Init::__init(init, slot) }
- -//! }
- -//! }
- -//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {
- -//! type PinData = __ThePinData;
- -//! unsafe fn __pin_data() -> Self::PinData {
- -//! __ThePinData {
- -//! __phantom: ::core::marker::PhantomData,
- -//! }
- -//! }
- -//! }
- -//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
- -//! type Datee = Foo;
- -//! }
- -//! #[allow(dead_code)]
- -//! struct __Unpin<'__pin> {
- -//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
- -//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
- -//! b: Bar<u32>,
- -//! }
- -//! #[doc(hidden)]
- -//! impl<'__pin> ::core::marker::Unpin for Foo
- -//! where
- -//! __Unpin<'__pin>: ::core::marker::Unpin,
- -//! {}
- -//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
- -//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
- -//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
- -//! impl ::core::ops::Drop for Foo {
- -//! fn drop(&mut self) {
- -//! // Since we are getting dropped, no one else has a reference to `self` and thus we
- -//! // can assume that we never move.
- -//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
- -//! // Create the unsafe token that proves that we are inside of a destructor, this
- -//! // type is only allowed to be created in a destructor.
- -//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
- -//! ::kernel::init::PinnedDrop::drop(pinned, token);
- -//! }
- -//! }
- -//! };
- -//! ```
- -//!
- -//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
- -//!
- -//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
- -//! extra parameter that should not be used at all. The macro hides that parameter.
- -//!
- -//! Here is the `PinnedDrop` impl for `Foo`:
- -//!
- -//! ```rust,ignore
- -//! #[pinned_drop]
- -//! impl PinnedDrop for Foo {
- -//! fn drop(self: Pin<&mut Self>) {
- -//! pr_info!("{self:p} is getting dropped.");
- -//! }
- -//! }
- -//! ```
- -//!
- -//! This expands to the following code:
- -//!
- -//! ```rust,ignore
- -//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
- -//! unsafe impl ::kernel::init::PinnedDrop for Foo {
- -//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
- -//! pr_info!("{self:p} is getting dropped.");
- -//! }
- -//! }
- -//! ```
- -//!
- -//! ## `pin_init!` on `Foo`
- -//!
- -//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
- -//! of `pin_init!` on `Foo`:
- -//!
- -//! ```rust,ignore
- -//! let a = 42;
- -//! let initializer = pin_init!(Foo {
- -//! a,
- -//! b <- Bar::new(36),
- -//! });
- -//! ```
- -//!
- -//! This expands to the following code:
- -//!
- -//! ```rust,ignore
- -//! let a = 42;
- -//! let initializer = {
- -//! struct __InitOk;
- -//! let data = unsafe {
- -//! use ::kernel::init::__internal::HasPinData;
- -//! Foo::__pin_data()
- -//! };
- -//! let init = ::kernel::init::__internal::PinData::make_closure::<
- -//! _,
- -//! __InitOk,
- -//! ::core::convert::Infallible,
- -//! >(data, move |slot| {
- -//! {
- -//! struct __InitOk;
- -//! {
- -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
- -//! }
- -//! let __a_guard = unsafe {
- -//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
- -//! };
- -//! let init = Bar::new(36);
- -//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
- -//! let __b_guard = unsafe {
- -//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
- -//! };
- -//! ::core::mem::forget(__b_guard);
- -//! ::core::mem::forget(__a_guard);
- -//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
- -//! let _ = || {
- -//! unsafe {
- -//! ::core::ptr::write(
- -//! slot,
- -//! Foo {
- -//! a: ::core::panic!(),
- -//! b: ::core::panic!(),
- -//! },
- -//! );
- -//! };
- -//! };
- -//! }
- -//! Ok(__InitOk)
- -//! });
- -//! let init = move |
- -//! slot,
- -//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
- -//! init(slot).map(|__InitOk| ())
- -//! };
- -//! let init = unsafe {
- -//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
- -//! };
- -//! init
- -//! };
- -//! ```
- -
- -/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
- -///
- -/// See [`PinnedDrop`] for more information.
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! __pinned_drop {
- - (
- - @impl_sig($($impl_sig:tt)*),
- - @impl_body(
- - $(#[$($attr:tt)*])*
- - fn drop($($sig:tt)*) {
- - $($inner:tt)*
- - }
- - ),
- - ) => {
- - unsafe $($impl_sig)* {
- - // Inherit all attributes and the type/ident tokens for the signature.
- - $(#[$($attr)*])*
- - fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
- - $($inner)*
- - }
- - }
- - }
- -}
- -
- -/// This macro first parses the struct definition such that it separates pinned and not pinned
- -/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! __pin_data {
- - // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
- - (parse_input:
- - @args($($pinned_drop:ident)?),
- - @sig(
- - $(#[$($struct_attr:tt)*])*
- - $vis:vis struct $name:ident
- - $(where $($whr:tt)*)?
- - ),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @body({ $($fields:tt)* }),
- - ) => {
- - // We now use token munching to iterate through all of the fields. While doing this we
- - // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
- - // wants these to be structurally pinned. The rest of the fields are the
- - // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
- - // order to declare the struct.
- - //
- - // In this call we also put some explaining comments for the parameters.
- - $crate::__pin_data!(find_pinned_fields:
- - // Attributes on the struct itself, these will just be propagated to be put onto the
- - // struct definition.
- - @struct_attrs($(#[$($struct_attr)*])*),
- - // The visibility of the struct.
- - @vis($vis),
- - // The name of the struct.
- - @name($name),
- - // The 'impl generics', the generics that will need to be specified on the struct inside
- - // of an `impl<$ty_generics>` block.
- - @impl_generics($($impl_generics)*),
- - // The 'ty generics', the generics that will need to be specified on the impl blocks.
- - @ty_generics($($ty_generics)*),
- - // The 'decl generics', the generics that need to be specified on the struct
- - // definition.
- - @decl_generics($($decl_generics)*),
- - // The where clause of any impl block and the declaration.
- - @where($($($whr)*)?),
- - // The remaining fields tokens that need to be processed.
- - // We add a `,` at the end to ensure correct parsing.
- - @fields_munch($($fields)* ,),
- - // The pinned fields.
- - @pinned(),
- - // The not pinned fields.
- - @not_pinned(),
- - // All fields.
- - @fields(),
- - // The accumulator containing all attributes already parsed.
- - @accum(),
- - // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
- - @is_pinned(),
- - // The proc-macro argument, this should be `PinnedDrop` or ``.
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - (find_pinned_fields:
- - @struct_attrs($($struct_attrs:tt)*),
- - @vis($vis:vis),
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @where($($whr:tt)*),
- - // We found a PhantomPinned field, this should generally be pinned!
- - @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
- - @pinned($($pinned:tt)*),
- - @not_pinned($($not_pinned:tt)*),
- - @fields($($fields:tt)*),
- - @accum($($accum:tt)*),
- - // This field is not pinned.
- - @is_pinned(),
- - @pinned_drop($($pinned_drop:ident)?),
- - ) => {
- - ::core::compile_error!(concat!(
- - "The field `",
- - stringify!($field),
- - "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
- - ));
- - $crate::__pin_data!(find_pinned_fields:
- - @struct_attrs($($struct_attrs)*),
- - @vis($vis),
- - @name($name),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @decl_generics($($decl_generics)*),
- - @where($($whr)*),
- - @fields_munch($($rest)*),
- - @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
- - @not_pinned($($not_pinned)*),
- - @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
- - @accum(),
- - @is_pinned(),
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - (find_pinned_fields:
- - @struct_attrs($($struct_attrs:tt)*),
- - @vis($vis:vis),
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @where($($whr:tt)*),
- - // We reached the field declaration.
- - @fields_munch($field:ident : $type:ty, $($rest:tt)*),
- - @pinned($($pinned:tt)*),
- - @not_pinned($($not_pinned:tt)*),
- - @fields($($fields:tt)*),
- - @accum($($accum:tt)*),
- - // This field is pinned.
- - @is_pinned(yes),
- - @pinned_drop($($pinned_drop:ident)?),
- - ) => {
- - $crate::__pin_data!(find_pinned_fields:
- - @struct_attrs($($struct_attrs)*),
- - @vis($vis),
- - @name($name),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @decl_generics($($decl_generics)*),
- - @where($($whr)*),
- - @fields_munch($($rest)*),
- - @pinned($($pinned)* $($accum)* $field: $type,),
- - @not_pinned($($not_pinned)*),
- - @fields($($fields)* $($accum)* $field: $type,),
- - @accum(),
- - @is_pinned(),
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - (find_pinned_fields:
- - @struct_attrs($($struct_attrs:tt)*),
- - @vis($vis:vis),
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @where($($whr:tt)*),
- - // We reached the field declaration.
- - @fields_munch($field:ident : $type:ty, $($rest:tt)*),
- - @pinned($($pinned:tt)*),
- - @not_pinned($($not_pinned:tt)*),
- - @fields($($fields:tt)*),
- - @accum($($accum:tt)*),
- - // This field is not pinned.
- - @is_pinned(),
- - @pinned_drop($($pinned_drop:ident)?),
- - ) => {
- - $crate::__pin_data!(find_pinned_fields:
- - @struct_attrs($($struct_attrs)*),
- - @vis($vis),
- - @name($name),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @decl_generics($($decl_generics)*),
- - @where($($whr)*),
- - @fields_munch($($rest)*),
- - @pinned($($pinned)*),
- - @not_pinned($($not_pinned)* $($accum)* $field: $type,),
- - @fields($($fields)* $($accum)* $field: $type,),
- - @accum(),
- - @is_pinned(),
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - (find_pinned_fields:
- - @struct_attrs($($struct_attrs:tt)*),
- - @vis($vis:vis),
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @where($($whr:tt)*),
- - // We found the `#[pin]` attr.
- - @fields_munch(#[pin] $($rest:tt)*),
- - @pinned($($pinned:tt)*),
- - @not_pinned($($not_pinned:tt)*),
- - @fields($($fields:tt)*),
- - @accum($($accum:tt)*),
- - @is_pinned($($is_pinned:ident)?),
- - @pinned_drop($($pinned_drop:ident)?),
- - ) => {
- - $crate::__pin_data!(find_pinned_fields:
- - @struct_attrs($($struct_attrs)*),
- - @vis($vis),
- - @name($name),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @decl_generics($($decl_generics)*),
- - @where($($whr)*),
- - @fields_munch($($rest)*),
- - // We do not include `#[pin]` in the list of attributes, since it is not actually an
- - // attribute that is defined somewhere.
- - @pinned($($pinned)*),
- - @not_pinned($($not_pinned)*),
- - @fields($($fields)*),
- - @accum($($accum)*),
- - // Set this to `yes`.
- - @is_pinned(yes),
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - (find_pinned_fields:
- - @struct_attrs($($struct_attrs:tt)*),
- - @vis($vis:vis),
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @where($($whr:tt)*),
- - // We reached the field declaration with visibility, for simplicity we only munch the
- - // visibility and put it into `$accum`.
- - @fields_munch($fvis:vis $field:ident $($rest:tt)*),
- - @pinned($($pinned:tt)*),
- - @not_pinned($($not_pinned:tt)*),
- - @fields($($fields:tt)*),
- - @accum($($accum:tt)*),
- - @is_pinned($($is_pinned:ident)?),
- - @pinned_drop($($pinned_drop:ident)?),
- - ) => {
- - $crate::__pin_data!(find_pinned_fields:
- - @struct_attrs($($struct_attrs)*),
- - @vis($vis),
- - @name($name),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @decl_generics($($decl_generics)*),
- - @where($($whr)*),
- - @fields_munch($field $($rest)*),
- - @pinned($($pinned)*),
- - @not_pinned($($not_pinned)*),
- - @fields($($fields)*),
- - @accum($($accum)* $fvis),
- - @is_pinned($($is_pinned)?),
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - (find_pinned_fields:
- - @struct_attrs($($struct_attrs:tt)*),
- - @vis($vis:vis),
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @where($($whr:tt)*),
- - // Some other attribute, just put it into `$accum`.
- - @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
- - @pinned($($pinned:tt)*),
- - @not_pinned($($not_pinned:tt)*),
- - @fields($($fields:tt)*),
- - @accum($($accum:tt)*),
- - @is_pinned($($is_pinned:ident)?),
- - @pinned_drop($($pinned_drop:ident)?),
- - ) => {
- - $crate::__pin_data!(find_pinned_fields:
- - @struct_attrs($($struct_attrs)*),
- - @vis($vis),
- - @name($name),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @decl_generics($($decl_generics)*),
- - @where($($whr)*),
- - @fields_munch($($rest)*),
- - @pinned($($pinned)*),
- - @not_pinned($($not_pinned)*),
- - @fields($($fields)*),
- - @accum($($accum)* #[$($attr)*]),
- - @is_pinned($($is_pinned)?),
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - (find_pinned_fields:
- - @struct_attrs($($struct_attrs:tt)*),
- - @vis($vis:vis),
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @decl_generics($($decl_generics:tt)*),
- - @where($($whr:tt)*),
- - // We reached the end of the fields, plus an optional additional comma, since we added one
- - // before and the user is also allowed to put a trailing comma.
- - @fields_munch($(,)?),
- - @pinned($($pinned:tt)*),
- - @not_pinned($($not_pinned:tt)*),
- - @fields($($fields:tt)*),
- - @accum(),
- - @is_pinned(),
- - @pinned_drop($($pinned_drop:ident)?),
- - ) => {
- - // Declare the struct with all fields in the correct order.
- - $($struct_attrs)*
- - $vis struct $name <$($decl_generics)*>
- - where $($whr)*
- - {
- - $($fields)*
- - }
- -
- - // We put the rest into this const item, because it then will not be accessible to anything
- - // outside.
- - const _: () = {
- - // We declare this struct which will host all of the projection function for our type.
- - // it will be invariant over all generic parameters which are inherited from the
- - // struct.
- - $vis struct __ThePinData<$($impl_generics)*>
- - where $($whr)*
- - {
- - __phantom: ::core::marker::PhantomData<
- - fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
- - >,
- - }
- -
- - impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
- - where $($whr)*
- - {
- - fn clone(&self) -> Self { *self }
- - }
- -
- - impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
- - where $($whr)*
- - {}
- -
- - // Make all projection functions.
- - $crate::__pin_data!(make_pin_data:
- - @pin_data(__ThePinData),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @where($($whr)*),
- - @pinned($($pinned)*),
- - @not_pinned($($not_pinned)*),
- - );
- -
- - // SAFETY: We have added the correct projection functions above to `__ThePinData` and
- - // we also use the least restrictive generics possible.
- - unsafe impl<$($impl_generics)*>
- - $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
- - where $($whr)*
- - {
- - type PinData = __ThePinData<$($ty_generics)*>;
- -
- - unsafe fn __pin_data() -> Self::PinData {
- - __ThePinData { __phantom: ::core::marker::PhantomData }
- - }
- - }
- -
- - unsafe impl<$($impl_generics)*>
- - $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
- - where $($whr)*
- - {
- - type Datee = $name<$($ty_generics)*>;
- - }
- -
- - // This struct will be used for the unpin analysis. Since only structurally pinned
- - // fields are relevant whether the struct should implement `Unpin`.
- - #[allow(dead_code)]
- - struct __Unpin <'__pin, $($impl_generics)*>
- - where $($whr)*
- - {
- - __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
- - __phantom: ::core::marker::PhantomData<
- - fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
- - >,
- - // Only the pinned fields.
- - $($pinned)*
- - }
- -
- - #[doc(hidden)]
- - impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
- - where
- - __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
- - $($whr)*
- - {}
- -
- - // We need to disallow normal `Drop` implementation, the exact behavior depends on
- - // whether `PinnedDrop` was specified as the parameter.
- - $crate::__pin_data!(drop_prevention:
- - @name($name),
- - @impl_generics($($impl_generics)*),
- - @ty_generics($($ty_generics)*),
- - @where($($whr)*),
- - @pinned_drop($($pinned_drop)?),
- - );
- - };
- - };
- - // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
- - (drop_prevention:
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @where($($whr:tt)*),
- - @pinned_drop(),
- - ) => {
- - // We prevent this by creating a trait that will be implemented for all types implementing
- - // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
- - // if it also implements `Drop`
- - trait MustNotImplDrop {}
- - #[allow(drop_bounds)]
- - impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
- - impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
- - where $($whr)* {}
- - // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
- - // They might implement `PinnedDrop` correctly for the struct, but forget to give
- - // `PinnedDrop` as the parameter to `#[pin_data]`.
- - #[allow(non_camel_case_types)]
- - trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
- - impl<T: $crate::init::PinnedDrop>
- - UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
- - impl<$($impl_generics)*>
- - UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
- - where $($whr)* {}
- - };
- - // When `PinnedDrop` was specified we just implement `Drop` and delegate.
- - (drop_prevention:
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @where($($whr:tt)*),
- - @pinned_drop(PinnedDrop),
- - ) => {
- - impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
- - where $($whr)*
- - {
- - fn drop(&mut self) {
- - // SAFETY: Since this is a destructor, `self` will not move after this function
- - // terminates, since it is inaccessible.
- - let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
- - // SAFETY: Since this is a drop function, we can create this token to call the
- - // pinned destructor of this type.
- - let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
- - $crate::init::PinnedDrop::drop(pinned, token);
- - }
- - }
- - };
- - // If some other parameter was specified, we emit a readable error.
- - (drop_prevention:
- - @name($name:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @where($($whr:tt)*),
- - @pinned_drop($($rest:tt)*),
- - ) => {
- - compile_error!(
- - "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
- - stringify!($($rest)*),
- - );
- - };
- - (make_pin_data:
- - @pin_data($pin_data:ident),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @where($($whr:tt)*),
- - @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
- - @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
- - ) => {
- - // For every field, we create a projection function according to its projection type. If a
- - // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
- - // structurally pinned, then it can be initialized via `Init`.
- - //
- - // The functions are `unsafe` to prevent accidentally calling them.
- - #[allow(dead_code)]
- - impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
- - where $($whr)*
- - {
- - $(
- - $(#[$($p_attr)*])*
- - $pvis unsafe fn $p_field<E>(
- - self,
- - slot: *mut $p_type,
- - init: impl $crate::init::PinInit<$p_type, E>,
- - ) -> ::core::result::Result<(), E> {
- - unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
- - }
- - )*
- - $(
- - $(#[$($attr)*])*
- - $fvis unsafe fn $field<E>(
- - self,
- - slot: *mut $type,
- - init: impl $crate::init::Init<$type, E>,
- - ) -> ::core::result::Result<(), E> {
- - unsafe { $crate::init::Init::__init(init, slot) }
- - }
- - )*
- - }
- - };
- -}
- -
- -/// The internal init macro. Do not call manually!
- -///
- -/// This is called by the `{try_}{pin_}init!` macros with various inputs.
- -///
- -/// This macro has multiple internal call configurations, these are always the very first ident:
- -/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
- -/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
- -/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
- -/// - `make_initializer`: recursively create the struct initializer that guarantees that every
- -/// field has been initialized exactly once.
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! __init_internal {
- - (
- - @this($($this:ident)?),
- - @typ($t:path),
- - @fields($($fields:tt)*),
- - @error($err:ty),
- - // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- - // case.
- - @data($data:ident, $($use_data:ident)?),
- - // `HasPinData` or `HasInitData`.
- - @has_data($has_data:ident, $get_data:ident),
- - // `pin_init_from_closure` or `init_from_closure`.
- - @construct_closure($construct_closure:ident),
- - @munch_fields(),
- - ) => {
- - $crate::__init_internal!(with_update_parsed:
- - @this($($this)?),
- - @typ($t),
- - @fields($($fields)*),
- - @error($err),
- - @data($data, $($use_data)?),
- - @has_data($has_data, $get_data),
- - @construct_closure($construct_closure),
- - @zeroed(), // Nothing means default behavior.
- - )
- - };
- - (
- - @this($($this:ident)?),
- - @typ($t:path),
- - @fields($($fields:tt)*),
- - @error($err:ty),
- - // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- - // case.
- - @data($data:ident, $($use_data:ident)?),
- - // `HasPinData` or `HasInitData`.
- - @has_data($has_data:ident, $get_data:ident),
- - // `pin_init_from_closure` or `init_from_closure`.
- - @construct_closure($construct_closure:ident),
- - @munch_fields(..Zeroable::zeroed()),
- - ) => {
- - $crate::__init_internal!(with_update_parsed:
- - @this($($this)?),
- - @typ($t),
- - @fields($($fields)*),
- - @error($err),
- - @data($data, $($use_data)?),
- - @has_data($has_data, $get_data),
- - @construct_closure($construct_closure),
- - @zeroed(()), // `()` means zero all fields not mentioned.
- - )
- - };
- - (
- - @this($($this:ident)?),
- - @typ($t:path),
- - @fields($($fields:tt)*),
- - @error($err:ty),
- - // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- - // case.
- - @data($data:ident, $($use_data:ident)?),
- - // `HasPinData` or `HasInitData`.
- - @has_data($has_data:ident, $get_data:ident),
- - // `pin_init_from_closure` or `init_from_closure`.
- - @construct_closure($construct_closure:ident),
- - @munch_fields($ignore:tt $($rest:tt)*),
- - ) => {
- - $crate::__init_internal!(
- - @this($($this)?),
- - @typ($t),
- - @fields($($fields)*),
- - @error($err),
- - @data($data, $($use_data)?),
- - @has_data($has_data, $get_data),
- - @construct_closure($construct_closure),
- - @munch_fields($($rest)*),
- - )
- - };
- - (with_update_parsed:
- - @this($($this:ident)?),
- - @typ($t:path),
- - @fields($($fields:tt)*),
- - @error($err:ty),
- - // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- - // case.
- - @data($data:ident, $($use_data:ident)?),
- - // `HasPinData` or `HasInitData`.
- - @has_data($has_data:ident, $get_data:ident),
- - // `pin_init_from_closure` or `init_from_closure`.
- - @construct_closure($construct_closure:ident),
- - @zeroed($($init_zeroed:expr)?),
- - ) => {{
- - // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
- - // type and shadow it later when we insert the arbitrary user code. That way there will be
- - // no possibility of returning without `unsafe`.
- - struct __InitOk;
- - // Get the data about fields from the supplied type.
- - let data = unsafe {
- - use $crate::init::__internal::$has_data;
- - // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- - // information that is associated to already parsed fragments, so a path fragment
- - // cannot be used in this position. Doing the retokenization results in valid rust
- - // code.
- - ::kernel::macros::paste!($t::$get_data())
- - };
- - // Ensure that `data` really is of type `$data` and help with type inference:
- - let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
- - data,
- - move |slot| {
- - {
- - // Shadow the structure so it cannot be used to return early.
- - struct __InitOk;
- - // If `$init_zeroed` is present we should zero the slot now and not emit an
- - // error when fields are missing (since they will be zeroed). We also have to
- - // check that the type actually implements `Zeroable`.
- - $({
- - fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
- - // Ensure that the struct is indeed `Zeroable`.
- - assert_zeroable(slot);
- - // SAFETY: The type implements `Zeroable` by the check above.
- - unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
- - $init_zeroed // This will be `()` if set.
- - })?
- - // Create the `this` so it can be referenced by the user inside of the
- - // expressions creating the individual fields.
- - $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
- - // Initialize every field.
- - $crate::__init_internal!(init_slot($($use_data)?):
- - @data(data),
- - @slot(slot),
- - @guards(),
- - @munch_fields($($fields)*,),
- - );
- - // We use unreachable code to ensure that all fields have been mentioned exactly
- - // once, this struct initializer will still be type-checked and complain with a
- - // very natural error message if a field is forgotten/mentioned more than once.
- - #[allow(unreachable_code, clippy::diverging_sub_expression)]
- - let _ = || {
- - $crate::__init_internal!(make_initializer:
- - @slot(slot),
- - @type_name($t),
- - @munch_fields($($fields)*,),
- - @acc(),
- - );
- - };
- - }
- - Ok(__InitOk)
- - }
- - );
- - let init = move |slot| -> ::core::result::Result<(), $err> {
- - init(slot).map(|__InitOk| ())
- - };
- - let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
- - init
- - }};
- - (init_slot($($use_data:ident)?):
- - @data($data:ident),
- - @slot($slot:ident),
- - @guards($($guards:ident,)*),
- - @munch_fields($(..Zeroable::zeroed())? $(,)?),
- - ) => {
- - // Endpoint of munching, no fields are left. If execution reaches this point, all fields
- - // have been initialized. Therefore we can now dismiss the guards by forgetting them.
- - $(::core::mem::forget($guards);)*
- - };
- - (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
- - @data($data:ident),
- - @slot($slot:ident),
- - @guards($($guards:ident,)*),
- - // In-place initialization syntax.
- - @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- - ) => {
- - let init = $val;
- - // Call the initializer.
- - //
- - // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
- - // return when an error/panic occurs.
- - // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
- - unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
- - // Create the drop guard:
- - //
- - // We rely on macro hygiene to make it impossible for users to access this local variable.
- - // We use `paste!` to create new hygiene for `$field`.
- - ::kernel::macros::paste! {
- - // SAFETY: We forget the guard later when initialization has succeeded.
- - let [< __ $field _guard >] = unsafe {
- - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- - };
- -
- - $crate::__init_internal!(init_slot($use_data):
- - @data($data),
- - @slot($slot),
- - @guards([< __ $field _guard >], $($guards,)*),
- - @munch_fields($($rest)*),
- - );
- - }
- - };
- - (init_slot(): // No `use_data`, so we use `Init::__init` directly.
- - @data($data:ident),
- - @slot($slot:ident),
- - @guards($($guards:ident,)*),
- - // In-place initialization syntax.
- - @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- - ) => {
- - let init = $val;
- - // Call the initializer.
- - //
- - // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
- - // return when an error/panic occurs.
- - unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
- - // Create the drop guard:
- - //
- - // We rely on macro hygiene to make it impossible for users to access this local variable.
- - // We use `paste!` to create new hygiene for `$field`.
- - ::kernel::macros::paste! {
- - // SAFETY: We forget the guard later when initialization has succeeded.
- - let [< __ $field _guard >] = unsafe {
- - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- - };
- -
- - $crate::__init_internal!(init_slot():
- - @data($data),
- - @slot($slot),
- - @guards([< __ $field _guard >], $($guards,)*),
- - @munch_fields($($rest)*),
- - );
- - }
- - };
- - (init_slot($($use_data:ident)?):
- - @data($data:ident),
- - @slot($slot:ident),
- - @guards($($guards:ident,)*),
- - // Init by-value.
- - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
- - ) => {
- - {
- - $(let $field = $val;)?
- - // Initialize the field.
- - //
- - // SAFETY: The memory at `slot` is uninitialized.
- - unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
- - }
- - // Create the drop guard:
- - //
- - // We rely on macro hygiene to make it impossible for users to access this local variable.
- - // We use `paste!` to create new hygiene for `$field`.
- - ::kernel::macros::paste! {
- - // SAFETY: We forget the guard later when initialization has succeeded.
- - let [< __ $field _guard >] = unsafe {
- - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- - };
- -
- - $crate::__init_internal!(init_slot($($use_data)?):
- - @data($data),
- - @slot($slot),
- - @guards([< __ $field _guard >], $($guards,)*),
- - @munch_fields($($rest)*),
- - );
- - }
- - };
- - (make_initializer:
- - @slot($slot:ident),
- - @type_name($t:path),
- - @munch_fields(..Zeroable::zeroed() $(,)?),
- - @acc($($acc:tt)*),
- - ) => {
- - // Endpoint, nothing more to munch, create the initializer. Since the users specified
- - // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
- - // not been overwritten are thus zero and initialized. We still check that all fields are
- - // actually accessible by using the struct update syntax ourselves.
- - // We are inside of a closure that is never executed and thus we can abuse `slot` to
- - // get the correct type inference here:
- - #[allow(unused_assignments)]
- - unsafe {
- - let mut zeroed = ::core::mem::zeroed();
- - // We have to use type inference here to make zeroed have the correct type. This does
- - // not get executed, so it has no effect.
- - ::core::ptr::write($slot, zeroed);
- - zeroed = ::core::mem::zeroed();
- - // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- - // information that is associated to already parsed fragments, so a path fragment
- - // cannot be used in this position. Doing the retokenization results in valid rust
- - // code.
- - ::kernel::macros::paste!(
- - ::core::ptr::write($slot, $t {
- - $($acc)*
- - ..zeroed
- - });
- - );
- - }
- - };
- - (make_initializer:
- - @slot($slot:ident),
- - @type_name($t:path),
- - @munch_fields($(,)?),
- - @acc($($acc:tt)*),
- - ) => {
- - // Endpoint, nothing more to munch, create the initializer.
- - // Since we are in the closure that is never called, this will never get executed.
- - // We abuse `slot` to get the correct type inference here:
- - unsafe {
- - // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- - // information that is associated to already parsed fragments, so a path fragment
- - // cannot be used in this position. Doing the retokenization results in valid rust
- - // code.
- - ::kernel::macros::paste!(
- - ::core::ptr::write($slot, $t {
- - $($acc)*
- - });
- - );
- - }
- - };
- - (make_initializer:
- - @slot($slot:ident),
- - @type_name($t:path),
- - @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- - @acc($($acc:tt)*),
- - ) => {
- - $crate::__init_internal!(make_initializer:
- - @slot($slot),
- - @type_name($t),
- - @munch_fields($($rest)*),
- - @acc($($acc)* $field: ::core::panic!(),),
- - );
- - };
- - (make_initializer:
- - @slot($slot:ident),
- - @type_name($t:path),
- - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
- - @acc($($acc:tt)*),
- - ) => {
- - $crate::__init_internal!(make_initializer:
- - @slot($slot),
- - @type_name($t),
- - @munch_fields($($rest)*),
- - @acc($($acc)* $field: ::core::panic!(),),
- - );
- - };
- -}
- -
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! __derive_zeroable {
- - (parse_input:
- - @sig(
- - $(#[$($struct_attr:tt)*])*
- - $vis:vis struct $name:ident
- - $(where $($whr:tt)*)?
- - ),
- - @impl_generics($($impl_generics:tt)*),
- - @ty_generics($($ty_generics:tt)*),
- - @body({
- - $(
- - $(#[$($field_attr:tt)*])*
- - $field:ident : $field_ty:ty
- - ),* $(,)?
- - }),
- - ) => {
- - // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
- - #[automatically_derived]
- - unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
- - where
- - $($($whr)*)?
- - {}
- - const _: () = {
- - fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
- - fn ensure_zeroable<$($impl_generics)*>()
- - where $($($whr)*)?
- - {
- - $(assert_zeroable::<$field_ty>();)*
- - }
- - };
- - };
- -}
- diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs
- deleted file mode 100644
- --- a/rust/kernel/ioctl.rs
- +++ /dev/null
- @@ -1,72 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! `ioctl()` number definitions.
- -//!
- -//! C header: [`include/asm-generic/ioctl.h`](srctree/include/asm-generic/ioctl.h)
- -
- -#![allow(non_snake_case)]
- -
- -use crate::build_assert;
- -
- -/// Build an ioctl number, analogous to the C macro of the same name.
- -#[inline(always)]
- -const fn _IOC(dir: u32, ty: u32, nr: u32, size: usize) -> u32 {
- - build_assert!(dir <= uapi::_IOC_DIRMASK);
- - build_assert!(ty <= uapi::_IOC_TYPEMASK);
- - build_assert!(nr <= uapi::_IOC_NRMASK);
- - build_assert!(size <= (uapi::_IOC_SIZEMASK as usize));
- -
- - (dir << uapi::_IOC_DIRSHIFT)
- - | (ty << uapi::_IOC_TYPESHIFT)
- - | (nr << uapi::_IOC_NRSHIFT)
- - | ((size as u32) << uapi::_IOC_SIZESHIFT)
- -}
- -
- -/// Build an ioctl number for an argumentless ioctl.
- -#[inline(always)]
- -pub const fn _IO(ty: u32, nr: u32) -> u32 {
- - _IOC(uapi::_IOC_NONE, ty, nr, 0)
- -}
- -
- -/// Build an ioctl number for a read-only ioctl.
- -#[inline(always)]
- -pub const fn _IOR<T>(ty: u32, nr: u32) -> u32 {
- - _IOC(uapi::_IOC_READ, ty, nr, core::mem::size_of::<T>())
- -}
- -
- -/// Build an ioctl number for a write-only ioctl.
- -#[inline(always)]
- -pub const fn _IOW<T>(ty: u32, nr: u32) -> u32 {
- - _IOC(uapi::_IOC_WRITE, ty, nr, core::mem::size_of::<T>())
- -}
- -
- -/// Build an ioctl number for a read-write ioctl.
- -#[inline(always)]
- -pub const fn _IOWR<T>(ty: u32, nr: u32) -> u32 {
- - _IOC(
- - uapi::_IOC_READ | uapi::_IOC_WRITE,
- - ty,
- - nr,
- - core::mem::size_of::<T>(),
- - )
- -}
- -
- -/// Get the ioctl direction from an ioctl number.
- -pub const fn _IOC_DIR(nr: u32) -> u32 {
- - (nr >> uapi::_IOC_DIRSHIFT) & uapi::_IOC_DIRMASK
- -}
- -
- -/// Get the ioctl type from an ioctl number.
- -pub const fn _IOC_TYPE(nr: u32) -> u32 {
- - (nr >> uapi::_IOC_TYPESHIFT) & uapi::_IOC_TYPEMASK
- -}
- -
- -/// Get the ioctl number from an ioctl number.
- -pub const fn _IOC_NR(nr: u32) -> u32 {
- - (nr >> uapi::_IOC_NRSHIFT) & uapi::_IOC_NRMASK
- -}
- -
- -/// Get the ioctl size from an ioctl number.
- -pub const fn _IOC_SIZE(nr: u32) -> usize {
- - ((nr >> uapi::_IOC_SIZESHIFT) & uapi::_IOC_SIZEMASK) as usize
- -}
- diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs
- deleted file mode 100644
- --- a/rust/kernel/kunit.rs
- +++ /dev/null
- @@ -1,163 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! KUnit-based macros for Rust unit tests.
- -//!
- -//! C header: [`include/kunit/test.h`](srctree/include/kunit/test.h)
- -//!
- -//! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html>
- -
- -use core::{ffi::c_void, fmt};
- -
- -/// Prints a KUnit error-level message.
- -///
- -/// Public but hidden since it should only be used from KUnit generated code.
- -#[doc(hidden)]
- -pub fn err(args: fmt::Arguments<'_>) {
- - // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we
- - // are passing.
- - #[cfg(CONFIG_PRINTK)]
- - unsafe {
- - bindings::_printk(
- - b"\x013%pA\0".as_ptr() as _,
- - &args as *const _ as *const c_void,
- - );
- - }
- -}
- -
- -/// Prints a KUnit info-level message.
- -///
- -/// Public but hidden since it should only be used from KUnit generated code.
- -#[doc(hidden)]
- -pub fn info(args: fmt::Arguments<'_>) {
- - // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we
- - // are passing.
- - #[cfg(CONFIG_PRINTK)]
- - unsafe {
- - bindings::_printk(
- - b"\x016%pA\0".as_ptr() as _,
- - &args as *const _ as *const c_void,
- - );
- - }
- -}
- -
- -/// Asserts that a boolean expression is `true` at runtime.
- -///
- -/// Public but hidden since it should only be used from generated tests.
- -///
- -/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit
- -/// facilities. See [`assert!`] for more details.
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! kunit_assert {
- - ($name:literal, $file:literal, $diff:expr, $condition:expr $(,)?) => {
- - 'out: {
- - // Do nothing if the condition is `true`.
- - if $condition {
- - break 'out;
- - }
- -
- - static FILE: &'static $crate::str::CStr = $crate::c_str!($file);
- - static LINE: i32 = core::line!() as i32 - $diff;
- - static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition));
- -
- - // SAFETY: FFI call without safety requirements.
- - let kunit_test = unsafe { $crate::bindings::kunit_get_current_test() };
- - if kunit_test.is_null() {
- - // The assertion failed but this task is not running a KUnit test, so we cannot call
- - // KUnit, but at least print an error to the kernel log. This may happen if this
- - // macro is called from an spawned thread in a test (see
- - // `scripts/rustdoc_test_gen.rs`) or if some non-test code calls this macro by
- - // mistake (it is hidden to prevent that).
- - //
- - // This mimics KUnit's failed assertion format.
- - $crate::kunit::err(format_args!(
- - " # {}: ASSERTION FAILED at {FILE}:{LINE}\n",
- - $name
- - ));
- - $crate::kunit::err(format_args!(
- - " Expected {CONDITION} to be true, but is false\n"
- - ));
- - $crate::kunit::err(format_args!(
- - " Failure not reported to KUnit since this is a non-KUnit task\n"
- - ));
- - break 'out;
- - }
- -
- - #[repr(transparent)]
- - struct Location($crate::bindings::kunit_loc);
- -
- - #[repr(transparent)]
- - struct UnaryAssert($crate::bindings::kunit_unary_assert);
- -
- - // SAFETY: There is only a static instance and in that one the pointer field points to
- - // an immutable C string.
- - unsafe impl Sync for Location {}
- -
- - // SAFETY: There is only a static instance and in that one the pointer field points to
- - // an immutable C string.
- - unsafe impl Sync for UnaryAssert {}
- -
- - static LOCATION: Location = Location($crate::bindings::kunit_loc {
- - file: FILE.as_char_ptr(),
- - line: LINE,
- - });
- - static ASSERTION: UnaryAssert = UnaryAssert($crate::bindings::kunit_unary_assert {
- - assert: $crate::bindings::kunit_assert {},
- - condition: CONDITION.as_char_ptr(),
- - expected_true: true,
- - });
- -
- - // SAFETY:
- - // - FFI call.
- - // - The `kunit_test` pointer is valid because we got it from
- - // `kunit_get_current_test()` and it was not null. This means we are in a KUnit
- - // test, and that the pointer can be passed to KUnit functions and assertions.
- - // - The string pointers (`file` and `condition` above) point to null-terminated
- - // strings since they are `CStr`s.
- - // - The function pointer (`format`) points to the proper function.
- - // - The pointers passed will remain valid since they point to `static`s.
- - // - The format string is allowed to be null.
- - // - There are, however, problems with this: first of all, this will end up stopping
- - // the thread, without running destructors. While that is problematic in itself,
- - // it is considered UB to have what is effectively a forced foreign unwind
- - // with `extern "C"` ABI. One could observe the stack that is now gone from
- - // another thread. We should avoid pinning stack variables to prevent library UB,
- - // too. For the moment, given that test failures are reported immediately before the
- - // next test runs, that test failures should be fixed and that KUnit is explicitly
- - // documented as not suitable for production environments, we feel it is reasonable.
- - unsafe {
- - $crate::bindings::__kunit_do_failed_assertion(
- - kunit_test,
- - core::ptr::addr_of!(LOCATION.0),
- - $crate::bindings::kunit_assert_type_KUNIT_ASSERTION,
- - core::ptr::addr_of!(ASSERTION.0.assert),
- - Some($crate::bindings::kunit_unary_assert_format),
- - core::ptr::null(),
- - );
- - }
- -
- - // SAFETY: FFI call; the `test` pointer is valid because this hidden macro should only
- - // be called by the generated documentation tests which forward the test pointer given
- - // by KUnit.
- - unsafe {
- - $crate::bindings::__kunit_abort(kunit_test);
- - }
- - }
- - };
- -}
- -
- -/// Asserts that two expressions are equal to each other (using [`PartialEq`]).
- -///
- -/// Public but hidden since it should only be used from generated tests.
- -///
- -/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit
- -/// facilities. See [`assert!`] for more details.
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! kunit_assert_eq {
- - ($name:literal, $file:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{
- - // For the moment, we just forward to the expression assert because, for binary asserts,
- - // KUnit supports only a few types (e.g. integers).
- - $crate::kunit_assert!($name, $file, $diff, $left == $right);
- - }};
- -}
- diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
- deleted file mode 100644
- --- a/rust/kernel/lib.rs
- +++ /dev/null
- @@ -1,138 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! The `kernel` crate.
- -//!
- -//! This crate contains the kernel APIs that have been ported or wrapped for
- -//! usage by Rust code in the kernel and is shared by all of them.
- -//!
- -//! In other words, all the rest of the Rust code in the kernel (e.g. kernel
- -//! modules written in Rust) depends on [`core`], [`alloc`] and this crate.
- -//!
- -//! If you need a kernel C API that is not ported or wrapped yet here, then
- -//! do so first instead of bypassing this crate.
- -
- -#![no_std]
- -#![feature(coerce_unsized)]
- -#![feature(dispatch_from_dyn)]
- -#![feature(new_uninit)]
- -#![feature(receiver_trait)]
- -#![feature(unsize)]
- -
- -// Ensure conditional compilation based on the kernel configuration works;
- -// otherwise we may silently break things like initcall handling.
- -#[cfg(not(CONFIG_RUST))]
- -compile_error!("Missing kernel configuration for conditional compilation");
- -
- -// Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate).
- -extern crate self as kernel;
- -
- -pub mod alloc;
- -mod build_assert;
- -pub mod error;
- -pub mod init;
- -pub mod ioctl;
- -#[cfg(CONFIG_KUNIT)]
- -pub mod kunit;
- -#[cfg(CONFIG_NET)]
- -pub mod net;
- -pub mod prelude;
- -pub mod print;
- -mod static_assert;
- -#[doc(hidden)]
- -pub mod std_vendor;
- -pub mod str;
- -pub mod sync;
- -pub mod task;
- -pub mod time;
- -pub mod types;
- -pub mod workqueue;
- -
- -#[doc(hidden)]
- -pub use bindings;
- -pub use macros;
- -pub use uapi;
- -
- -#[doc(hidden)]
- -pub use build_error::build_error;
- -
- -/// Prefix to appear before log messages printed from within the `kernel` crate.
- -const __LOG_PREFIX: &[u8] = b"rust_kernel\0";
- -
- -/// The top level entrypoint to implementing a kernel module.
- -///
- -/// For any teardown or cleanup operations, your type may implement [`Drop`].
- -pub trait Module: Sized + Sync + Send {
- - /// Called at module initialization time.
- - ///
- - /// Use this method to perform whatever setup or registration your module
- - /// should do.
- - ///
- - /// Equivalent to the `module_init` macro in the C API.
- - fn init(module: &'static ThisModule) -> error::Result<Self>;
- -}
- -
- -/// Equivalent to `THIS_MODULE` in the C API.
- -///
- -/// C header: [`include/linux/export.h`](srctree/include/linux/export.h)
- -pub struct ThisModule(*mut bindings::module);
- -
- -// SAFETY: `THIS_MODULE` may be used from all threads within a module.
- -unsafe impl Sync for ThisModule {}
- -
- -impl ThisModule {
- - /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer.
- - ///
- - /// # Safety
- - ///
- - /// The pointer must be equal to the right `THIS_MODULE`.
- - pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule {
- - ThisModule(ptr)
- - }
- -
- - /// Access the raw pointer for this module.
- - ///
- - /// It is up to the user to use it correctly.
- - pub const fn as_ptr(&self) -> *mut bindings::module {
- - self.0
- - }
- -}
- -
- -#[cfg(not(any(testlib, test)))]
- -#[panic_handler]
- -fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
- - pr_emerg!("{}\n", info);
- - // SAFETY: FFI call.
- - unsafe { bindings::BUG() };
- -}
- -
- -/// Produces a pointer to an object from a pointer to one of its fields.
- -///
- -/// # Safety
- -///
- -/// The pointer passed to this macro, and the pointer returned by this macro, must both be in
- -/// bounds of the same allocation.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// # use kernel::container_of;
- -/// struct Test {
- -/// a: u64,
- -/// b: u32,
- -/// }
- -///
- -/// let test = Test { a: 10, b: 20 };
- -/// let b_ptr = &test.b;
- -/// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be
- -/// // in-bounds of the same allocation as `b_ptr`.
- -/// let test_alias = unsafe { container_of!(b_ptr, Test, b) };
- -/// assert!(core::ptr::eq(&test, test_alias));
- -/// ```
- -#[macro_export]
- -macro_rules! container_of {
- - ($ptr:expr, $type:ty, $($f:tt)*) => {{
- - let ptr = $ptr as *const _ as *const u8;
- - let offset: usize = ::core::mem::offset_of!($type, $($f)*);
- - ptr.sub(offset) as *const $type
- - }}
- -}
- diff --git a/rust/kernel/net.rs b/rust/kernel/net.rs
- deleted file mode 100644
- --- a/rust/kernel/net.rs
- +++ /dev/null
- @@ -1,6 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Networking.
- -
- -#[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)]
- -pub mod phy;
- diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs
- deleted file mode 100644
- --- a/rust/kernel/net/phy.rs
- +++ /dev/null
- @@ -1,905 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -// Copyright (C) 2023 FUJITA Tomonori <fujita.tomonori@gmail.com>
- -
- -//! Network PHY device.
- -//!
- -//! C headers: [`include/linux/phy.h`](srctree/include/linux/phy.h).
- -
- -use crate::{error::*, prelude::*, types::Opaque};
- -
- -use core::marker::PhantomData;
- -
- -/// PHY state machine states.
- -///
- -/// Corresponds to the kernel's [`enum phy_state`].
- -///
- -/// Some of PHY drivers access to the state of PHY's software state machine.
- -///
- -/// [`enum phy_state`]: srctree/include/linux/phy.h
- -#[derive(PartialEq, Eq)]
- -pub enum DeviceState {
- - /// PHY device and driver are not ready for anything.
- - Down,
- - /// PHY is ready to send and receive packets.
- - Ready,
- - /// PHY is up, but no polling or interrupts are done.
- - Halted,
- - /// PHY is up, but is in an error state.
- - Error,
- - /// PHY and attached device are ready to do work.
- - Up,
- - /// PHY is currently running.
- - Running,
- - /// PHY is up, but not currently plugged in.
- - NoLink,
- - /// PHY is performing a cable test.
- - CableTest,
- -}
- -
- -/// A mode of Ethernet communication.
- -///
- -/// PHY drivers get duplex information from hardware and update the current state.
- -pub enum DuplexMode {
- - /// PHY is in full-duplex mode.
- - Full,
- - /// PHY is in half-duplex mode.
- - Half,
- - /// PHY is in unknown duplex mode.
- - Unknown,
- -}
- -
- -/// An instance of a PHY device.
- -///
- -/// Wraps the kernel's [`struct phy_device`].
- -///
- -/// A [`Device`] instance is created when a callback in [`Driver`] is executed. A PHY driver
- -/// executes [`Driver`]'s methods during the callback.
- -///
- -/// # Invariants
- -///
- -/// Referencing a `phy_device` using this struct asserts that you are in
- -/// a context where all methods defined on this struct are safe to call.
- -///
- -/// [`struct phy_device`]: srctree/include/linux/phy.h
- -// During the calls to most functions in [`Driver`], the C side (`PHYLIB`) holds a lock that is
- -// unique for every instance of [`Device`]. `PHYLIB` uses a different serialization technique for
- -// [`Driver::resume`] and [`Driver::suspend`]: `PHYLIB` updates `phy_device`'s state with
- -// the lock held, thus guaranteeing that [`Driver::resume`] has exclusive access to the instance.
- -// [`Driver::resume`] and [`Driver::suspend`] also are called where only one thread can access
- -// to the instance.
- -#[repr(transparent)]
- -pub struct Device(Opaque<bindings::phy_device>);
- -
- -impl Device {
- - /// Creates a new [`Device`] instance from a raw pointer.
- - ///
- - /// # Safety
- - ///
- - /// For the duration of 'a, the pointer must point at a valid `phy_device`,
- - /// and the caller must be in a context where all methods defined on this struct
- - /// are safe to call.
- - unsafe fn from_raw<'a>(ptr: *mut bindings::phy_device) -> &'a mut Self {
- - // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::phy_device`.
- - let ptr = ptr.cast::<Self>();
- - // SAFETY: by the function requirements the pointer is valid and we have unique access for
- - // the duration of `'a`.
- - unsafe { &mut *ptr }
- - }
- -
- - /// Gets the id of the PHY.
- - pub fn phy_id(&self) -> u32 {
- - let phydev = self.0.get();
- - // SAFETY: The struct invariant ensures that we may access
- - // this field without additional synchronization.
- - unsafe { (*phydev).phy_id }
- - }
- -
- - /// Gets the state of PHY state machine states.
- - pub fn state(&self) -> DeviceState {
- - let phydev = self.0.get();
- - // SAFETY: The struct invariant ensures that we may access
- - // this field without additional synchronization.
- - let state = unsafe { (*phydev).state };
- - // TODO: this conversion code will be replaced with automatically generated code by bindgen
- - // when it becomes possible.
- - match state {
- - bindings::phy_state_PHY_DOWN => DeviceState::Down,
- - bindings::phy_state_PHY_READY => DeviceState::Ready,
- - bindings::phy_state_PHY_HALTED => DeviceState::Halted,
- - bindings::phy_state_PHY_ERROR => DeviceState::Error,
- - bindings::phy_state_PHY_UP => DeviceState::Up,
- - bindings::phy_state_PHY_RUNNING => DeviceState::Running,
- - bindings::phy_state_PHY_NOLINK => DeviceState::NoLink,
- - bindings::phy_state_PHY_CABLETEST => DeviceState::CableTest,
- - _ => DeviceState::Error,
- - }
- - }
- -
- - /// Gets the current link state.
- - ///
- - /// It returns true if the link is up.
- - pub fn is_link_up(&self) -> bool {
- - const LINK_IS_UP: u64 = 1;
- - // TODO: the code to access to the bit field will be replaced with automatically
- - // generated code by bindgen when it becomes possible.
- - // SAFETY: The struct invariant ensures that we may access
- - // this field without additional synchronization.
- - let bit_field = unsafe { &(*self.0.get())._bitfield_1 };
- - bit_field.get(14, 1) == LINK_IS_UP
- - }
- -
- - /// Gets the current auto-negotiation configuration.
- - ///
- - /// It returns true if auto-negotiation is enabled.
- - pub fn is_autoneg_enabled(&self) -> bool {
- - // TODO: the code to access to the bit field will be replaced with automatically
- - // generated code by bindgen when it becomes possible.
- - // SAFETY: The struct invariant ensures that we may access
- - // this field without additional synchronization.
- - let bit_field = unsafe { &(*self.0.get())._bitfield_1 };
- - bit_field.get(13, 1) == bindings::AUTONEG_ENABLE as u64
- - }
- -
- - /// Gets the current auto-negotiation state.
- - ///
- - /// It returns true if auto-negotiation is completed.
- - pub fn is_autoneg_completed(&self) -> bool {
- - const AUTONEG_COMPLETED: u64 = 1;
- - // TODO: the code to access to the bit field will be replaced with automatically
- - // generated code by bindgen when it becomes possible.
- - // SAFETY: The struct invariant ensures that we may access
- - // this field without additional synchronization.
- - let bit_field = unsafe { &(*self.0.get())._bitfield_1 };
- - bit_field.get(15, 1) == AUTONEG_COMPLETED
- - }
- -
- - /// Sets the speed of the PHY.
- - pub fn set_speed(&mut self, speed: u32) {
- - let phydev = self.0.get();
- - // SAFETY: The struct invariant ensures that we may access
- - // this field without additional synchronization.
- - unsafe { (*phydev).speed = speed as i32 };
- - }
- -
- - /// Sets duplex mode.
- - pub fn set_duplex(&mut self, mode: DuplexMode) {
- - let phydev = self.0.get();
- - let v = match mode {
- - DuplexMode::Full => bindings::DUPLEX_FULL as i32,
- - DuplexMode::Half => bindings::DUPLEX_HALF as i32,
- - DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN as i32,
- - };
- - // SAFETY: The struct invariant ensures that we may access
- - // this field without additional synchronization.
- - unsafe { (*phydev).duplex = v };
- - }
- -
- - /// Reads a given C22 PHY register.
- - // This function reads a hardware register and updates the stats so takes `&mut self`.
- - pub fn read(&mut self, regnum: u16) -> Result<u16> {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call, open code of `phy_read()` with a valid `phy_device` pointer
- - // `phydev`.
- - let ret = unsafe {
- - bindings::mdiobus_read((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into())
- - };
- - if ret < 0 {
- - Err(Error::from_errno(ret))
- - } else {
- - Ok(ret as u16)
- - }
- - }
- -
- - /// Writes a given C22 PHY register.
- - pub fn write(&mut self, regnum: u16, val: u16) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call, open code of `phy_write()` with a valid `phy_device` pointer
- - // `phydev`.
- - to_result(unsafe {
- - bindings::mdiobus_write((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into(), val)
- - })
- - }
- -
- - /// Reads a paged register.
- - pub fn read_paged(&mut self, page: u16, regnum: u16) -> Result<u16> {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - let ret = unsafe { bindings::phy_read_paged(phydev, page.into(), regnum.into()) };
- - if ret < 0 {
- - Err(Error::from_errno(ret))
- - } else {
- - Ok(ret as u16)
- - }
- - }
- -
- - /// Resolves the advertisements into PHY settings.
- - pub fn resolve_aneg_linkmode(&mut self) {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - unsafe { bindings::phy_resolve_aneg_linkmode(phydev) };
- - }
- -
- - /// Executes software reset the PHY via `BMCR_RESET` bit.
- - pub fn genphy_soft_reset(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::genphy_soft_reset(phydev) })
- - }
- -
- - /// Initializes the PHY.
- - pub fn init_hw(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::phy_init_hw(phydev) })
- - }
- -
- - /// Starts auto-negotiation.
- - pub fn start_aneg(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::_phy_start_aneg(phydev) })
- - }
- -
- - /// Resumes the PHY via `BMCR_PDOWN` bit.
- - pub fn genphy_resume(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::genphy_resume(phydev) })
- - }
- -
- - /// Suspends the PHY via `BMCR_PDOWN` bit.
- - pub fn genphy_suspend(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::genphy_suspend(phydev) })
- - }
- -
- - /// Checks the link status and updates current link state.
- - pub fn genphy_read_status(&mut self) -> Result<u16> {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - let ret = unsafe { bindings::genphy_read_status(phydev) };
- - if ret < 0 {
- - Err(Error::from_errno(ret))
- - } else {
- - Ok(ret as u16)
- - }
- - }
- -
- - /// Updates the link status.
- - pub fn genphy_update_link(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::genphy_update_link(phydev) })
- - }
- -
- - /// Reads link partner ability.
- - pub fn genphy_read_lpa(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::genphy_read_lpa(phydev) })
- - }
- -
- - /// Reads PHY abilities.
- - pub fn genphy_read_abilities(&mut self) -> Result {
- - let phydev = self.0.get();
- - // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`.
- - // So it's just an FFI call.
- - to_result(unsafe { bindings::genphy_read_abilities(phydev) })
- - }
- -}
- -
- -/// Defines certain other features this PHY supports (like interrupts).
- -///
- -/// These flag values are used in [`Driver::FLAGS`].
- -pub mod flags {
- - /// PHY is internal.
- - pub const IS_INTERNAL: u32 = bindings::PHY_IS_INTERNAL;
- - /// PHY needs to be reset after the refclk is enabled.
- - pub const RST_AFTER_CLK_EN: u32 = bindings::PHY_RST_AFTER_CLK_EN;
- - /// Polling is used to detect PHY status changes.
- - pub const POLL_CABLE_TEST: u32 = bindings::PHY_POLL_CABLE_TEST;
- - /// Don't suspend.
- - pub const ALWAYS_CALL_SUSPEND: u32 = bindings::PHY_ALWAYS_CALL_SUSPEND;
- -}
- -
- -/// An adapter for the registration of a PHY driver.
- -struct Adapter<T: Driver> {
- - _p: PhantomData<T>,
- -}
- -
- -impl<T: Driver> Adapter<T> {
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn soft_reset_callback(
- - phydev: *mut bindings::phy_device,
- - ) -> core::ffi::c_int {
- - from_result(|| {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::soft_reset(dev)?;
- - Ok(0)
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn get_features_callback(
- - phydev: *mut bindings::phy_device,
- - ) -> core::ffi::c_int {
- - from_result(|| {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::get_features(dev)?;
- - Ok(0)
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int {
- - from_result(|| {
- - // SAFETY: The C core code ensures that the accessors on
- - // `Device` are okay to call even though `phy_device->lock`
- - // might not be held.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::suspend(dev)?;
- - Ok(0)
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int {
- - from_result(|| {
- - // SAFETY: The C core code ensures that the accessors on
- - // `Device` are okay to call even though `phy_device->lock`
- - // might not be held.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::resume(dev)?;
- - Ok(0)
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn config_aneg_callback(
- - phydev: *mut bindings::phy_device,
- - ) -> core::ffi::c_int {
- - from_result(|| {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::config_aneg(dev)?;
- - Ok(0)
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn read_status_callback(
- - phydev: *mut bindings::phy_device,
- - ) -> core::ffi::c_int {
- - from_result(|| {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::read_status(dev)?;
- - Ok(0)
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn match_phy_device_callback(
- - phydev: *mut bindings::phy_device,
- - ) -> core::ffi::c_int {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::match_phy_device(dev) as i32
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn read_mmd_callback(
- - phydev: *mut bindings::phy_device,
- - devnum: i32,
- - regnum: u16,
- - ) -> i32 {
- - from_result(|| {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - // CAST: the C side verifies devnum < 32.
- - let ret = T::read_mmd(dev, devnum as u8, regnum)?;
- - Ok(ret.into())
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn write_mmd_callback(
- - phydev: *mut bindings::phy_device,
- - devnum: i32,
- - regnum: u16,
- - val: u16,
- - ) -> i32 {
- - from_result(|| {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::write_mmd(dev, devnum as u8, regnum, val)?;
- - Ok(0)
- - })
- - }
- -
- - /// # Safety
- - ///
- - /// `phydev` must be passed by the corresponding callback in `phy_driver`.
- - unsafe extern "C" fn link_change_notify_callback(phydev: *mut bindings::phy_device) {
- - // SAFETY: This callback is called only in contexts
- - // where we hold `phy_device->lock`, so the accessors on
- - // `Device` are okay to call.
- - let dev = unsafe { Device::from_raw(phydev) };
- - T::link_change_notify(dev);
- - }
- -}
- -
- -/// Driver structure for a particular PHY type.
- -///
- -/// Wraps the kernel's [`struct phy_driver`].
- -/// This is used to register a driver for a particular PHY type with the kernel.
- -///
- -/// # Invariants
- -///
- -/// `self.0` is always in a valid state.
- -///
- -/// [`struct phy_driver`]: srctree/include/linux/phy.h
- -#[repr(transparent)]
- -pub struct DriverVTable(Opaque<bindings::phy_driver>);
- -
- -// SAFETY: `DriverVTable` doesn't expose any &self method to access internal data, so it's safe to
- -// share `&DriverVTable` across execution context boundries.
- -unsafe impl Sync for DriverVTable {}
- -
- -/// Creates a [`DriverVTable`] instance from [`Driver`].
- -///
- -/// This is used by [`module_phy_driver`] macro to create a static array of `phy_driver`.
- -///
- -/// [`module_phy_driver`]: crate::module_phy_driver
- -pub const fn create_phy_driver<T: Driver>() -> DriverVTable {
- - // INVARIANT: All the fields of `struct phy_driver` are initialized properly.
- - DriverVTable(Opaque::new(bindings::phy_driver {
- - name: T::NAME.as_char_ptr().cast_mut(),
- - flags: T::FLAGS,
- - phy_id: T::PHY_DEVICE_ID.id,
- - phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(),
- - soft_reset: if T::HAS_SOFT_RESET {
- - Some(Adapter::<T>::soft_reset_callback)
- - } else {
- - None
- - },
- - get_features: if T::HAS_GET_FEATURES {
- - Some(Adapter::<T>::get_features_callback)
- - } else {
- - None
- - },
- - match_phy_device: if T::HAS_MATCH_PHY_DEVICE {
- - Some(Adapter::<T>::match_phy_device_callback)
- - } else {
- - None
- - },
- - suspend: if T::HAS_SUSPEND {
- - Some(Adapter::<T>::suspend_callback)
- - } else {
- - None
- - },
- - resume: if T::HAS_RESUME {
- - Some(Adapter::<T>::resume_callback)
- - } else {
- - None
- - },
- - config_aneg: if T::HAS_CONFIG_ANEG {
- - Some(Adapter::<T>::config_aneg_callback)
- - } else {
- - None
- - },
- - read_status: if T::HAS_READ_STATUS {
- - Some(Adapter::<T>::read_status_callback)
- - } else {
- - None
- - },
- - read_mmd: if T::HAS_READ_MMD {
- - Some(Adapter::<T>::read_mmd_callback)
- - } else {
- - None
- - },
- - write_mmd: if T::HAS_WRITE_MMD {
- - Some(Adapter::<T>::write_mmd_callback)
- - } else {
- - None
- - },
- - link_change_notify: if T::HAS_LINK_CHANGE_NOTIFY {
- - Some(Adapter::<T>::link_change_notify_callback)
- - } else {
- - None
- - },
- - // SAFETY: The rest is zeroed out to initialize `struct phy_driver`,
- - // sets `Option<&F>` to be `None`.
- - ..unsafe { core::mem::MaybeUninit::<bindings::phy_driver>::zeroed().assume_init() }
- - }))
- -}
- -
- -/// Driver implementation for a particular PHY type.
- -///
- -/// This trait is used to create a [`DriverVTable`].
- -#[vtable]
- -pub trait Driver {
- - /// Defines certain other features this PHY supports.
- - /// It is a combination of the flags in the [`flags`] module.
- - const FLAGS: u32 = 0;
- -
- - /// The friendly name of this PHY type.
- - const NAME: &'static CStr;
- -
- - /// This driver only works for PHYs with IDs which match this field.
- - /// The default id and mask are zero.
- - const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_custom_mask(0, 0);
- -
- - /// Issues a PHY software reset.
- - fn soft_reset(_dev: &mut Device) -> Result {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Probes the hardware to determine what abilities it has.
- - fn get_features(_dev: &mut Device) -> Result {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Returns true if this is a suitable driver for the given phydev.
- - /// If not implemented, matching is based on [`Driver::PHY_DEVICE_ID`].
- - fn match_phy_device(_dev: &Device) -> bool {
- - false
- - }
- -
- - /// Configures the advertisement and resets auto-negotiation
- - /// if auto-negotiation is enabled.
- - fn config_aneg(_dev: &mut Device) -> Result {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Determines the negotiated speed and duplex.
- - fn read_status(_dev: &mut Device) -> Result<u16> {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Suspends the hardware, saving state if needed.
- - fn suspend(_dev: &mut Device) -> Result {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Resumes the hardware, restoring state if needed.
- - fn resume(_dev: &mut Device) -> Result {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Overrides the default MMD read function for reading a MMD register.
- - fn read_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16) -> Result<u16> {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Overrides the default MMD write function for writing a MMD register.
- - fn write_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16, _val: u16) -> Result {
- - kernel::build_error(VTABLE_DEFAULT_ERROR)
- - }
- -
- - /// Callback for notification of link change.
- - fn link_change_notify(_dev: &mut Device) {}
- -}
- -
- -/// Registration structure for PHY drivers.
- -///
- -/// Registers [`DriverVTable`] instances with the kernel. They will be unregistered when dropped.
- -///
- -/// # Invariants
- -///
- -/// The `drivers` slice are currently registered to the kernel via `phy_drivers_register`.
- -pub struct Registration {
- - drivers: Pin<&'static mut [DriverVTable]>,
- -}
- -
- -// SAFETY: The only action allowed in a `Registration` instance is dropping it, which is safe to do
- -// from any thread because `phy_drivers_unregister` can be called from any thread context.
- -unsafe impl Send for Registration {}
- -
- -impl Registration {
- - /// Registers a PHY driver.
- - pub fn register(
- - module: &'static crate::ThisModule,
- - drivers: Pin<&'static mut [DriverVTable]>,
- - ) -> Result<Self> {
- - if drivers.is_empty() {
- - return Err(code::EINVAL);
- - }
- - // SAFETY: The type invariants of [`DriverVTable`] ensure that all elements of
- - // the `drivers` slice are initialized properly. `drivers` will not be moved.
- - // So it's just an FFI call.
- - to_result(unsafe {
- - bindings::phy_drivers_register(drivers[0].0.get(), drivers.len().try_into()?, module.0)
- - })?;
- - // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`.
- - Ok(Registration { drivers })
- - }
- -}
- -
- -impl Drop for Registration {
- - fn drop(&mut self) {
- - // SAFETY: The type invariants guarantee that `self.drivers` is valid.
- - // So it's just an FFI call.
- - unsafe {
- - bindings::phy_drivers_unregister(self.drivers[0].0.get(), self.drivers.len() as i32)
- - };
- - }
- -}
- -
- -/// An identifier for PHY devices on an MDIO/MII bus.
- -///
- -/// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate
- -/// PHY driver.
- -pub struct DeviceId {
- - id: u32,
- - mask: DeviceMask,
- -}
- -
- -impl DeviceId {
- - /// Creates a new instance with the exact match mask.
- - pub const fn new_with_exact_mask(id: u32) -> Self {
- - DeviceId {
- - id,
- - mask: DeviceMask::Exact,
- - }
- - }
- -
- - /// Creates a new instance with the model match mask.
- - pub const fn new_with_model_mask(id: u32) -> Self {
- - DeviceId {
- - id,
- - mask: DeviceMask::Model,
- - }
- - }
- -
- - /// Creates a new instance with the vendor match mask.
- - pub const fn new_with_vendor_mask(id: u32) -> Self {
- - DeviceId {
- - id,
- - mask: DeviceMask::Vendor,
- - }
- - }
- -
- - /// Creates a new instance with a custom match mask.
- - pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self {
- - DeviceId {
- - id,
- - mask: DeviceMask::Custom(mask),
- - }
- - }
- -
- - /// Creates a new instance from [`Driver`].
- - pub const fn new_with_driver<T: Driver>() -> Self {
- - T::PHY_DEVICE_ID
- - }
- -
- - /// Get a `mask` as u32.
- - pub const fn mask_as_int(&self) -> u32 {
- - self.mask.as_int()
- - }
- -
- - // macro use only
- - #[doc(hidden)]
- - pub const fn mdio_device_id(&self) -> bindings::mdio_device_id {
- - bindings::mdio_device_id {
- - phy_id: self.id,
- - phy_id_mask: self.mask.as_int(),
- - }
- - }
- -}
- -
- -enum DeviceMask {
- - Exact,
- - Model,
- - Vendor,
- - Custom(u32),
- -}
- -
- -impl DeviceMask {
- - const MASK_EXACT: u32 = !0;
- - const MASK_MODEL: u32 = !0 << 4;
- - const MASK_VENDOR: u32 = !0 << 10;
- -
- - const fn as_int(&self) -> u32 {
- - match self {
- - DeviceMask::Exact => Self::MASK_EXACT,
- - DeviceMask::Model => Self::MASK_MODEL,
- - DeviceMask::Vendor => Self::MASK_VENDOR,
- - DeviceMask::Custom(mask) => *mask,
- - }
- - }
- -}
- -
- -/// Declares a kernel module for PHYs drivers.
- -///
- -/// This creates a static array of kernel's `struct phy_driver` and registers it.
- -/// This also corresponds to the kernel's `MODULE_DEVICE_TABLE` macro, which embeds the information
- -/// for module loading into the module binary file. Every driver needs an entry in `device_table`.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// # mod module_phy_driver_sample {
- -/// use kernel::c_str;
- -/// use kernel::net::phy::{self, DeviceId};
- -/// use kernel::prelude::*;
- -///
- -/// kernel::module_phy_driver! {
- -/// drivers: [PhySample],
- -/// device_table: [
- -/// DeviceId::new_with_driver::<PhySample>()
- -/// ],
- -/// name: "rust_sample_phy",
- -/// author: "Rust for Linux Contributors",
- -/// description: "Rust sample PHYs driver",
- -/// license: "GPL",
- -/// }
- -///
- -/// struct PhySample;
- -///
- -/// #[vtable]
- -/// impl phy::Driver for PhySample {
- -/// const NAME: &'static CStr = c_str!("PhySample");
- -/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001);
- -/// }
- -/// # }
- -/// ```
- -///
- -/// This expands to the following code:
- -///
- -/// ```ignore
- -/// use kernel::c_str;
- -/// use kernel::net::phy::{self, DeviceId};
- -/// use kernel::prelude::*;
- -///
- -/// struct Module {
- -/// _reg: ::kernel::net::phy::Registration,
- -/// }
- -///
- -/// module! {
- -/// type: Module,
- -/// name: "rust_sample_phy",
- -/// author: "Rust for Linux Contributors",
- -/// description: "Rust sample PHYs driver",
- -/// license: "GPL",
- -/// }
- -///
- -/// struct PhySample;
- -///
- -/// #[vtable]
- -/// impl phy::Driver for PhySample {
- -/// const NAME: &'static CStr = c_str!("PhySample");
- -/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001);
- -/// }
- -///
- -/// const _: () = {
- -/// static mut DRIVERS: [::kernel::net::phy::DriverVTable; 1] =
- -/// [::kernel::net::phy::create_phy_driver::<PhySample>()];
- -///
- -/// impl ::kernel::Module for Module {
- -/// fn init(module: &'static ThisModule) -> Result<Self> {
- -/// let drivers = unsafe { &mut DRIVERS };
- -/// let mut reg = ::kernel::net::phy::Registration::register(
- -/// module,
- -/// ::core::pin::Pin::static_mut(drivers),
- -/// )?;
- -/// Ok(Module { _reg: reg })
- -/// }
- -/// }
- -/// };
- -///
- -/// #[cfg(MODULE)]
- -/// #[no_mangle]
- -/// static __mod_mdio__phydev_device_table: [::kernel::bindings::mdio_device_id; 2] = [
- -/// ::kernel::bindings::mdio_device_id {
- -/// phy_id: 0x00000001,
- -/// phy_id_mask: 0xffffffff,
- -/// },
- -/// ::kernel::bindings::mdio_device_id {
- -/// phy_id: 0,
- -/// phy_id_mask: 0,
- -/// },
- -/// ];
- -/// ```
- -#[macro_export]
- -macro_rules! module_phy_driver {
- - (@replace_expr $_t:tt $sub:expr) => {$sub};
- -
- - (@count_devices $($x:expr),*) => {
- - 0usize $(+ $crate::module_phy_driver!(@replace_expr $x 1usize))*
- - };
- -
- - (@device_table [$($dev:expr),+]) => {
- - // SAFETY: C will not read off the end of this constant since the last element is zero.
- - #[cfg(MODULE)]
- - #[no_mangle]
- - static __mod_mdio__phydev_device_table: [$crate::bindings::mdio_device_id;
- - $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = [
- - $($dev.mdio_device_id()),+,
- - $crate::bindings::mdio_device_id {
- - phy_id: 0,
- - phy_id_mask: 0
- - }
- - ];
- - };
- -
- - (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => {
- - struct Module {
- - _reg: $crate::net::phy::Registration,
- - }
- -
- - $crate::prelude::module! {
- - type: Module,
- - $($f)*
- - }
- -
- - const _: () = {
- - static mut DRIVERS: [$crate::net::phy::DriverVTable;
- - $crate::module_phy_driver!(@count_devices $($driver),+)] =
- - [$($crate::net::phy::create_phy_driver::<$driver>()),+];
- -
- - impl $crate::Module for Module {
- - fn init(module: &'static ThisModule) -> Result<Self> {
- - // SAFETY: The anonymous constant guarantees that nobody else can access
- - // the `DRIVERS` static. The array is used only in the C side.
- - let drivers = unsafe { &mut DRIVERS };
- - let mut reg = $crate::net::phy::Registration::register(
- - module,
- - ::core::pin::Pin::static_mut(drivers),
- - )?;
- - Ok(Module { _reg: reg })
- - }
- - }
- - };
- -
- - $crate::module_phy_driver!(@device_table [$($dev),+]);
- - }
- -}
- diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs
- deleted file mode 100644
- --- a/rust/kernel/prelude.rs
- +++ /dev/null
- @@ -1,42 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! The `kernel` prelude.
- -//!
- -//! These are the most common items used by Rust code in the kernel,
- -//! intended to be imported by all Rust code, for convenience.
- -//!
- -//! # Examples
- -//!
- -//! ```
- -//! use kernel::prelude::*;
- -//! ```
- -
- -#[doc(no_inline)]
- -pub use core::pin::Pin;
- -
- -pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt};
- -
- -#[doc(no_inline)]
- -pub use alloc::{boxed::Box, vec::Vec};
- -
- -#[doc(no_inline)]
- -pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable};
- -
- -pub use super::build_assert;
- -
- -// `super::std_vendor` is hidden, which makes the macro inline for some reason.
- -#[doc(no_inline)]
- -pub use super::dbg;
- -pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn};
- -
- -pub use super::{init, pin_init, try_init, try_pin_init};
- -
- -pub use super::static_assert;
- -
- -pub use super::error::{code::*, Error, Result};
- -
- -pub use super::{str::CStr, ThisModule};
- -
- -pub use super::init::{InPlaceInit, Init, PinInit};
- -
- -pub use super::current;
- diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs
- deleted file mode 100644
- --- a/rust/kernel/print.rs
- +++ /dev/null
- @@ -1,413 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Printing facilities.
- -//!
- -//! C header: [`include/linux/printk.h`](srctree/include/linux/printk.h)
- -//!
- -//! Reference: <https://www.kernel.org/doc/html/latest/core-api/printk-basics.html>
- -
- -use core::{
- - ffi::{c_char, c_void},
- - fmt,
- -};
- -
- -use crate::str::RawFormatter;
- -
- -// Called from `vsprintf` with format specifier `%pA`.
- -#[no_mangle]
- -unsafe extern "C" fn rust_fmt_argument(
- - buf: *mut c_char,
- - end: *mut c_char,
- - ptr: *const c_void,
- -) -> *mut c_char {
- - use fmt::Write;
- - // SAFETY: The C contract guarantees that `buf` is valid if it's less than `end`.
- - let mut w = unsafe { RawFormatter::from_ptrs(buf.cast(), end.cast()) };
- - let _ = w.write_fmt(unsafe { *(ptr as *const fmt::Arguments<'_>) });
- - w.pos().cast()
- -}
- -
- -/// Format strings.
- -///
- -/// Public but hidden since it should only be used from public macros.
- -#[doc(hidden)]
- -pub mod format_strings {
- - /// The length we copy from the `KERN_*` kernel prefixes.
- - const LENGTH_PREFIX: usize = 2;
- -
- - /// The length of the fixed format strings.
- - pub const LENGTH: usize = 10;
- -
- - /// Generates a fixed format string for the kernel's [`_printk`].
- - ///
- - /// The format string is always the same for a given level, i.e. for a
- - /// given `prefix`, which are the kernel's `KERN_*` constants.
- - ///
- - /// [`_printk`]: srctree/include/linux/printk.h
- - const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] {
- - // Ensure the `KERN_*` macros are what we expect.
- - assert!(prefix[0] == b'\x01');
- - if is_cont {
- - assert!(prefix[1] == b'c');
- - } else {
- - assert!(prefix[1] >= b'0' && prefix[1] <= b'7');
- - }
- - assert!(prefix[2] == b'\x00');
- -
- - let suffix: &[u8; LENGTH - LENGTH_PREFIX] = if is_cont {
- - b"%pA\0\0\0\0\0"
- - } else {
- - b"%s: %pA\0"
- - };
- -
- - [
- - prefix[0], prefix[1], suffix[0], suffix[1], suffix[2], suffix[3], suffix[4], suffix[5],
- - suffix[6], suffix[7],
- - ]
- - }
- -
- - // Generate the format strings at compile-time.
- - //
- - // This avoids the compiler generating the contents on the fly in the stack.
- - //
- - // Furthermore, `static` instead of `const` is used to share the strings
- - // for all the kernel.
- - pub static EMERG: [u8; LENGTH] = generate(false, bindings::KERN_EMERG);
- - pub static ALERT: [u8; LENGTH] = generate(false, bindings::KERN_ALERT);
- - pub static CRIT: [u8; LENGTH] = generate(false, bindings::KERN_CRIT);
- - pub static ERR: [u8; LENGTH] = generate(false, bindings::KERN_ERR);
- - pub static WARNING: [u8; LENGTH] = generate(false, bindings::KERN_WARNING);
- - pub static NOTICE: [u8; LENGTH] = generate(false, bindings::KERN_NOTICE);
- - pub static INFO: [u8; LENGTH] = generate(false, bindings::KERN_INFO);
- - pub static DEBUG: [u8; LENGTH] = generate(false, bindings::KERN_DEBUG);
- - pub static CONT: [u8; LENGTH] = generate(true, bindings::KERN_CONT);
- -}
- -
- -/// Prints a message via the kernel's [`_printk`].
- -///
- -/// Public but hidden since it should only be used from public macros.
- -///
- -/// # Safety
- -///
- -/// The format string must be one of the ones in [`format_strings`], and
- -/// the module name must be null-terminated.
- -///
- -/// [`_printk`]: srctree/include/linux/_printk.h
- -#[doc(hidden)]
- -#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))]
- -pub unsafe fn call_printk(
- - format_string: &[u8; format_strings::LENGTH],
- - module_name: &[u8],
- - args: fmt::Arguments<'_>,
- -) {
- - // `_printk` does not seem to fail in any path.
- - #[cfg(CONFIG_PRINTK)]
- - unsafe {
- - bindings::_printk(
- - format_string.as_ptr() as _,
- - module_name.as_ptr(),
- - &args as *const _ as *const c_void,
- - );
- - }
- -}
- -
- -/// Prints a message via the kernel's [`_printk`] for the `CONT` level.
- -///
- -/// Public but hidden since it should only be used from public macros.
- -///
- -/// [`_printk`]: srctree/include/linux/printk.h
- -#[doc(hidden)]
- -#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))]
- -pub fn call_printk_cont(args: fmt::Arguments<'_>) {
- - // `_printk` does not seem to fail in any path.
- - //
- - // SAFETY: The format string is fixed.
- - #[cfg(CONFIG_PRINTK)]
- - unsafe {
- - bindings::_printk(
- - format_strings::CONT.as_ptr() as _,
- - &args as *const _ as *const c_void,
- - );
- - }
- -}
- -
- -/// Performs formatting and forwards the string to [`call_printk`].
- -///
- -/// Public but hidden since it should only be used from public macros.
- -#[doc(hidden)]
- -#[cfg(not(testlib))]
- -#[macro_export]
- -#[allow(clippy::crate_in_macro_def)]
- -macro_rules! print_macro (
- - // The non-continuation cases (most of them, e.g. `INFO`).
- - ($format_string:path, false, $($arg:tt)+) => (
- - // To remain sound, `arg`s must be expanded outside the `unsafe` block.
- - // Typically one would use a `let` binding for that; however, `format_args!`
- - // takes borrows on the arguments, but does not extend the scope of temporaries.
- - // Therefore, a `match` expression is used to keep them around, since
- - // the scrutinee is kept until the end of the `match`.
- - match format_args!($($arg)+) {
- - // SAFETY: This hidden macro should only be called by the documented
- - // printing macros which ensure the format string is one of the fixed
- - // ones. All `__LOG_PREFIX`s are null-terminated as they are generated
- - // by the `module!` proc macro or fixed values defined in a kernel
- - // crate.
- - args => unsafe {
- - $crate::print::call_printk(
- - &$format_string,
- - crate::__LOG_PREFIX,
- - args,
- - );
- - }
- - }
- - );
- -
- - // The `CONT` case.
- - ($format_string:path, true, $($arg:tt)+) => (
- - $crate::print::call_printk_cont(
- - format_args!($($arg)+),
- - );
- - );
- -);
- -
- -/// Stub for doctests
- -#[cfg(testlib)]
- -#[macro_export]
- -macro_rules! print_macro (
- - ($format_string:path, $e:expr, $($arg:tt)+) => (
- - ()
- - );
- -);
- -
- -// We could use a macro to generate these macros. However, doing so ends
- -// up being a bit ugly: it requires the dollar token trick to escape `$` as
- -// well as playing with the `doc` attribute. Furthermore, they cannot be easily
- -// imported in the prelude due to [1]. So, for the moment, we just write them
- -// manually, like in the C side; while keeping most of the logic in another
- -// macro, i.e. [`print_macro`].
- -//
- -// [1]: https://github.com/rust-lang/rust/issues/52234
- -
- -/// Prints an emergency-level message (level 0).
- -///
- -/// Use this level if the system is unusable.
- -///
- -/// Equivalent to the kernel's [`pr_emerg`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_emerg`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_emerg
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_emerg!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -macro_rules! pr_emerg (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::EMERG, false, $($arg)*)
- - )
- -);
- -
- -/// Prints an alert-level message (level 1).
- -///
- -/// Use this level if action must be taken immediately.
- -///
- -/// Equivalent to the kernel's [`pr_alert`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_alert`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_alert
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_alert!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -macro_rules! pr_alert (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::ALERT, false, $($arg)*)
- - )
- -);
- -
- -/// Prints a critical-level message (level 2).
- -///
- -/// Use this level for critical conditions.
- -///
- -/// Equivalent to the kernel's [`pr_crit`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_crit`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_crit
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_crit!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -macro_rules! pr_crit (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::CRIT, false, $($arg)*)
- - )
- -);
- -
- -/// Prints an error-level message (level 3).
- -///
- -/// Use this level for error conditions.
- -///
- -/// Equivalent to the kernel's [`pr_err`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_err`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_err
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_err!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -macro_rules! pr_err (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::ERR, false, $($arg)*)
- - )
- -);
- -
- -/// Prints a warning-level message (level 4).
- -///
- -/// Use this level for warning conditions.
- -///
- -/// Equivalent to the kernel's [`pr_warn`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_warn`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_warn
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_warn!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -macro_rules! pr_warn (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::WARNING, false, $($arg)*)
- - )
- -);
- -
- -/// Prints a notice-level message (level 5).
- -///
- -/// Use this level for normal but significant conditions.
- -///
- -/// Equivalent to the kernel's [`pr_notice`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_notice`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_notice
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_notice!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -macro_rules! pr_notice (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::NOTICE, false, $($arg)*)
- - )
- -);
- -
- -/// Prints an info-level message (level 6).
- -///
- -/// Use this level for informational messages.
- -///
- -/// Equivalent to the kernel's [`pr_info`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_info`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_info
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_info!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -#[doc(alias = "print")]
- -macro_rules! pr_info (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::INFO, false, $($arg)*)
- - )
- -);
- -
- -/// Prints a debug-level message (level 7).
- -///
- -/// Use this level for debug messages.
- -///
- -/// Equivalent to the kernel's [`pr_debug`] macro, except that it doesn't support dynamic debug
- -/// yet.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_debug`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_debug
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// pr_debug!("hello {}\n", "there");
- -/// ```
- -#[macro_export]
- -#[doc(alias = "print")]
- -macro_rules! pr_debug (
- - ($($arg:tt)*) => (
- - if cfg!(debug_assertions) {
- - $crate::print_macro!($crate::print::format_strings::DEBUG, false, $($arg)*)
- - }
- - )
- -);
- -
- -/// Continues a previous log message in the same line.
- -///
- -/// Use only when continuing a previous `pr_*!` macro (e.g. [`pr_info!`]).
- -///
- -/// Equivalent to the kernel's [`pr_cont`] macro.
- -///
- -/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
- -/// `alloc::format!` for information about the formatting syntax.
- -///
- -/// [`pr_info!`]: crate::pr_info!
- -/// [`pr_cont`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_cont
- -/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
- -///
- -/// # Examples
- -///
- -/// ```
- -/// # use kernel::pr_cont;
- -/// pr_info!("hello");
- -/// pr_cont!(" {}\n", "there");
- -/// ```
- -#[macro_export]
- -macro_rules! pr_cont (
- - ($($arg:tt)*) => (
- - $crate::print_macro!($crate::print::format_strings::CONT, true, $($arg)*)
- - )
- -);
- diff --git a/rust/kernel/static_assert.rs b/rust/kernel/static_assert.rs
- deleted file mode 100644
- --- a/rust/kernel/static_assert.rs
- +++ /dev/null
- @@ -1,34 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Static assert.
- -
- -/// Static assert (i.e. compile-time assert).
- -///
- -/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`].
- -///
- -/// The feature may be added to Rust in the future: see [RFC 2790].
- -///
- -/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert
- -/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert
- -/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790
- -///
- -/// # Examples
- -///
- -/// ```
- -/// static_assert!(42 > 24);
- -/// static_assert!(core::mem::size_of::<u8>() == 1);
- -///
- -/// const X: &[u8] = b"bar";
- -/// static_assert!(X[1] == b'a');
- -///
- -/// const fn f(x: i32) -> i32 {
- -/// x + 2
- -/// }
- -/// static_assert!(f(40) == 42);
- -/// ```
- -#[macro_export]
- -macro_rules! static_assert {
- - ($condition:expr) => {
- - const _: () = core::assert!($condition);
- - };
- -}
- diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs
- deleted file mode 100644
- --- a/rust/kernel/std_vendor.rs
- +++ /dev/null
- @@ -1,166 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -//! The contents of this file come from the Rust standard library, hosted in
- -//! the <https://github.com/rust-lang/rust> repository, licensed under
- -//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details,
- -//! see <https://github.com/rust-lang/rust/blob/master/COPYRIGHT>.
- -
- -/// [`std::dbg`], but using [`pr_info`] instead of [`eprintln`].
- -///
- -/// Prints and returns the value of a given expression for quick and dirty
- -/// debugging.
- -///
- -/// An example:
- -///
- -/// ```rust
- -/// let a = 2;
- -/// # #[allow(clippy::dbg_macro)]
- -/// let b = dbg!(a * 2) + 1;
- -/// // ^-- prints: [src/main.rs:2] a * 2 = 4
- -/// assert_eq!(b, 5);
- -/// ```
- -///
- -/// The macro works by using the `Debug` implementation of the type of
- -/// the given expression to print the value with [`printk`] along with the
- -/// source location of the macro invocation as well as the source code
- -/// of the expression.
- -///
- -/// Invoking the macro on an expression moves and takes ownership of it
- -/// before returning the evaluated expression unchanged. If the type
- -/// of the expression does not implement `Copy` and you don't want
- -/// to give up ownership, you can instead borrow with `dbg!(&expr)`
- -/// for some expression `expr`.
- -///
- -/// The `dbg!` macro works exactly the same in release builds.
- -/// This is useful when debugging issues that only occur in release
- -/// builds or when debugging in release mode is significantly faster.
- -///
- -/// Note that the macro is intended as a temporary debugging tool to be
- -/// used during development. Therefore, avoid committing `dbg!` macro
- -/// invocations into the kernel tree.
- -///
- -/// For debug output that is intended to be kept in the kernel tree,
- -/// use [`pr_debug`] and similar facilities instead.
- -///
- -/// # Stability
- -///
- -/// The exact output printed by this macro should not be relied upon
- -/// and is subject to future changes.
- -///
- -/// # Further examples
- -///
- -/// With a method call:
- -///
- -/// ```rust
- -/// # #[allow(clippy::dbg_macro)]
- -/// fn foo(n: usize) {
- -/// if dbg!(n.checked_sub(4)).is_some() {
- -/// // ...
- -/// }
- -/// }
- -///
- -/// foo(3)
- -/// ```
- -///
- -/// This prints to the kernel log:
- -///
- -/// ```text,ignore
- -/// [src/main.rs:4] n.checked_sub(4) = None
- -/// ```
- -///
- -/// Naive factorial implementation:
- -///
- -/// ```rust
- -/// # #[allow(clippy::dbg_macro)]
- -/// # {
- -/// fn factorial(n: u32) -> u32 {
- -/// if dbg!(n <= 1) {
- -/// dbg!(1)
- -/// } else {
- -/// dbg!(n * factorial(n - 1))
- -/// }
- -/// }
- -///
- -/// dbg!(factorial(4));
- -/// # }
- -/// ```
- -///
- -/// This prints to the kernel log:
- -///
- -/// ```text,ignore
- -/// [src/main.rs:3] n <= 1 = false
- -/// [src/main.rs:3] n <= 1 = false
- -/// [src/main.rs:3] n <= 1 = false
- -/// [src/main.rs:3] n <= 1 = true
- -/// [src/main.rs:4] 1 = 1
- -/// [src/main.rs:5] n * factorial(n - 1) = 2
- -/// [src/main.rs:5] n * factorial(n - 1) = 6
- -/// [src/main.rs:5] n * factorial(n - 1) = 24
- -/// [src/main.rs:11] factorial(4) = 24
- -/// ```
- -///
- -/// The `dbg!(..)` macro moves the input:
- -///
- -/// ```ignore
- -/// /// A wrapper around `usize` which importantly is not Copyable.
- -/// #[derive(Debug)]
- -/// struct NoCopy(usize);
- -///
- -/// let a = NoCopy(42);
- -/// let _ = dbg!(a); // <-- `a` is moved here.
- -/// let _ = dbg!(a); // <-- `a` is moved again; error!
- -/// ```
- -///
- -/// You can also use `dbg!()` without a value to just print the
- -/// file and line whenever it's reached.
- -///
- -/// Finally, if you want to `dbg!(..)` multiple values, it will treat them as
- -/// a tuple (and return it, too):
- -///
- -/// ```
- -/// # #[allow(clippy::dbg_macro)]
- -/// assert_eq!(dbg!(1usize, 2u32), (1, 2));
- -/// ```
- -///
- -/// However, a single argument with a trailing comma will still not be treated
- -/// as a tuple, following the convention of ignoring trailing commas in macro
- -/// invocations. You can use a 1-tuple directly if you need one:
- -///
- -/// ```
- -/// # #[allow(clippy::dbg_macro)]
- -/// # {
- -/// assert_eq!(1, dbg!(1u32,)); // trailing comma ignored
- -/// assert_eq!((1,), dbg!((1u32,))); // 1-tuple
- -/// # }
- -/// ```
- -///
- -/// [`std::dbg`]: https://doc.rust-lang.org/std/macro.dbg.html
- -/// [`eprintln`]: https://doc.rust-lang.org/std/macro.eprintln.html
- -/// [`printk`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html
- -/// [`pr_info`]: crate::pr_info!
- -/// [`pr_debug`]: crate::pr_debug!
- -#[macro_export]
- -macro_rules! dbg {
- - // NOTE: We cannot use `concat!` to make a static string as a format argument
- - // of `pr_info!` because `file!` could contain a `{` or
- - // `$val` expression could be a block (`{ .. }`), in which case the `pr_info!`
- - // will be malformed.
- - () => {
- - $crate::pr_info!("[{}:{}:{}]\n", ::core::file!(), ::core::line!(), ::core::column!())
- - };
- - ($val:expr $(,)?) => {
- - // Use of `match` here is intentional because it affects the lifetimes
- - // of temporaries - https://stackoverflow.com/a/48732525/1063961
- - match $val {
- - tmp => {
- - $crate::pr_info!("[{}:{}:{}] {} = {:#?}\n",
- - ::core::file!(), ::core::line!(), ::core::column!(),
- - ::core::stringify!($val), &tmp);
- - tmp
- - }
- - }
- - };
- - ($($val:expr),+ $(,)?) => {
- - ($($crate::dbg!($val)),+,)
- - };
- -}
- diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs
- deleted file mode 100644
- --- a/rust/kernel/str.rs
- +++ /dev/null
- @@ -1,874 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! String representations.
- -
- -use crate::alloc::{flags::*, vec_ext::VecExt, AllocError};
- -use alloc::vec::Vec;
- -use core::fmt::{self, Write};
- -use core::ops::{self, Deref, DerefMut, Index};
- -
- -use crate::error::{code::*, Error};
- -
- -/// Byte string without UTF-8 validity guarantee.
- -#[repr(transparent)]
- -pub struct BStr([u8]);
- -
- -impl BStr {
- - /// Returns the length of this string.
- - #[inline]
- - pub const fn len(&self) -> usize {
- - self.0.len()
- - }
- -
- - /// Returns `true` if the string is empty.
- - #[inline]
- - pub const fn is_empty(&self) -> bool {
- - self.len() == 0
- - }
- -
- - /// Creates a [`BStr`] from a `[u8]`.
- - #[inline]
- - pub const fn from_bytes(bytes: &[u8]) -> &Self {
- - // SAFETY: `BStr` is transparent to `[u8]`.
- - unsafe { &*(bytes as *const [u8] as *const BStr) }
- - }
- -}
- -
- -impl fmt::Display for BStr {
- - /// Formats printable ASCII characters, escaping the rest.
- - ///
- - /// ```
- - /// # use kernel::{fmt, b_str, str::{BStr, CString}};
- - /// let ascii = b_str!("Hello, BStr!");
- - /// let s = CString::try_from_fmt(fmt!("{}", ascii)).unwrap();
- - /// assert_eq!(s.as_bytes(), "Hello, BStr!".as_bytes());
- - ///
- - /// let non_ascii = b_str!("🦀");
- - /// let s = CString::try_from_fmt(fmt!("{}", non_ascii)).unwrap();
- - /// assert_eq!(s.as_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes());
- - /// ```
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - for &b in &self.0 {
- - match b {
- - // Common escape codes.
- - b'\t' => f.write_str("\\t")?,
- - b'\n' => f.write_str("\\n")?,
- - b'\r' => f.write_str("\\r")?,
- - // Printable characters.
- - 0x20..=0x7e => f.write_char(b as char)?,
- - _ => write!(f, "\\x{:02x}", b)?,
- - }
- - }
- - Ok(())
- - }
- -}
- -
- -impl fmt::Debug for BStr {
- - /// Formats printable ASCII characters with a double quote on either end,
- - /// escaping the rest.
- - ///
- - /// ```
- - /// # use kernel::{fmt, b_str, str::{BStr, CString}};
- - /// // Embedded double quotes are escaped.
- - /// let ascii = b_str!("Hello, \"BStr\"!");
- - /// let s = CString::try_from_fmt(fmt!("{:?}", ascii)).unwrap();
- - /// assert_eq!(s.as_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes());
- - ///
- - /// let non_ascii = b_str!("😺");
- - /// let s = CString::try_from_fmt(fmt!("{:?}", non_ascii)).unwrap();
- - /// assert_eq!(s.as_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes());
- - /// ```
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - f.write_char('"')?;
- - for &b in &self.0 {
- - match b {
- - // Common escape codes.
- - b'\t' => f.write_str("\\t")?,
- - b'\n' => f.write_str("\\n")?,
- - b'\r' => f.write_str("\\r")?,
- - // String escape characters.
- - b'\"' => f.write_str("\\\"")?,
- - b'\\' => f.write_str("\\\\")?,
- - // Printable characters.
- - 0x20..=0x7e => f.write_char(b as char)?,
- - _ => write!(f, "\\x{:02x}", b)?,
- - }
- - }
- - f.write_char('"')
- - }
- -}
- -
- -impl Deref for BStr {
- - type Target = [u8];
- -
- - #[inline]
- - fn deref(&self) -> &Self::Target {
- - &self.0
- - }
- -}
- -
- -/// Creates a new [`BStr`] from a string literal.
- -///
- -/// `b_str!` converts the supplied string literal to byte string, so non-ASCII
- -/// characters can be included.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// # use kernel::b_str;
- -/// # use kernel::str::BStr;
- -/// const MY_BSTR: &BStr = b_str!("My awesome BStr!");
- -/// ```
- -#[macro_export]
- -macro_rules! b_str {
- - ($str:literal) => {{
- - const S: &'static str = $str;
- - const C: &'static $crate::str::BStr = $crate::str::BStr::from_bytes(S.as_bytes());
- - C
- - }};
- -}
- -
- -/// Possible errors when using conversion functions in [`CStr`].
- -#[derive(Debug, Clone, Copy)]
- -pub enum CStrConvertError {
- - /// Supplied bytes contain an interior `NUL`.
- - InteriorNul,
- -
- - /// Supplied bytes are not terminated by `NUL`.
- - NotNulTerminated,
- -}
- -
- -impl From<CStrConvertError> for Error {
- - #[inline]
- - fn from(_: CStrConvertError) -> Error {
- - EINVAL
- - }
- -}
- -
- -/// A string that is guaranteed to have exactly one `NUL` byte, which is at the
- -/// end.
- -///
- -/// Used for interoperability with kernel APIs that take C strings.
- -#[repr(transparent)]
- -pub struct CStr([u8]);
- -
- -impl CStr {
- - /// Returns the length of this string excluding `NUL`.
- - #[inline]
- - pub const fn len(&self) -> usize {
- - self.len_with_nul() - 1
- - }
- -
- - /// Returns the length of this string with `NUL`.
- - #[inline]
- - pub const fn len_with_nul(&self) -> usize {
- - // SAFETY: This is one of the invariant of `CStr`.
- - // We add a `unreachable_unchecked` here to hint the optimizer that
- - // the value returned from this function is non-zero.
- - if self.0.is_empty() {
- - unsafe { core::hint::unreachable_unchecked() };
- - }
- - self.0.len()
- - }
- -
- - /// Returns `true` if the string only includes `NUL`.
- - #[inline]
- - pub const fn is_empty(&self) -> bool {
- - self.len() == 0
- - }
- -
- - /// Wraps a raw C string pointer.
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must be a valid pointer to a `NUL`-terminated C string, and it must
- - /// last at least `'a`. When `CStr` is alive, the memory pointed by `ptr`
- - /// must not be mutated.
- - #[inline]
- - pub unsafe fn from_char_ptr<'a>(ptr: *const core::ffi::c_char) -> &'a Self {
- - // SAFETY: The safety precondition guarantees `ptr` is a valid pointer
- - // to a `NUL`-terminated C string.
- - let len = unsafe { bindings::strlen(ptr) } + 1;
- - // SAFETY: Lifetime guaranteed by the safety precondition.
- - let bytes = unsafe { core::slice::from_raw_parts(ptr as _, len as _) };
- - // SAFETY: As `len` is returned by `strlen`, `bytes` does not contain interior `NUL`.
- - // As we have added 1 to `len`, the last byte is known to be `NUL`.
- - unsafe { Self::from_bytes_with_nul_unchecked(bytes) }
- - }
- -
- - /// Creates a [`CStr`] from a `[u8]`.
- - ///
- - /// The provided slice must be `NUL`-terminated, does not contain any
- - /// interior `NUL` bytes.
- - pub const fn from_bytes_with_nul(bytes: &[u8]) -> Result<&Self, CStrConvertError> {
- - if bytes.is_empty() {
- - return Err(CStrConvertError::NotNulTerminated);
- - }
- - if bytes[bytes.len() - 1] != 0 {
- - return Err(CStrConvertError::NotNulTerminated);
- - }
- - let mut i = 0;
- - // `i + 1 < bytes.len()` allows LLVM to optimize away bounds checking,
- - // while it couldn't optimize away bounds checks for `i < bytes.len() - 1`.
- - while i + 1 < bytes.len() {
- - if bytes[i] == 0 {
- - return Err(CStrConvertError::InteriorNul);
- - }
- - i += 1;
- - }
- - // SAFETY: We just checked that all properties hold.
- - Ok(unsafe { Self::from_bytes_with_nul_unchecked(bytes) })
- - }
- -
- - /// Creates a [`CStr`] from a `[u8]` without performing any additional
- - /// checks.
- - ///
- - /// # Safety
- - ///
- - /// `bytes` *must* end with a `NUL` byte, and should only have a single
- - /// `NUL` byte (or the string will be truncated).
- - #[inline]
- - pub const unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr {
- - // SAFETY: Properties of `bytes` guaranteed by the safety precondition.
- - unsafe { core::mem::transmute(bytes) }
- - }
- -
- - /// Creates a mutable [`CStr`] from a `[u8]` without performing any
- - /// additional checks.
- - ///
- - /// # Safety
- - ///
- - /// `bytes` *must* end with a `NUL` byte, and should only have a single
- - /// `NUL` byte (or the string will be truncated).
- - #[inline]
- - pub unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut CStr {
- - // SAFETY: Properties of `bytes` guaranteed by the safety precondition.
- - unsafe { &mut *(bytes as *mut [u8] as *mut CStr) }
- - }
- -
- - /// Returns a C pointer to the string.
- - #[inline]
- - pub const fn as_char_ptr(&self) -> *const core::ffi::c_char {
- - self.0.as_ptr() as _
- - }
- -
- - /// Convert the string to a byte slice without the trailing `NUL` byte.
- - #[inline]
- - pub fn as_bytes(&self) -> &[u8] {
- - &self.0[..self.len()]
- - }
- -
- - /// Convert the string to a byte slice containing the trailing `NUL` byte.
- - #[inline]
- - pub const fn as_bytes_with_nul(&self) -> &[u8] {
- - &self.0
- - }
- -
- - /// Yields a [`&str`] slice if the [`CStr`] contains valid UTF-8.
- - ///
- - /// If the contents of the [`CStr`] are valid UTF-8 data, this
- - /// function will return the corresponding [`&str`] slice. Otherwise,
- - /// it will return an error with details of where UTF-8 validation failed.
- - ///
- - /// # Examples
- - ///
- - /// ```
- - /// # use kernel::str::CStr;
- - /// let cstr = CStr::from_bytes_with_nul(b"foo\0").unwrap();
- - /// assert_eq!(cstr.to_str(), Ok("foo"));
- - /// ```
- - #[inline]
- - pub fn to_str(&self) -> Result<&str, core::str::Utf8Error> {
- - core::str::from_utf8(self.as_bytes())
- - }
- -
- - /// Unsafely convert this [`CStr`] into a [`&str`], without checking for
- - /// valid UTF-8.
- - ///
- - /// # Safety
- - ///
- - /// The contents must be valid UTF-8.
- - ///
- - /// # Examples
- - ///
- - /// ```
- - /// # use kernel::c_str;
- - /// # use kernel::str::CStr;
- - /// let bar = c_str!("ツ");
- - /// // SAFETY: String literals are guaranteed to be valid UTF-8
- - /// // by the Rust compiler.
- - /// assert_eq!(unsafe { bar.as_str_unchecked() }, "ツ");
- - /// ```
- - #[inline]
- - pub unsafe fn as_str_unchecked(&self) -> &str {
- - unsafe { core::str::from_utf8_unchecked(self.as_bytes()) }
- - }
- -
- - /// Convert this [`CStr`] into a [`CString`] by allocating memory and
- - /// copying over the string data.
- - pub fn to_cstring(&self) -> Result<CString, AllocError> {
- - CString::try_from(self)
- - }
- -
- - /// Converts this [`CStr`] to its ASCII lower case equivalent in-place.
- - ///
- - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
- - /// but non-ASCII letters are unchanged.
- - ///
- - /// To return a new lowercased value without modifying the existing one, use
- - /// [`to_ascii_lowercase()`].
- - ///
- - /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase
- - pub fn make_ascii_lowercase(&mut self) {
- - // INVARIANT: This doesn't introduce or remove NUL bytes in the C
- - // string.
- - self.0.make_ascii_lowercase();
- - }
- -
- - /// Converts this [`CStr`] to its ASCII upper case equivalent in-place.
- - ///
- - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
- - /// but non-ASCII letters are unchanged.
- - ///
- - /// To return a new uppercased value without modifying the existing one, use
- - /// [`to_ascii_uppercase()`].
- - ///
- - /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase
- - pub fn make_ascii_uppercase(&mut self) {
- - // INVARIANT: This doesn't introduce or remove NUL bytes in the C
- - // string.
- - self.0.make_ascii_uppercase();
- - }
- -
- - /// Returns a copy of this [`CString`] where each character is mapped to its
- - /// ASCII lower case equivalent.
- - ///
- - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
- - /// but non-ASCII letters are unchanged.
- - ///
- - /// To lowercase the value in-place, use [`make_ascii_lowercase`].
- - ///
- - /// [`make_ascii_lowercase`]: str::make_ascii_lowercase
- - pub fn to_ascii_lowercase(&self) -> Result<CString, AllocError> {
- - let mut s = self.to_cstring()?;
- -
- - s.make_ascii_lowercase();
- -
- - Ok(s)
- - }
- -
- - /// Returns a copy of this [`CString`] where each character is mapped to its
- - /// ASCII upper case equivalent.
- - ///
- - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
- - /// but non-ASCII letters are unchanged.
- - ///
- - /// To uppercase the value in-place, use [`make_ascii_uppercase`].
- - ///
- - /// [`make_ascii_uppercase`]: str::make_ascii_uppercase
- - pub fn to_ascii_uppercase(&self) -> Result<CString, AllocError> {
- - let mut s = self.to_cstring()?;
- -
- - s.make_ascii_uppercase();
- -
- - Ok(s)
- - }
- -}
- -
- -impl fmt::Display for CStr {
- - /// Formats printable ASCII characters, escaping the rest.
- - ///
- - /// ```
- - /// # use kernel::c_str;
- - /// # use kernel::fmt;
- - /// # use kernel::str::CStr;
- - /// # use kernel::str::CString;
- - /// let penguin = c_str!("🐧");
- - /// let s = CString::try_from_fmt(fmt!("{}", penguin)).unwrap();
- - /// assert_eq!(s.as_bytes_with_nul(), "\\xf0\\x9f\\x90\\xa7\0".as_bytes());
- - ///
- - /// let ascii = c_str!("so \"cool\"");
- - /// let s = CString::try_from_fmt(fmt!("{}", ascii)).unwrap();
- - /// assert_eq!(s.as_bytes_with_nul(), "so \"cool\"\0".as_bytes());
- - /// ```
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - for &c in self.as_bytes() {
- - if (0x20..0x7f).contains(&c) {
- - // Printable character.
- - f.write_char(c as char)?;
- - } else {
- - write!(f, "\\x{:02x}", c)?;
- - }
- - }
- - Ok(())
- - }
- -}
- -
- -impl fmt::Debug for CStr {
- - /// Formats printable ASCII characters with a double quote on either end, escaping the rest.
- - ///
- - /// ```
- - /// # use kernel::c_str;
- - /// # use kernel::fmt;
- - /// # use kernel::str::CStr;
- - /// # use kernel::str::CString;
- - /// let penguin = c_str!("🐧");
- - /// let s = CString::try_from_fmt(fmt!("{:?}", penguin)).unwrap();
- - /// assert_eq!(s.as_bytes_with_nul(), "\"\\xf0\\x9f\\x90\\xa7\"\0".as_bytes());
- - ///
- - /// // Embedded double quotes are escaped.
- - /// let ascii = c_str!("so \"cool\"");
- - /// let s = CString::try_from_fmt(fmt!("{:?}", ascii)).unwrap();
- - /// assert_eq!(s.as_bytes_with_nul(), "\"so \\\"cool\\\"\"\0".as_bytes());
- - /// ```
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - f.write_str("\"")?;
- - for &c in self.as_bytes() {
- - match c {
- - // Printable characters.
- - b'\"' => f.write_str("\\\"")?,
- - 0x20..=0x7e => f.write_char(c as char)?,
- - _ => write!(f, "\\x{:02x}", c)?,
- - }
- - }
- - f.write_str("\"")
- - }
- -}
- -
- -impl AsRef<BStr> for CStr {
- - #[inline]
- - fn as_ref(&self) -> &BStr {
- - BStr::from_bytes(self.as_bytes())
- - }
- -}
- -
- -impl Deref for CStr {
- - type Target = BStr;
- -
- - #[inline]
- - fn deref(&self) -> &Self::Target {
- - self.as_ref()
- - }
- -}
- -
- -impl Index<ops::RangeFrom<usize>> for CStr {
- - type Output = CStr;
- -
- - #[inline]
- - fn index(&self, index: ops::RangeFrom<usize>) -> &Self::Output {
- - // Delegate bounds checking to slice.
- - // Assign to _ to mute clippy's unnecessary operation warning.
- - let _ = &self.as_bytes()[index.start..];
- - // SAFETY: We just checked the bounds.
- - unsafe { Self::from_bytes_with_nul_unchecked(&self.0[index.start..]) }
- - }
- -}
- -
- -impl Index<ops::RangeFull> for CStr {
- - type Output = CStr;
- -
- - #[inline]
- - fn index(&self, _index: ops::RangeFull) -> &Self::Output {
- - self
- - }
- -}
- -
- -mod private {
- - use core::ops;
- -
- - // Marker trait for index types that can be forward to `BStr`.
- - pub trait CStrIndex {}
- -
- - impl CStrIndex for usize {}
- - impl CStrIndex for ops::Range<usize> {}
- - impl CStrIndex for ops::RangeInclusive<usize> {}
- - impl CStrIndex for ops::RangeToInclusive<usize> {}
- -}
- -
- -impl<Idx> Index<Idx> for CStr
- -where
- - Idx: private::CStrIndex,
- - BStr: Index<Idx>,
- -{
- - type Output = <BStr as Index<Idx>>::Output;
- -
- - #[inline]
- - fn index(&self, index: Idx) -> &Self::Output {
- - &self.as_ref()[index]
- - }
- -}
- -
- -/// Creates a new [`CStr`] from a string literal.
- -///
- -/// The string literal should not contain any `NUL` bytes.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// # use kernel::c_str;
- -/// # use kernel::str::CStr;
- -/// const MY_CSTR: &CStr = c_str!("My awesome CStr!");
- -/// ```
- -#[macro_export]
- -macro_rules! c_str {
- - ($str:expr) => {{
- - const S: &str = concat!($str, "\0");
- - const C: &$crate::str::CStr = match $crate::str::CStr::from_bytes_with_nul(S.as_bytes()) {
- - Ok(v) => v,
- - Err(_) => panic!("string contains interior NUL"),
- - };
- - C
- - }};
- -}
- -
- -#[cfg(test)]
- -mod tests {
- - use super::*;
- - use alloc::format;
- -
- - const ALL_ASCII_CHARS: &'static str =
- - "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\
- - \\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f \
- - !\"#$%&'()*+,-./0123456789:;<=>?@\
- - ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\
- - \\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d\\x8e\\x8f\
- - \\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b\\x9c\\x9d\\x9e\\x9f\
- - \\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9\\xaa\\xab\\xac\\xad\\xae\\xaf\
- - \\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\
- - \\xc0\\xc1\\xc2\\xc3\\xc4\\xc5\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\
- - \\xd0\\xd1\\xd2\\xd3\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\
- - \\xe0\\xe1\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef\
- - \\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd\\xfe\\xff";
- -
- - #[test]
- - fn test_cstr_to_str() {
- - let good_bytes = b"\xf0\x9f\xa6\x80\0";
- - let checked_cstr = CStr::from_bytes_with_nul(good_bytes).unwrap();
- - let checked_str = checked_cstr.to_str().unwrap();
- - assert_eq!(checked_str, "🦀");
- - }
- -
- - #[test]
- - #[should_panic]
- - fn test_cstr_to_str_panic() {
- - let bad_bytes = b"\xc3\x28\0";
- - let checked_cstr = CStr::from_bytes_with_nul(bad_bytes).unwrap();
- - checked_cstr.to_str().unwrap();
- - }
- -
- - #[test]
- - fn test_cstr_as_str_unchecked() {
- - let good_bytes = b"\xf0\x9f\x90\xA7\0";
- - let checked_cstr = CStr::from_bytes_with_nul(good_bytes).unwrap();
- - let unchecked_str = unsafe { checked_cstr.as_str_unchecked() };
- - assert_eq!(unchecked_str, "🐧");
- - }
- -
- - #[test]
- - fn test_cstr_display() {
- - let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap();
- - assert_eq!(format!("{}", hello_world), "hello, world!");
- - let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap();
- - assert_eq!(format!("{}", non_printables), "\\x01\\x09\\x0a");
- - let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap();
- - assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu");
- - let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap();
- - assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80");
- - }
- -
- - #[test]
- - fn test_cstr_display_all_bytes() {
- - let mut bytes: [u8; 256] = [0; 256];
- - // fill `bytes` with [1..=255] + [0]
- - for i in u8::MIN..=u8::MAX {
- - bytes[i as usize] = i.wrapping_add(1);
- - }
- - let cstr = CStr::from_bytes_with_nul(&bytes).unwrap();
- - assert_eq!(format!("{}", cstr), ALL_ASCII_CHARS);
- - }
- -
- - #[test]
- - fn test_cstr_debug() {
- - let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap();
- - assert_eq!(format!("{:?}", hello_world), "\"hello, world!\"");
- - let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap();
- - assert_eq!(format!("{:?}", non_printables), "\"\\x01\\x09\\x0a\"");
- - let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap();
- - assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\"");
- - let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap();
- - assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\"");
- - }
- -
- - #[test]
- - fn test_bstr_display() {
- - let hello_world = BStr::from_bytes(b"hello, world!");
- - assert_eq!(format!("{}", hello_world), "hello, world!");
- - let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_");
- - assert_eq!(format!("{}", escapes), "_\\t_\\n_\\r_\\_'_\"_");
- - let others = BStr::from_bytes(b"\x01");
- - assert_eq!(format!("{}", others), "\\x01");
- - let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu");
- - assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu");
- - let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80");
- - assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80");
- - }
- -
- - #[test]
- - fn test_bstr_debug() {
- - let hello_world = BStr::from_bytes(b"hello, world!");
- - assert_eq!(format!("{:?}", hello_world), "\"hello, world!\"");
- - let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_");
- - assert_eq!(format!("{:?}", escapes), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\"");
- - let others = BStr::from_bytes(b"\x01");
- - assert_eq!(format!("{:?}", others), "\"\\x01\"");
- - let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu");
- - assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\"");
- - let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80");
- - assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\"");
- - }
- -}
- -
- -/// Allows formatting of [`fmt::Arguments`] into a raw buffer.
- -///
- -/// It does not fail if callers write past the end of the buffer so that they can calculate the
- -/// size required to fit everything.
- -///
- -/// # Invariants
- -///
- -/// The memory region between `pos` (inclusive) and `end` (exclusive) is valid for writes if `pos`
- -/// is less than `end`.
- -pub(crate) struct RawFormatter {
- - // Use `usize` to use `saturating_*` functions.
- - beg: usize,
- - pos: usize,
- - end: usize,
- -}
- -
- -impl RawFormatter {
- - /// Creates a new instance of [`RawFormatter`] with an empty buffer.
- - fn new() -> Self {
- - // INVARIANT: The buffer is empty, so the region that needs to be writable is empty.
- - Self {
- - beg: 0,
- - pos: 0,
- - end: 0,
- - }
- - }
- -
- - /// Creates a new instance of [`RawFormatter`] with the given buffer pointers.
- - ///
- - /// # Safety
- - ///
- - /// If `pos` is less than `end`, then the region between `pos` (inclusive) and `end`
- - /// (exclusive) must be valid for writes for the lifetime of the returned [`RawFormatter`].
- - pub(crate) unsafe fn from_ptrs(pos: *mut u8, end: *mut u8) -> Self {
- - // INVARIANT: The safety requirements guarantee the type invariants.
- - Self {
- - beg: pos as _,
- - pos: pos as _,
- - end: end as _,
- - }
- - }
- -
- - /// Creates a new instance of [`RawFormatter`] with the given buffer.
- - ///
- - /// # Safety
- - ///
- - /// The memory region starting at `buf` and extending for `len` bytes must be valid for writes
- - /// for the lifetime of the returned [`RawFormatter`].
- - pub(crate) unsafe fn from_buffer(buf: *mut u8, len: usize) -> Self {
- - let pos = buf as usize;
- - // INVARIANT: We ensure that `end` is never less then `buf`, and the safety requirements
- - // guarantees that the memory region is valid for writes.
- - Self {
- - pos,
- - beg: pos,
- - end: pos.saturating_add(len),
- - }
- - }
- -
- - /// Returns the current insert position.
- - ///
- - /// N.B. It may point to invalid memory.
- - pub(crate) fn pos(&self) -> *mut u8 {
- - self.pos as _
- - }
- -
- - /// Returns the number of bytes written to the formatter.
- - pub(crate) fn bytes_written(&self) -> usize {
- - self.pos - self.beg
- - }
- -}
- -
- -impl fmt::Write for RawFormatter {
- - fn write_str(&mut self, s: &str) -> fmt::Result {
- - // `pos` value after writing `len` bytes. This does not have to be bounded by `end`, but we
- - // don't want it to wrap around to 0.
- - let pos_new = self.pos.saturating_add(s.len());
- -
- - // Amount that we can copy. `saturating_sub` ensures we get 0 if `pos` goes past `end`.
- - let len_to_copy = core::cmp::min(pos_new, self.end).saturating_sub(self.pos);
- -
- - if len_to_copy > 0 {
- - // SAFETY: If `len_to_copy` is non-zero, then we know `pos` has not gone past `end`
- - // yet, so it is valid for write per the type invariants.
- - unsafe {
- - core::ptr::copy_nonoverlapping(
- - s.as_bytes().as_ptr(),
- - self.pos as *mut u8,
- - len_to_copy,
- - )
- - };
- - }
- -
- - self.pos = pos_new;
- - Ok(())
- - }
- -}
- -
- -/// Allows formatting of [`fmt::Arguments`] into a raw buffer.
- -///
- -/// Fails if callers attempt to write more than will fit in the buffer.
- -pub(crate) struct Formatter(RawFormatter);
- -
- -impl Formatter {
- - /// Creates a new instance of [`Formatter`] with the given buffer.
- - ///
- - /// # Safety
- - ///
- - /// The memory region starting at `buf` and extending for `len` bytes must be valid for writes
- - /// for the lifetime of the returned [`Formatter`].
- - pub(crate) unsafe fn from_buffer(buf: *mut u8, len: usize) -> Self {
- - // SAFETY: The safety requirements of this function satisfy those of the callee.
- - Self(unsafe { RawFormatter::from_buffer(buf, len) })
- - }
- -}
- -
- -impl Deref for Formatter {
- - type Target = RawFormatter;
- -
- - fn deref(&self) -> &Self::Target {
- - &self.0
- - }
- -}
- -
- -impl fmt::Write for Formatter {
- - fn write_str(&mut self, s: &str) -> fmt::Result {
- - self.0.write_str(s)?;
- -
- - // Fail the request if we go past the end of the buffer.
- - if self.0.pos > self.0.end {
- - Err(fmt::Error)
- - } else {
- - Ok(())
- - }
- - }
- -}
- -
- -/// An owned string that is guaranteed to have exactly one `NUL` byte, which is at the end.
- -///
- -/// Used for interoperability with kernel APIs that take C strings.
- -///
- -/// # Invariants
- -///
- -/// The string is always `NUL`-terminated and contains no other `NUL` bytes.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// use kernel::{str::CString, fmt};
- -///
- -/// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20)).unwrap();
- -/// assert_eq!(s.as_bytes_with_nul(), "abc1020\0".as_bytes());
- -///
- -/// let tmp = "testing";
- -/// let s = CString::try_from_fmt(fmt!("{tmp}{}", 123)).unwrap();
- -/// assert_eq!(s.as_bytes_with_nul(), "testing123\0".as_bytes());
- -///
- -/// // This fails because it has an embedded `NUL` byte.
- -/// let s = CString::try_from_fmt(fmt!("a\0b{}", 123));
- -/// assert_eq!(s.is_ok(), false);
- -/// ```
- -pub struct CString {
- - buf: Vec<u8>,
- -}
- -
- -impl CString {
- - /// Creates an instance of [`CString`] from the given formatted arguments.
- - pub fn try_from_fmt(args: fmt::Arguments<'_>) -> Result<Self, Error> {
- - // Calculate the size needed (formatted string plus `NUL` terminator).
- - let mut f = RawFormatter::new();
- - f.write_fmt(args)?;
- - f.write_str("\0")?;
- - let size = f.bytes_written();
- -
- - // Allocate a vector with the required number of bytes, and write to it.
- - let mut buf = <Vec<_> as VecExt<_>>::with_capacity(size, GFP_KERNEL)?;
- - // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes.
- - let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) };
- - f.write_fmt(args)?;
- - f.write_str("\0")?;
- -
- - // SAFETY: The number of bytes that can be written to `f` is bounded by `size`, which is
- - // `buf`'s capacity. The contents of the buffer have been initialised by writes to `f`.
- - unsafe { buf.set_len(f.bytes_written()) };
- -
- - // Check that there are no `NUL` bytes before the end.
- - // SAFETY: The buffer is valid for read because `f.bytes_written()` is bounded by `size`
- - // (which the minimum buffer size) and is non-zero (we wrote at least the `NUL` terminator)
- - // so `f.bytes_written() - 1` doesn't underflow.
- - let ptr = unsafe { bindings::memchr(buf.as_ptr().cast(), 0, (f.bytes_written() - 1) as _) };
- - if !ptr.is_null() {
- - return Err(EINVAL);
- - }
- -
- - // INVARIANT: We wrote the `NUL` terminator and checked above that no other `NUL` bytes
- - // exist in the buffer.
- - Ok(Self { buf })
- - }
- -}
- -
- -impl Deref for CString {
- - type Target = CStr;
- -
- - fn deref(&self) -> &Self::Target {
- - // SAFETY: The type invariants guarantee that the string is `NUL`-terminated and that no
- - // other `NUL` bytes exist.
- - unsafe { CStr::from_bytes_with_nul_unchecked(self.buf.as_slice()) }
- - }
- -}
- -
- -impl DerefMut for CString {
- - fn deref_mut(&mut self) -> &mut Self::Target {
- - // SAFETY: A `CString` is always NUL-terminated and contains no other
- - // NUL bytes.
- - unsafe { CStr::from_bytes_with_nul_unchecked_mut(self.buf.as_mut_slice()) }
- - }
- -}
- -
- -impl<'a> TryFrom<&'a CStr> for CString {
- - type Error = AllocError;
- -
- - fn try_from(cstr: &'a CStr) -> Result<CString, AllocError> {
- - let mut buf = Vec::new();
- -
- - <Vec<_> as VecExt<_>>::extend_from_slice(&mut buf, cstr.as_bytes_with_nul(), GFP_KERNEL)
- - .map_err(|_| AllocError)?;
- -
- - // INVARIANT: The `CStr` and `CString` types have the same invariants for
- - // the string data, and we copied it over without changes.
- - Ok(CString { buf })
- - }
- -}
- -
- -impl fmt::Debug for CString {
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - fmt::Debug::fmt(&**self, f)
- - }
- -}
- -
- -/// A convenience alias for [`core::format_args`].
- -#[macro_export]
- -macro_rules! fmt {
- - ($($f:tt)*) => ( core::format_args!($($f)*) )
- -}
- diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs
- deleted file mode 100644
- --- a/rust/kernel/sync.rs
- +++ /dev/null
- @@ -1,67 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Synchronisation primitives.
- -//!
- -//! This module contains the kernel APIs related to synchronisation that have been ported or
- -//! wrapped for usage by Rust code in the kernel.
- -
- -use crate::types::Opaque;
- -
- -mod arc;
- -mod condvar;
- -pub mod lock;
- -mod locked_by;
- -
- -pub use arc::{Arc, ArcBorrow, UniqueArc};
- -pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult};
- -pub use lock::mutex::{new_mutex, Mutex};
- -pub use lock::spinlock::{new_spinlock, SpinLock};
- -pub use locked_by::LockedBy;
- -
- -/// Represents a lockdep class. It's a wrapper around C's `lock_class_key`.
- -#[repr(transparent)]
- -pub struct LockClassKey(Opaque<bindings::lock_class_key>);
- -
- -// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and
- -// provides its own synchronization.
- -unsafe impl Sync for LockClassKey {}
- -
- -impl LockClassKey {
- - /// Creates a new lock class key.
- - pub const fn new() -> Self {
- - Self(Opaque::uninit())
- - }
- -
- - pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key {
- - self.0.get()
- - }
- -}
- -
- -impl Default for LockClassKey {
- - fn default() -> Self {
- - Self::new()
- - }
- -}
- -
- -/// Defines a new static lock class and returns a pointer to it.
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! static_lock_class {
- - () => {{
- - static CLASS: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new();
- - &CLASS
- - }};
- -}
- -
- -/// Returns the given string, if one is provided, otherwise generates one based on the source code
- -/// location.
- -#[doc(hidden)]
- -#[macro_export]
- -macro_rules! optional_name {
- - () => {
- - $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!()))
- - };
- - ($name:literal) => {
- - $crate::c_str!($name)
- - };
- -}
- diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs
- deleted file mode 100644
- --- a/rust/kernel/sync/arc.rs
- +++ /dev/null
- @@ -1,782 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! A reference-counted pointer.
- -//!
- -//! This module implements a way for users to create reference-counted objects and pointers to
- -//! them. Such a pointer automatically increments and decrements the count, and drops the
- -//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
- -//! threads.
- -//!
- -//! It is different from the standard library's [`Arc`] in a few ways:
- -//! 1. It is backed by the kernel's `refcount_t` type.
- -//! 2. It does not support weak references, which allows it to be half the size.
- -//! 3. It saturates the reference count instead of aborting when it goes over a threshold.
- -//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
- -//!
- -//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
- -
- -use crate::{
- - alloc::{box_ext::BoxExt, AllocError, Flags},
- - error::{self, Error},
- - init::{self, InPlaceInit, Init, PinInit},
- - try_init,
- - types::{ForeignOwnable, Opaque},
- -};
- -use alloc::boxed::Box;
- -use core::{
- - alloc::Layout,
- - fmt,
- - marker::{PhantomData, Unsize},
- - mem::{ManuallyDrop, MaybeUninit},
- - ops::{Deref, DerefMut},
- - pin::Pin,
- - ptr::NonNull,
- -};
- -use macros::pin_data;
- -
- -mod std_vendor;
- -
- -/// A reference-counted pointer to an instance of `T`.
- -///
- -/// The reference count is incremented when new instances of [`Arc`] are created, and decremented
- -/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
- -///
- -/// # Invariants
- -///
- -/// The reference count on an instance of [`Arc`] is always non-zero.
- -/// The object pointed to by [`Arc`] is always pinned.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// use kernel::sync::Arc;
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// // Create a refcounted instance of `Example`.
- -/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
- -///
- -/// // Get a new pointer to `obj` and increment the refcount.
- -/// let cloned = obj.clone();
- -///
- -/// // Assert that both `obj` and `cloned` point to the same underlying object.
- -/// assert!(core::ptr::eq(&*obj, &*cloned));
- -///
- -/// // Destroy `obj` and decrement its refcount.
- -/// drop(obj);
- -///
- -/// // Check that the values are still accessible through `cloned`.
- -/// assert_eq!(cloned.a, 10);
- -/// assert_eq!(cloned.b, 20);
- -///
- -/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
- -/// # Ok::<(), Error>(())
- -/// ```
- -///
- -/// Using `Arc<T>` as the type of `self`:
- -///
- -/// ```
- -/// use kernel::sync::Arc;
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// impl Example {
- -/// fn take_over(self: Arc<Self>) {
- -/// // ...
- -/// }
- -///
- -/// fn use_reference(self: &Arc<Self>) {
- -/// // ...
- -/// }
- -/// }
- -///
- -/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
- -/// obj.use_reference();
- -/// obj.take_over();
- -/// # Ok::<(), Error>(())
- -/// ```
- -///
- -/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
- -///
- -/// ```
- -/// use kernel::sync::{Arc, ArcBorrow};
- -///
- -/// trait MyTrait {
- -/// // Trait has a function whose `self` type is `Arc<Self>`.
- -/// fn example1(self: Arc<Self>) {}
- -///
- -/// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
- -/// fn example2(self: ArcBorrow<'_, Self>) {}
- -/// }
- -///
- -/// struct Example;
- -/// impl MyTrait for Example {}
- -///
- -/// // `obj` has type `Arc<Example>`.
- -/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;
- -///
- -/// // `coerced` has type `Arc<dyn MyTrait>`.
- -/// let coerced: Arc<dyn MyTrait> = obj;
- -/// # Ok::<(), Error>(())
- -/// ```
- -pub struct Arc<T: ?Sized> {
- - ptr: NonNull<ArcInner<T>>,
- - _p: PhantomData<ArcInner<T>>,
- -}
- -
- -#[pin_data]
- -#[repr(C)]
- -struct ArcInner<T: ?Sized> {
- - refcount: Opaque<bindings::refcount_t>,
- - data: T,
- -}
- -
- -impl<T: ?Sized> ArcInner<T> {
- - /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
- - /// not yet have been destroyed.
- - unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
- - let refcount_layout = Layout::new::<bindings::refcount_t>();
- - // SAFETY: The caller guarantees that the pointer is valid.
- - let val_layout = Layout::for_value(unsafe { &*ptr });
- - // SAFETY: We're computing the layout of a real struct that existed when compiling this
- - // binary, so its layout is not so large that it can trigger arithmetic overflow.
- - let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
- -
- - // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
- - // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
- - //
- - // This is documented at:
- - // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
- - let ptr = ptr as *const ArcInner<T>;
- -
- - // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
- - // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is
- - // still valid.
- - let ptr = unsafe { ptr.byte_sub(val_offset) };
- -
- - // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null
- - // address.
- - unsafe { NonNull::new_unchecked(ptr.cast_mut()) }
- - }
- -}
- -
- -// This is to allow [`Arc`] (and variants) to be used as the type of `self`.
- -impl<T: ?Sized> core::ops::Receiver for Arc<T> {}
- -
- -// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
- -// dynamically-sized type (DST) `U`.
- -impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
- -
- -// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
- -impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
- -
- -// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
- -// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
- -// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
- -// mutable reference when the reference count reaches zero and `T` is dropped.
- -unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
- -
- -// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
- -// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
- -// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
- -// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
- -// the reference count reaches zero and `T` is dropped.
- -unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
- -
- -impl<T> Arc<T> {
- - /// Constructs a new reference counted instance of `T`.
- - pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
- - // INVARIANT: The refcount is initialised to a non-zero value.
- - let value = ArcInner {
- - // SAFETY: There are no safety requirements for this FFI call.
- - refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
- - data: contents,
- - };
- -
- - let inner = <Box<_> as BoxExt<_>>::new(value, flags)?;
- -
- - // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
- - // `Arc` object.
- - Ok(unsafe { Self::from_inner(Box::leak(inner).into()) })
- - }
- -
- - /// Use the given initializer to in-place initialize a `T`.
- - ///
- - /// If `T: !Unpin` it will not be able to move afterwards.
- - #[inline]
- - pub fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self>
- - where
- - Error: From<E>,
- - {
- - UniqueArc::pin_init(init, flags).map(|u| u.into())
- - }
- -
- - /// Use the given initializer to in-place initialize a `T`.
- - ///
- - /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned.
- - #[inline]
- - pub fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
- - where
- - Error: From<E>,
- - {
- - UniqueArc::init(init, flags).map(|u| u.into())
- - }
- -}
- -
- -impl<T: ?Sized> Arc<T> {
- - /// Constructs a new [`Arc`] from an existing [`ArcInner`].
- - ///
- - /// # Safety
- - ///
- - /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
- - /// count, one of which will be owned by the new [`Arc`] instance.
- - unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
- - // INVARIANT: By the safety requirements, the invariants hold.
- - Arc {
- - ptr: inner,
- - _p: PhantomData,
- - }
- - }
- -
- - /// Convert the [`Arc`] into a raw pointer.
- - ///
- - /// The raw pointer has ownership of the refcount that this Arc object owned.
- - pub fn into_raw(self) -> *const T {
- - let ptr = self.ptr.as_ptr();
- - core::mem::forget(self);
- - // SAFETY: The pointer is valid.
- - unsafe { core::ptr::addr_of!((*ptr).data) }
- - }
- -
- - /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
- - /// must not be called more than once for each previous call to [`Arc::into_raw`].
- - pub unsafe fn from_raw(ptr: *const T) -> Self {
- - // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
- - // `Arc` that is still valid.
- - let ptr = unsafe { ArcInner::container_of(ptr) };
- -
- - // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
- - // reference count held then will be owned by the new `Arc` object.
- - unsafe { Self::from_inner(ptr) }
- - }
- -
- - /// Returns an [`ArcBorrow`] from the given [`Arc`].
- - ///
- - /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
- - /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
- - #[inline]
- - pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
- - // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
- - // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
- - // reference can be created.
- - unsafe { ArcBorrow::new(self.ptr) }
- - }
- -
- - /// Compare whether two [`Arc`] pointers reference the same underlying object.
- - pub fn ptr_eq(this: &Self, other: &Self) -> bool {
- - core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
- - }
- -
- - /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
- - ///
- - /// When this destroys the `Arc`, it does so while properly avoiding races. This means that
- - /// this method will never call the destructor of the value.
- - ///
- - /// # Examples
- - ///
- - /// ```
- - /// use kernel::sync::{Arc, UniqueArc};
- - ///
- - /// let arc = Arc::new(42, GFP_KERNEL)?;
- - /// let unique_arc = arc.into_unique_or_drop();
- - ///
- - /// // The above conversion should succeed since refcount of `arc` is 1.
- - /// assert!(unique_arc.is_some());
- - ///
- - /// assert_eq!(*(unique_arc.unwrap()), 42);
- - ///
- - /// # Ok::<(), Error>(())
- - /// ```
- - ///
- - /// ```
- - /// use kernel::sync::{Arc, UniqueArc};
- - ///
- - /// let arc = Arc::new(42, GFP_KERNEL)?;
- - /// let another = arc.clone();
- - ///
- - /// let unique_arc = arc.into_unique_or_drop();
- - ///
- - /// // The above conversion should fail since refcount of `arc` is >1.
- - /// assert!(unique_arc.is_none());
- - ///
- - /// # Ok::<(), Error>(())
- - /// ```
- - pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> {
- - // We will manually manage the refcount in this method, so we disable the destructor.
- - let me = ManuallyDrop::new(self);
- - // SAFETY: We own a refcount, so the pointer is still valid.
- - let refcount = unsafe { me.ptr.as_ref() }.refcount.get();
- -
- - // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
- - // return without further touching the `Arc`. If the refcount reaches zero, then there are
- - // no other arcs, and we can create a `UniqueArc`.
- - //
- - // SAFETY: We own a refcount, so the pointer is not dangling.
- - let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
- - if is_zero {
- - // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized
- - // accesses to the refcount.
- - unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) };
- -
- - // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
- - // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
- - // their values.
- - Some(Pin::from(UniqueArc {
- - inner: ManuallyDrop::into_inner(me),
- - }))
- - } else {
- - None
- - }
- - }
- -}
- -
- -impl<T: 'static> ForeignOwnable for Arc<T> {
- - type Borrowed<'a> = ArcBorrow<'a, T>;
- -
- - fn into_foreign(self) -> *const core::ffi::c_void {
- - ManuallyDrop::new(self).ptr.as_ptr() as _
- - }
- -
- - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> {
- - // SAFETY: By the safety requirement of this function, we know that `ptr` came from
- - // a previous call to `Arc::into_foreign`.
- - let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap();
- -
- - // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
- - // for the lifetime of the returned value.
- - unsafe { ArcBorrow::new(inner) }
- - }
- -
- - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {
- - // SAFETY: By the safety requirement of this function, we know that `ptr` came from
- - // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
- - // holds a reference count increment that is transferrable to us.
- - unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) }
- - }
- -}
- -
- -impl<T: ?Sized> Deref for Arc<T> {
- - type Target = T;
- -
- - fn deref(&self) -> &Self::Target {
- - // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
- - // safe to dereference it.
- - unsafe { &self.ptr.as_ref().data }
- - }
- -}
- -
- -impl<T: ?Sized> AsRef<T> for Arc<T> {
- - fn as_ref(&self) -> &T {
- - self.deref()
- - }
- -}
- -
- -impl<T: ?Sized> Clone for Arc<T> {
- - fn clone(&self) -> Self {
- - // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
- - // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
- - // safe to increment the refcount.
- - unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) };
- -
- - // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
- - unsafe { Self::from_inner(self.ptr) }
- - }
- -}
- -
- -impl<T: ?Sized> Drop for Arc<T> {
- - fn drop(&mut self) {
- - // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
- - // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
- - // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
- - // freed/invalid memory as long as it is never dereferenced.
- - let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
- -
- - // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
- - // this instance is being dropped, so the broken invariant is not observable.
- - // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
- - let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
- - if is_zero {
- - // The count reached zero, we must free the memory.
- - //
- - // SAFETY: The pointer was initialised from the result of `Box::leak`.
- - unsafe { drop(Box::from_raw(self.ptr.as_ptr())) };
- - }
- - }
- -}
- -
- -impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
- - fn from(item: UniqueArc<T>) -> Self {
- - item.inner
- - }
- -}
- -
- -impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
- - fn from(item: Pin<UniqueArc<T>>) -> Self {
- - // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
- - unsafe { Pin::into_inner_unchecked(item).inner }
- - }
- -}
- -
- -/// A borrowed reference to an [`Arc`] instance.
- -///
- -/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
- -/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
- -///
- -/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
- -/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
- -/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
- -/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
- -/// needed.
- -///
- -/// # Invariants
- -///
- -/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
- -/// lifetime of the [`ArcBorrow`] instance.
- -///
- -/// # Example
- -///
- -/// ```
- -/// use kernel::sync::{Arc, ArcBorrow};
- -///
- -/// struct Example;
- -///
- -/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
- -/// e.into()
- -/// }
- -///
- -/// let obj = Arc::new(Example, GFP_KERNEL)?;
- -/// let cloned = do_something(obj.as_arc_borrow());
- -///
- -/// // Assert that both `obj` and `cloned` point to the same underlying object.
- -/// assert!(core::ptr::eq(&*obj, &*cloned));
- -/// # Ok::<(), Error>(())
- -/// ```
- -///
- -/// Using `ArcBorrow<T>` as the type of `self`:
- -///
- -/// ```
- -/// use kernel::sync::{Arc, ArcBorrow};
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// impl Example {
- -/// fn use_reference(self: ArcBorrow<'_, Self>) {
- -/// // ...
- -/// }
- -/// }
- -///
- -/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
- -/// obj.as_arc_borrow().use_reference();
- -/// # Ok::<(), Error>(())
- -/// ```
- -pub struct ArcBorrow<'a, T: ?Sized + 'a> {
- - inner: NonNull<ArcInner<T>>,
- - _p: PhantomData<&'a ()>,
- -}
- -
- -// This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`.
- -impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {}
- -
- -// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
- -// `ArcBorrow<U>`.
- -impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
- - for ArcBorrow<'_, T>
- -{
- -}
- -
- -impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
- - fn clone(&self) -> Self {
- - *self
- - }
- -}
- -
- -impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
- -
- -impl<T: ?Sized> ArcBorrow<'_, T> {
- - /// Creates a new [`ArcBorrow`] instance.
- - ///
- - /// # Safety
- - ///
- - /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
- - /// 1. That `inner` remains valid;
- - /// 2. That no mutable references to `inner` are created.
- - unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
- - // INVARIANT: The safety requirements guarantee the invariants.
- - Self {
- - inner,
- - _p: PhantomData,
- - }
- - }
- -
- - /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
- - /// [`Arc::into_raw`].
- - ///
- - /// # Safety
- - ///
- - /// * The provided pointer must originate from a call to [`Arc::into_raw`].
- - /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
- - /// not hit zero.
- - /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
- - /// [`UniqueArc`] reference to this value.
- - pub unsafe fn from_raw(ptr: *const T) -> Self {
- - // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
- - // `Arc` that is still valid.
- - let ptr = unsafe { ArcInner::container_of(ptr) };
- -
- - // SAFETY: The caller promises that the value remains valid since the reference count must
- - // not hit zero, and no mutable reference will be created since that would involve a
- - // `UniqueArc`.
- - unsafe { Self::new(ptr) }
- - }
- -}
- -
- -impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
- - fn from(b: ArcBorrow<'_, T>) -> Self {
- - // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
- - // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
- - // increment.
- - ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
- - .deref()
- - .clone()
- - }
- -}
- -
- -impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
- - type Target = T;
- -
- - fn deref(&self) -> &Self::Target {
- - // SAFETY: By the type invariant, the underlying object is still alive with no mutable
- - // references to it, so it is safe to create a shared reference.
- - unsafe { &self.inner.as_ref().data }
- - }
- -}
- -
- -/// A refcounted object that is known to have a refcount of 1.
- -///
- -/// It is mutable and can be converted to an [`Arc`] so that it can be shared.
- -///
- -/// # Invariants
- -///
- -/// `inner` always has a reference count of 1.
- -///
- -/// # Examples
- -///
- -/// In the following example, we make changes to the inner object before turning it into an
- -/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
- -/// cannot fail.
- -///
- -/// ```
- -/// use kernel::sync::{Arc, UniqueArc};
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// fn test() -> Result<Arc<Example>> {
- -/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
- -/// x.a += 1;
- -/// x.b += 1;
- -/// Ok(x.into())
- -/// }
- -///
- -/// # test().unwrap();
- -/// ```
- -///
- -/// In the following example we first allocate memory for a refcounted `Example` but we don't
- -/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
- -/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
- -/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
- -///
- -/// ```
- -/// use kernel::sync::{Arc, UniqueArc};
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// fn test() -> Result<Arc<Example>> {
- -/// let x = UniqueArc::new_uninit(GFP_KERNEL)?;
- -/// Ok(x.write(Example { a: 10, b: 20 }).into())
- -/// }
- -///
- -/// # test().unwrap();
- -/// ```
- -///
- -/// In the last example below, the caller gets a pinned instance of `Example` while converting to
- -/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
- -/// initialisation, for example, when initialising fields that are wrapped in locks.
- -///
- -/// ```
- -/// use kernel::sync::{Arc, UniqueArc};
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// fn test() -> Result<Arc<Example>> {
- -/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);
- -/// // We can modify `pinned` because it is `Unpin`.
- -/// pinned.as_mut().a += 1;
- -/// Ok(pinned.into())
- -/// }
- -///
- -/// # test().unwrap();
- -/// ```
- -pub struct UniqueArc<T: ?Sized> {
- - inner: Arc<T>,
- -}
- -
- -impl<T> UniqueArc<T> {
- - /// Tries to allocate a new [`UniqueArc`] instance.
- - pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
- - Ok(Self {
- - // INVARIANT: The newly-created object has a refcount of 1.
- - inner: Arc::new(value, flags)?,
- - })
- - }
- -
- - /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
- - pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
- - // INVARIANT: The refcount is initialised to a non-zero value.
- - let inner = Box::try_init::<AllocError>(
- - try_init!(ArcInner {
- - // SAFETY: There are no safety requirements for this FFI call.
- - refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
- - data <- init::uninit::<T, AllocError>(),
- - }? AllocError),
- - flags,
- - )?;
- - Ok(UniqueArc {
- - // INVARIANT: The newly-created object has a refcount of 1.
- - // SAFETY: The pointer from the `Box` is valid.
- - inner: unsafe { Arc::from_inner(Box::leak(inner).into()) },
- - })
- - }
- -}
- -
- -impl<T> UniqueArc<MaybeUninit<T>> {
- - /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
- - pub fn write(mut self, value: T) -> UniqueArc<T> {
- - self.deref_mut().write(value);
- - // SAFETY: We just wrote the value to be initialized.
- - unsafe { self.assume_init() }
- - }
- -
- - /// Unsafely assume that `self` is initialized.
- - ///
- - /// # Safety
- - ///
- - /// The caller guarantees that the value behind this pointer has been initialized. It is
- - /// *immediate* UB to call this when the value is not initialized.
- - pub unsafe fn assume_init(self) -> UniqueArc<T> {
- - let inner = ManuallyDrop::new(self).inner.ptr;
- - UniqueArc {
- - // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
- - // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
- - inner: unsafe { Arc::from_inner(inner.cast()) },
- - }
- - }
- -
- - /// Initialize `self` using the given initializer.
- - pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
- - // SAFETY: The supplied pointer is valid for initialization.
- - match unsafe { init.__init(self.as_mut_ptr()) } {
- - // SAFETY: Initialization completed successfully.
- - Ok(()) => Ok(unsafe { self.assume_init() }),
- - Err(err) => Err(err),
- - }
- - }
- -
- - /// Pin-initialize `self` using the given pin-initializer.
- - pub fn pin_init_with<E>(
- - mut self,
- - init: impl PinInit<T, E>,
- - ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
- - // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
- - // to ensure it does not move.
- - match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
- - // SAFETY: Initialization completed successfully.
- - Ok(()) => Ok(unsafe { self.assume_init() }.into()),
- - Err(err) => Err(err),
- - }
- - }
- -}
- -
- -impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
- - fn from(obj: UniqueArc<T>) -> Self {
- - // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
- - // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
- - unsafe { Pin::new_unchecked(obj) }
- - }
- -}
- -
- -impl<T: ?Sized> Deref for UniqueArc<T> {
- - type Target = T;
- -
- - fn deref(&self) -> &Self::Target {
- - self.inner.deref()
- - }
- -}
- -
- -impl<T: ?Sized> DerefMut for UniqueArc<T> {
- - fn deref_mut(&mut self) -> &mut Self::Target {
- - // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
- - // it is safe to dereference it. Additionally, we know there is only one reference when
- - // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
- - unsafe { &mut self.inner.ptr.as_mut().data }
- - }
- -}
- -
- -impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - fmt::Display::fmt(self.deref(), f)
- - }
- -}
- -
- -impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - fmt::Display::fmt(self.deref(), f)
- - }
- -}
- -
- -impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - fmt::Debug::fmt(self.deref(), f)
- - }
- -}
- -
- -impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
- - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- - fmt::Debug::fmt(self.deref(), f)
- - }
- -}
- diff --git a/rust/kernel/sync/arc/std_vendor.rs b/rust/kernel/sync/arc/std_vendor.rs
- deleted file mode 100644
- --- a/rust/kernel/sync/arc/std_vendor.rs
- +++ /dev/null
- @@ -1,28 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -//! The contents of this file come from the Rust standard library, hosted in
- -//! the <https://github.com/rust-lang/rust> repository, licensed under
- -//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details,
- -//! see <https://github.com/rust-lang/rust/blob/master/COPYRIGHT>.
- -
- -use crate::sync::{arc::ArcInner, Arc};
- -use core::any::Any;
- -
- -impl Arc<dyn Any + Send + Sync> {
- - /// Attempt to downcast the `Arc<dyn Any + Send + Sync>` to a concrete type.
- - pub fn downcast<T>(self) -> core::result::Result<Arc<T>, Self>
- - where
- - T: Any + Send + Sync,
- - {
- - if (*self).is::<T>() {
- - // SAFETY: We have just checked that the type is correct, so we can cast the pointer.
- - unsafe {
- - let ptr = self.ptr.cast::<ArcInner<T>>();
- - core::mem::forget(self);
- - Ok(Arc::from_inner(ptr))
- - }
- - } else {
- - Err(self)
- - }
- - }
- -}
- diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs
- deleted file mode 100644
- --- a/rust/kernel/sync/condvar.rs
- +++ /dev/null
- @@ -1,238 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! A condition variable.
- -//!
- -//! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition
- -//! variable.
- -
- -use super::{lock::Backend, lock::Guard, LockClassKey};
- -use crate::{
- - init::PinInit,
- - pin_init,
- - str::CStr,
- - task::{MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE},
- - time::Jiffies,
- - types::Opaque,
- -};
- -use core::ffi::{c_int, c_long};
- -use core::marker::PhantomPinned;
- -use core::ptr;
- -use macros::pin_data;
- -
- -/// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class.
- -#[macro_export]
- -macro_rules! new_condvar {
- - ($($name:literal)?) => {
- - $crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!())
- - };
- -}
- -pub use new_condvar;
- -
- -/// A conditional variable.
- -///
- -/// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to
- -/// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And
- -/// it wakes up when notified by another thread (via [`CondVar::notify_one`] or
- -/// [`CondVar::notify_all`]) or because the thread received a signal. It may also wake up
- -/// spuriously.
- -///
- -/// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such
- -/// instances is with the [`pin_init`](crate::pin_init) and [`new_condvar`] macros.
- -///
- -/// # Examples
- -///
- -/// The following is an example of using a condvar with a mutex:
- -///
- -/// ```
- -/// use kernel::sync::{new_condvar, new_mutex, CondVar, Mutex};
- -///
- -/// #[pin_data]
- -/// pub struct Example {
- -/// #[pin]
- -/// value: Mutex<u32>,
- -///
- -/// #[pin]
- -/// value_changed: CondVar,
- -/// }
- -///
- -/// /// Waits for `e.value` to become `v`.
- -/// fn wait_for_value(e: &Example, v: u32) {
- -/// let mut guard = e.value.lock();
- -/// while *guard != v {
- -/// e.value_changed.wait(&mut guard);
- -/// }
- -/// }
- -///
- -/// /// Increments `e.value` and notifies all potential waiters.
- -/// fn increment(e: &Example) {
- -/// *e.value.lock() += 1;
- -/// e.value_changed.notify_all();
- -/// }
- -///
- -/// /// Allocates a new boxed `Example`.
- -/// fn new_example() -> Result<Pin<Box<Example>>> {
- -/// Box::pin_init(pin_init!(Example {
- -/// value <- new_mutex!(0),
- -/// value_changed <- new_condvar!(),
- -/// }), GFP_KERNEL)
- -/// }
- -/// ```
- -///
- -/// [`struct wait_queue_head`]: srctree/include/linux/wait.h
- -#[pin_data]
- -pub struct CondVar {
- - #[pin]
- - pub(crate) wait_queue_head: Opaque<bindings::wait_queue_head>,
- -
- - /// A condvar needs to be pinned because it contains a [`struct list_head`] that is
- - /// self-referential, so it cannot be safely moved once it is initialised.
- - ///
- - /// [`struct list_head`]: srctree/include/linux/types.h
- - #[pin]
- - _pin: PhantomPinned,
- -}
- -
- -// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread.
- -#[allow(clippy::non_send_fields_in_send_ty)]
- -unsafe impl Send for CondVar {}
- -
- -// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads
- -// concurrently.
- -unsafe impl Sync for CondVar {}
- -
- -impl CondVar {
- - /// Constructs a new condvar initialiser.
- - pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> {
- - pin_init!(Self {
- - _pin: PhantomPinned,
- - // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have
- - // static lifetimes so they live indefinitely.
- - wait_queue_head <- Opaque::ffi_init(|slot| unsafe {
- - bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr())
- - }),
- - })
- - }
- -
- - fn wait_internal<T: ?Sized, B: Backend>(
- - &self,
- - wait_state: c_int,
- - guard: &mut Guard<'_, T, B>,
- - timeout_in_jiffies: c_long,
- - ) -> c_long {
- - let wait = Opaque::<bindings::wait_queue_entry>::uninit();
- -
- - // SAFETY: `wait` points to valid memory.
- - unsafe { bindings::init_wait(wait.get()) };
- -
- - // SAFETY: Both `wait` and `wait_queue_head` point to valid memory.
- - unsafe {
- - bindings::prepare_to_wait_exclusive(self.wait_queue_head.get(), wait.get(), wait_state)
- - };
- -
- - // SAFETY: Switches to another thread. The timeout can be any number.
- - let ret = guard.do_unlocked(|| unsafe { bindings::schedule_timeout(timeout_in_jiffies) });
- -
- - // SAFETY: Both `wait` and `wait_queue_head` point to valid memory.
- - unsafe { bindings::finish_wait(self.wait_queue_head.get(), wait.get()) };
- -
- - ret
- - }
- -
- - /// Releases the lock and waits for a notification in uninterruptible mode.
- - ///
- - /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the
- - /// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by
- - /// [`CondVar::notify_one`] or [`CondVar::notify_all`]. Note that it may also wake up
- - /// spuriously.
- - pub fn wait<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) {
- - self.wait_internal(TASK_UNINTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT);
- - }
- -
- - /// Releases the lock and waits for a notification in interruptible mode.
- - ///
- - /// Similar to [`CondVar::wait`], except that the wait is interruptible. That is, the thread may
- - /// wake up due to signals. It may also wake up spuriously.
- - ///
- - /// Returns whether there is a signal pending.
- - #[must_use = "wait_interruptible returns if a signal is pending, so the caller must check the return value"]
- - pub fn wait_interruptible<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) -> bool {
- - self.wait_internal(TASK_INTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT);
- - crate::current!().signal_pending()
- - }
- -
- - /// Releases the lock and waits for a notification in interruptible mode.
- - ///
- - /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the
- - /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or
- - /// [`CondVar::notify_all`], or when a timeout occurs, or when the thread receives a signal.
- - #[must_use = "wait_interruptible_timeout returns if a signal is pending, so the caller must check the return value"]
- - pub fn wait_interruptible_timeout<T: ?Sized, B: Backend>(
- - &self,
- - guard: &mut Guard<'_, T, B>,
- - jiffies: Jiffies,
- - ) -> CondVarTimeoutResult {
- - let jiffies = jiffies.try_into().unwrap_or(MAX_SCHEDULE_TIMEOUT);
- - let res = self.wait_internal(TASK_INTERRUPTIBLE, guard, jiffies);
- -
- - match (res as Jiffies, crate::current!().signal_pending()) {
- - (jiffies, true) => CondVarTimeoutResult::Signal { jiffies },
- - (0, false) => CondVarTimeoutResult::Timeout,
- - (jiffies, false) => CondVarTimeoutResult::Woken { jiffies },
- - }
- - }
- -
- - /// Calls the kernel function to notify the appropriate number of threads.
- - fn notify(&self, count: c_int) {
- - // SAFETY: `wait_queue_head` points to valid memory.
- - unsafe {
- - bindings::__wake_up(
- - self.wait_queue_head.get(),
- - TASK_NORMAL,
- - count,
- - ptr::null_mut(),
- - )
- - };
- - }
- -
- - /// Calls the kernel function to notify one thread synchronously.
- - ///
- - /// This method behaves like `notify_one`, except that it hints to the scheduler that the
- - /// current thread is about to go to sleep, so it should schedule the target thread on the same
- - /// CPU.
- - pub fn notify_sync(&self) {
- - // SAFETY: `wait_queue_head` points to valid memory.
- - unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) };
- - }
- -
- - /// Wakes a single waiter up, if any.
- - ///
- - /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost
- - /// completely (as opposed to automatically waking up the next waiter).
- - pub fn notify_one(&self) {
- - self.notify(1);
- - }
- -
- - /// Wakes all waiters up, if any.
- - ///
- - /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost
- - /// completely (as opposed to automatically waking up the next waiter).
- - pub fn notify_all(&self) {
- - self.notify(0);
- - }
- -}
- -
- -/// The return type of `wait_timeout`.
- -pub enum CondVarTimeoutResult {
- - /// The timeout was reached.
- - Timeout,
- - /// Somebody woke us up.
- - Woken {
- - /// Remaining sleep duration.
- - jiffies: Jiffies,
- - },
- - /// A signal occurred.
- - Signal {
- - /// Remaining sleep duration.
- - jiffies: Jiffies,
- - },
- -}
- diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs
- deleted file mode 100644
- --- a/rust/kernel/sync/lock.rs
- +++ /dev/null
- @@ -1,197 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Generic kernel lock and guard.
- -//!
- -//! It contains a generic Rust lock and guard that allow for different backends (e.g., mutexes,
- -//! spinlocks, raw spinlocks) to be provided with minimal effort.
- -
- -use super::LockClassKey;
- -use crate::{init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard};
- -use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned};
- -use macros::pin_data;
- -
- -pub mod mutex;
- -pub mod spinlock;
- -
- -/// The "backend" of a lock.
- -///
- -/// It is the actual implementation of the lock, without the need to repeat patterns used in all
- -/// locks.
- -///
- -/// # Safety
- -///
- -/// - Implementers must ensure that only one thread/CPU may access the protected data once the lock
- -/// is owned, that is, between calls to [`lock`] and [`unlock`].
- -/// - Implementers must also ensure that [`relock`] uses the same locking method as the original
- -/// lock operation.
- -///
- -/// [`lock`]: Backend::lock
- -/// [`unlock`]: Backend::unlock
- -/// [`relock`]: Backend::relock
- -pub unsafe trait Backend {
- - /// The state required by the lock.
- - type State;
- -
- - /// The state required to be kept between [`lock`] and [`unlock`].
- - ///
- - /// [`lock`]: Backend::lock
- - /// [`unlock`]: Backend::unlock
- - type GuardState;
- -
- - /// Initialises the lock.
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must be valid for write for the duration of the call, while `name` and `key` must
- - /// remain valid for read indefinitely.
- - unsafe fn init(
- - ptr: *mut Self::State,
- - name: *const core::ffi::c_char,
- - key: *mut bindings::lock_class_key,
- - );
- -
- - /// Acquires the lock, making the caller its owner.
- - ///
- - /// # Safety
- - ///
- - /// Callers must ensure that [`Backend::init`] has been previously called.
- - #[must_use]
- - unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState;
- -
- - /// Releases the lock, giving up its ownership.
- - ///
- - /// # Safety
- - ///
- - /// It must only be called by the current owner of the lock.
- - unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState);
- -
- - /// Reacquires the lock, making the caller its owner.
- - ///
- - /// # Safety
- - ///
- - /// Callers must ensure that `guard_state` comes from a previous call to [`Backend::lock`] (or
- - /// variant) that has been unlocked with [`Backend::unlock`] and will be relocked now.
- - unsafe fn relock(ptr: *mut Self::State, guard_state: &mut Self::GuardState) {
- - // SAFETY: The safety requirements ensure that the lock is initialised.
- - *guard_state = unsafe { Self::lock(ptr) };
- - }
- -}
- -
- -/// A mutual exclusion primitive.
- -///
- -/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock
- -/// [`Backend`] specified as the generic parameter `B`.
- -#[pin_data]
- -pub struct Lock<T: ?Sized, B: Backend> {
- - /// The kernel lock object.
- - #[pin]
- - state: Opaque<B::State>,
- -
- - /// Some locks are known to be self-referential (e.g., mutexes), while others are architecture
- - /// or config defined (e.g., spinlocks). So we conservatively require them to be pinned in case
- - /// some architecture uses self-references now or in the future.
- - #[pin]
- - _pin: PhantomPinned,
- -
- - /// The data protected by the lock.
- - pub(crate) data: UnsafeCell<T>,
- -}
- -
- -// SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can.
- -unsafe impl<T: ?Sized + Send, B: Backend> Send for Lock<T, B> {}
- -
- -// SAFETY: `Lock` serialises the interior mutability it provides, so it is `Sync` as long as the
- -// data it protects is `Send`.
- -unsafe impl<T: ?Sized + Send, B: Backend> Sync for Lock<T, B> {}
- -
- -impl<T, B: Backend> Lock<T, B> {
- - /// Constructs a new lock initialiser.
- - pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> {
- - pin_init!(Self {
- - data: UnsafeCell::new(t),
- - _pin: PhantomPinned,
- - // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have
- - // static lifetimes so they live indefinitely.
- - state <- Opaque::ffi_init(|slot| unsafe {
- - B::init(slot, name.as_char_ptr(), key.as_ptr())
- - }),
- - })
- - }
- -}
- -
- -impl<T: ?Sized, B: Backend> Lock<T, B> {
- - /// Acquires the lock and gives the caller access to the data protected by it.
- - pub fn lock(&self) -> Guard<'_, T, B> {
- - // SAFETY: The constructor of the type calls `init`, so the existence of the object proves
- - // that `init` was called.
- - let state = unsafe { B::lock(self.state.get()) };
- - // SAFETY: The lock was just acquired.
- - unsafe { Guard::new(self, state) }
- - }
- -}
- -
- -/// A lock guard.
- -///
- -/// Allows mutual exclusion primitives that implement the [`Backend`] trait to automatically unlock
- -/// when a guard goes out of scope. It also provides a safe and convenient way to access the data
- -/// protected by the lock.
- -#[must_use = "the lock unlocks immediately when the guard is unused"]
- -pub struct Guard<'a, T: ?Sized, B: Backend> {
- - pub(crate) lock: &'a Lock<T, B>,
- - pub(crate) state: B::GuardState,
- - _not_send: PhantomData<*mut ()>,
- -}
- -
- -// SAFETY: `Guard` is sync when the data protected by the lock is also sync.
- -unsafe impl<T: Sync + ?Sized, B: Backend> Sync for Guard<'_, T, B> {}
- -
- -impl<T: ?Sized, B: Backend> Guard<'_, T, B> {
- - pub(crate) fn do_unlocked<U>(&mut self, cb: impl FnOnce() -> U) -> U {
- - // SAFETY: The caller owns the lock, so it is safe to unlock it.
- - unsafe { B::unlock(self.lock.state.get(), &self.state) };
- -
- - // SAFETY: The lock was just unlocked above and is being relocked now.
- - let _relock =
- - ScopeGuard::new(|| unsafe { B::relock(self.lock.state.get(), &mut self.state) });
- -
- - cb()
- - }
- -}
- -
- -impl<T: ?Sized, B: Backend> core::ops::Deref for Guard<'_, T, B> {
- - type Target = T;
- -
- - fn deref(&self) -> &Self::Target {
- - // SAFETY: The caller owns the lock, so it is safe to deref the protected data.
- - unsafe { &*self.lock.data.get() }
- - }
- -}
- -
- -impl<T: ?Sized, B: Backend> core::ops::DerefMut for Guard<'_, T, B> {
- - fn deref_mut(&mut self) -> &mut Self::Target {
- - // SAFETY: The caller owns the lock, so it is safe to deref the protected data.
- - unsafe { &mut *self.lock.data.get() }
- - }
- -}
- -
- -impl<T: ?Sized, B: Backend> Drop for Guard<'_, T, B> {
- - fn drop(&mut self) {
- - // SAFETY: The caller owns the lock, so it is safe to unlock it.
- - unsafe { B::unlock(self.lock.state.get(), &self.state) };
- - }
- -}
- -
- -impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> {
- - /// Constructs a new immutable lock guard.
- - ///
- - /// # Safety
- - ///
- - /// The caller must ensure that it owns the lock.
- - pub(crate) unsafe fn new(lock: &'a Lock<T, B>, state: B::GuardState) -> Self {
- - Self {
- - lock,
- - state,
- - _not_send: PhantomData,
- - }
- - }
- -}
- diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs
- deleted file mode 100644
- --- a/rust/kernel/sync/lock/mutex.rs
- +++ /dev/null
- @@ -1,118 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! A kernel mutex.
- -//!
- -//! This module allows Rust code to use the kernel's `struct mutex`.
- -
- -/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class.
- -///
- -/// It uses the name if one is given, otherwise it generates one based on the file name and line
- -/// number.
- -#[macro_export]
- -macro_rules! new_mutex {
- - ($inner:expr $(, $name:literal)? $(,)?) => {
- - $crate::sync::Mutex::new(
- - $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
- - };
- -}
- -pub use new_mutex;
- -
- -/// A mutual exclusion primitive.
- -///
- -/// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex,
- -/// only one at a time is allowed to progress, the others will block (sleep) until the mutex is
- -/// unlocked, at which point another thread will be allowed to wake up and make progress.
- -///
- -/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts.
- -///
- -/// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such
- -/// instances is with the [`pin_init`](crate::pin_init) and [`new_mutex`] macros.
- -///
- -/// # Examples
- -///
- -/// The following example shows how to declare, allocate and initialise a struct (`Example`) that
- -/// contains an inner struct (`Inner`) that is protected by a mutex.
- -///
- -/// ```
- -/// use kernel::sync::{new_mutex, Mutex};
- -///
- -/// struct Inner {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// #[pin_data]
- -/// struct Example {
- -/// c: u32,
- -/// #[pin]
- -/// d: Mutex<Inner>,
- -/// }
- -///
- -/// impl Example {
- -/// fn new() -> impl PinInit<Self> {
- -/// pin_init!(Self {
- -/// c: 10,
- -/// d <- new_mutex!(Inner { a: 20, b: 30 }),
- -/// })
- -/// }
- -/// }
- -///
- -/// // Allocate a boxed `Example`.
- -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?;
- -/// assert_eq!(e.c, 10);
- -/// assert_eq!(e.d.lock().a, 20);
- -/// assert_eq!(e.d.lock().b, 30);
- -/// # Ok::<(), Error>(())
- -/// ```
- -///
- -/// The following example shows how to use interior mutability to modify the contents of a struct
- -/// protected by a mutex despite only having a shared reference:
- -///
- -/// ```
- -/// use kernel::sync::Mutex;
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// fn example(m: &Mutex<Example>) {
- -/// let mut guard = m.lock();
- -/// guard.a += 10;
- -/// guard.b += 20;
- -/// }
- -/// ```
- -///
- -/// [`struct mutex`]: srctree/include/linux/mutex.h
- -pub type Mutex<T> = super::Lock<T, MutexBackend>;
- -
- -/// A kernel `struct mutex` lock backend.
- -pub struct MutexBackend;
- -
- -// SAFETY: The underlying kernel `struct mutex` object ensures mutual exclusion.
- -unsafe impl super::Backend for MutexBackend {
- - type State = bindings::mutex;
- - type GuardState = ();
- -
- - unsafe fn init(
- - ptr: *mut Self::State,
- - name: *const core::ffi::c_char,
- - key: *mut bindings::lock_class_key,
- - ) {
- - // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
- - // `key` are valid for read indefinitely.
- - unsafe { bindings::__mutex_init(ptr, name, key) }
- - }
- -
- - unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
- - // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
- - // memory, and that it has been initialised before.
- - unsafe { bindings::mutex_lock(ptr) };
- - }
- -
- - unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
- - // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
- - // caller is the owner of the mutex.
- - unsafe { bindings::mutex_unlock(ptr) };
- - }
- -}
- diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs
- deleted file mode 100644
- --- a/rust/kernel/sync/lock/spinlock.rs
- +++ /dev/null
- @@ -1,117 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! A kernel spinlock.
- -//!
- -//! This module allows Rust code to use the kernel's `spinlock_t`.
- -
- -/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
- -///
- -/// It uses the name if one is given, otherwise it generates one based on the file name and line
- -/// number.
- -#[macro_export]
- -macro_rules! new_spinlock {
- - ($inner:expr $(, $name:literal)? $(,)?) => {
- - $crate::sync::SpinLock::new(
- - $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
- - };
- -}
- -pub use new_spinlock;
- -
- -/// A spinlock.
- -///
- -/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only
- -/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is
- -/// unlocked, at which point another CPU will be allowed to make progress.
- -///
- -/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such
- -/// instances is with the [`pin_init`](crate::pin_init) and [`new_spinlock`] macros.
- -///
- -/// # Examples
- -///
- -/// The following example shows how to declare, allocate and initialise a struct (`Example`) that
- -/// contains an inner struct (`Inner`) that is protected by a spinlock.
- -///
- -/// ```
- -/// use kernel::sync::{new_spinlock, SpinLock};
- -///
- -/// struct Inner {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// #[pin_data]
- -/// struct Example {
- -/// c: u32,
- -/// #[pin]
- -/// d: SpinLock<Inner>,
- -/// }
- -///
- -/// impl Example {
- -/// fn new() -> impl PinInit<Self> {
- -/// pin_init!(Self {
- -/// c: 10,
- -/// d <- new_spinlock!(Inner { a: 20, b: 30 }),
- -/// })
- -/// }
- -/// }
- -///
- -/// // Allocate a boxed `Example`.
- -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?;
- -/// assert_eq!(e.c, 10);
- -/// assert_eq!(e.d.lock().a, 20);
- -/// assert_eq!(e.d.lock().b, 30);
- -/// # Ok::<(), Error>(())
- -/// ```
- -///
- -/// The following example shows how to use interior mutability to modify the contents of a struct
- -/// protected by a spinlock despite only having a shared reference:
- -///
- -/// ```
- -/// use kernel::sync::SpinLock;
- -///
- -/// struct Example {
- -/// a: u32,
- -/// b: u32,
- -/// }
- -///
- -/// fn example(m: &SpinLock<Example>) {
- -/// let mut guard = m.lock();
- -/// guard.a += 10;
- -/// guard.b += 20;
- -/// }
- -/// ```
- -///
- -/// [`spinlock_t`]: srctree/include/linux/spinlock.h
- -pub type SpinLock<T> = super::Lock<T, SpinLockBackend>;
- -
- -/// A kernel `spinlock_t` lock backend.
- -pub struct SpinLockBackend;
- -
- -// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
- -// default implementation that always calls the same locking method.
- -unsafe impl super::Backend for SpinLockBackend {
- - type State = bindings::spinlock_t;
- - type GuardState = ();
- -
- - unsafe fn init(
- - ptr: *mut Self::State,
- - name: *const core::ffi::c_char,
- - key: *mut bindings::lock_class_key,
- - ) {
- - // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
- - // `key` are valid for read indefinitely.
- - unsafe { bindings::__spin_lock_init(ptr, name, key) }
- - }
- -
- - unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
- - // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
- - // memory, and that it has been initialised before.
- - unsafe { bindings::spin_lock(ptr) }
- - }
- -
- - unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
- - // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
- - // caller is the owner of the spinlock.
- - unsafe { bindings::spin_unlock(ptr) }
- - }
- -}
- diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs
- deleted file mode 100644
- --- a/rust/kernel/sync/locked_by.rs
- +++ /dev/null
- @@ -1,159 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! A wrapper for data protected by a lock that does not wrap it.
- -
- -use super::{lock::Backend, lock::Lock};
- -use crate::build_assert;
- -use core::{cell::UnsafeCell, mem::size_of, ptr};
- -
- -/// Allows access to some data to be serialised by a lock that does not wrap it.
- -///
- -/// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g.,
- -/// [`Mutex`] or [`SpinLock`]. [`LockedBy`] is meant for cases when this is not possible.
- -/// For example, if a container has a lock and some data in the contained elements needs
- -/// to be protected by the same lock.
- -///
- -/// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it
- -/// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence
- -/// refers to the wrong instance of the lock.
- -///
- -/// [`Mutex`]: super::Mutex
- -/// [`SpinLock`]: super::SpinLock
- -///
- -/// # Examples
- -///
- -/// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an
- -/// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in
- -/// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at
- -/// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also
- -/// locked; we enforce at run time that the right `InnerDirectory` is locked.
- -///
- -/// ```
- -/// use kernel::sync::{LockedBy, Mutex};
- -///
- -/// struct InnerFile {
- -/// bytes_used: u64,
- -/// }
- -///
- -/// struct File {
- -/// _ino: u32,
- -/// inner: LockedBy<InnerFile, InnerDirectory>,
- -/// }
- -///
- -/// struct InnerDirectory {
- -/// /// The sum of the bytes used by all files.
- -/// bytes_used: u64,
- -/// _files: Vec<File>,
- -/// }
- -///
- -/// struct Directory {
- -/// _ino: u32,
- -/// inner: Mutex<InnerDirectory>,
- -/// }
- -///
- -/// /// Prints `bytes_used` from both the directory and file.
- -/// fn print_bytes_used(dir: &Directory, file: &File) {
- -/// let guard = dir.inner.lock();
- -/// let inner_file = file.inner.access(&guard);
- -/// pr_info!("{} {}", guard.bytes_used, inner_file.bytes_used);
- -/// }
- -///
- -/// /// Increments `bytes_used` for both the directory and file.
- -/// fn inc_bytes_used(dir: &Directory, file: &File) {
- -/// let mut guard = dir.inner.lock();
- -/// guard.bytes_used += 10;
- -///
- -/// let file_inner = file.inner.access_mut(&mut guard);
- -/// file_inner.bytes_used += 10;
- -/// }
- -///
- -/// /// Creates a new file.
- -/// fn new_file(ino: u32, dir: &Directory) -> File {
- -/// File {
- -/// _ino: ino,
- -/// inner: LockedBy::new(&dir.inner, InnerFile { bytes_used: 0 }),
- -/// }
- -/// }
- -/// ```
- -pub struct LockedBy<T: ?Sized, U: ?Sized> {
- - owner: *const U,
- - data: UnsafeCell<T>,
- -}
- -
- -// SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can.
- -unsafe impl<T: ?Sized + Send, U: ?Sized> Send for LockedBy<T, U> {}
- -
- -// SAFETY: `LockedBy` serialises the interior mutability it provides, so it is `Sync` as long as the
- -// data it protects is `Send`.
- -unsafe impl<T: ?Sized + Send, U: ?Sized> Sync for LockedBy<T, U> {}
- -
- -impl<T, U> LockedBy<T, U> {
- - /// Constructs a new instance of [`LockedBy`].
- - ///
- - /// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure
- - /// that the right owner is being used to access the protected data. If the owner is freed, the
- - /// data becomes inaccessible; if another instance of the owner is allocated *on the same
- - /// memory location*, the data becomes accessible again: none of this affects memory safety
- - /// because in any case at most one thread (or CPU) can access the protected data at a time.
- - pub fn new<B: Backend>(owner: &Lock<U, B>, data: T) -> Self {
- - build_assert!(
- - size_of::<Lock<U, B>>() > 0,
- - "The lock type cannot be a ZST because it may be impossible to distinguish instances"
- - );
- - Self {
- - owner: owner.data.get(),
- - data: UnsafeCell::new(data),
- - }
- - }
- -}
- -
- -impl<T: ?Sized, U> LockedBy<T, U> {
- - /// Returns a reference to the protected data when the caller provides evidence (via a
- - /// reference) that the owner is locked.
- - ///
- - /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&U` that matches
- - /// the data protected by the lock without actually holding it.
- - ///
- - /// # Panics
- - ///
- - /// Panics if `owner` is different from the data protected by the lock used in
- - /// [`new`](LockedBy::new).
- - pub fn access<'a>(&'a self, owner: &'a U) -> &'a T {
- - build_assert!(
- - size_of::<U>() > 0,
- - "`U` cannot be a ZST because `owner` wouldn't be unique"
- - );
- - if !ptr::eq(owner, self.owner) {
- - panic!("mismatched owners");
- - }
- -
- - // SAFETY: `owner` is evidence that the owner is locked.
- - unsafe { &*self.data.get() }
- - }
- -
- - /// Returns a mutable reference to the protected data when the caller provides evidence (via a
- - /// mutable owner) that the owner is locked mutably.
- - ///
- - /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&mut U` that
- - /// matches the data protected by the lock without actually holding it.
- - ///
- - /// Showing a mutable reference to the owner is sufficient because we know no other references
- - /// can exist to it.
- - ///
- - /// # Panics
- - ///
- - /// Panics if `owner` is different from the data protected by the lock used in
- - /// [`new`](LockedBy::new).
- - pub fn access_mut<'a>(&'a self, owner: &'a mut U) -> &'a mut T {
- - build_assert!(
- - size_of::<U>() > 0,
- - "`U` cannot be a ZST because `owner` wouldn't be unique"
- - );
- - if !ptr::eq(owner, self.owner) {
- - panic!("mismatched owners");
- - }
- -
- - // SAFETY: `owner` is evidence that there is only one reference to the owner.
- - unsafe { &mut *self.data.get() }
- - }
- -}
- diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs
- deleted file mode 100644
- --- a/rust/kernel/task.rs
- +++ /dev/null
- @@ -1,177 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Tasks (threads and processes).
- -//!
- -//! C header: [`include/linux/sched.h`](srctree/include/linux/sched.h).
- -
- -use crate::types::Opaque;
- -use core::{
- - ffi::{c_int, c_long, c_uint},
- - marker::PhantomData,
- - ops::Deref,
- - ptr,
- -};
- -
- -/// A sentinel value used for infinite timeouts.
- -pub const MAX_SCHEDULE_TIMEOUT: c_long = c_long::MAX;
- -
- -/// Bitmask for tasks that are sleeping in an interruptible state.
- -pub const TASK_INTERRUPTIBLE: c_int = bindings::TASK_INTERRUPTIBLE as c_int;
- -/// Bitmask for tasks that are sleeping in an uninterruptible state.
- -pub const TASK_UNINTERRUPTIBLE: c_int = bindings::TASK_UNINTERRUPTIBLE as c_int;
- -/// Convenience constant for waking up tasks regardless of whether they are in interruptible or
- -/// uninterruptible sleep.
- -pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint;
- -
- -/// Returns the currently running task.
- -#[macro_export]
- -macro_rules! current {
- - () => {
- - // SAFETY: Deref + addr-of below create a temporary `TaskRef` that cannot outlive the
- - // caller.
- - unsafe { &*$crate::task::Task::current() }
- - };
- -}
- -
- -/// Wraps the kernel's `struct task_struct`.
- -///
- -/// # Invariants
- -///
- -/// All instances are valid tasks created by the C portion of the kernel.
- -///
- -/// Instances of this type are always refcounted, that is, a call to `get_task_struct` ensures
- -/// that the allocation remains valid at least until the matching call to `put_task_struct`.
- -///
- -/// # Examples
- -///
- -/// The following is an example of getting the PID of the current thread with zero additional cost
- -/// when compared to the C version:
- -///
- -/// ```
- -/// let pid = current!().pid();
- -/// ```
- -///
- -/// Getting the PID of the current process, also zero additional cost:
- -///
- -/// ```
- -/// let pid = current!().group_leader().pid();
- -/// ```
- -///
- -/// Getting the current task and storing it in some struct. The reference count is automatically
- -/// incremented when creating `State` and decremented when it is dropped:
- -///
- -/// ```
- -/// use kernel::{task::Task, types::ARef};
- -///
- -/// struct State {
- -/// creator: ARef<Task>,
- -/// index: u32,
- -/// }
- -///
- -/// impl State {
- -/// fn new() -> Self {
- -/// Self {
- -/// creator: current!().into(),
- -/// index: 0,
- -/// }
- -/// }
- -/// }
- -/// ```
- -#[repr(transparent)]
- -pub struct Task(pub(crate) Opaque<bindings::task_struct>);
- -
- -// SAFETY: By design, the only way to access a `Task` is via the `current` function or via an
- -// `ARef<Task>` obtained through the `AlwaysRefCounted` impl. This means that the only situation in
- -// which a `Task` can be accessed mutably is when the refcount drops to zero and the destructor
- -// runs. It is safe for that to happen on any thread, so it is ok for this type to be `Send`.
- -unsafe impl Send for Task {}
- -
- -// SAFETY: It's OK to access `Task` through shared references from other threads because we're
- -// either accessing properties that don't change (e.g., `pid`, `group_leader`) or that are properly
- -// synchronised by C code (e.g., `signal_pending`).
- -unsafe impl Sync for Task {}
- -
- -/// The type of process identifiers (PIDs).
- -type Pid = bindings::pid_t;
- -
- -impl Task {
- - /// Returns a task reference for the currently executing task/thread.
- - ///
- - /// The recommended way to get the current task/thread is to use the
- - /// [`current`] macro because it is safe.
- - ///
- - /// # Safety
- - ///
- - /// Callers must ensure that the returned object doesn't outlive the current task/thread.
- - pub unsafe fn current() -> impl Deref<Target = Task> {
- - struct TaskRef<'a> {
- - task: &'a Task,
- - _not_send: PhantomData<*mut ()>,
- - }
- -
- - impl Deref for TaskRef<'_> {
- - type Target = Task;
- -
- - fn deref(&self) -> &Self::Target {
- - self.task
- - }
- - }
- -
- - // SAFETY: Just an FFI call with no additional safety requirements.
- - let ptr = unsafe { bindings::get_current() };
- -
- - TaskRef {
- - // SAFETY: If the current thread is still running, the current task is valid. Given
- - // that `TaskRef` is not `Send`, we know it cannot be transferred to another thread
- - // (where it could potentially outlive the caller).
- - task: unsafe { &*ptr.cast() },
- - _not_send: PhantomData,
- - }
- - }
- -
- - /// Returns the group leader of the given task.
- - pub fn group_leader(&self) -> &Task {
- - // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always
- - // have a valid `group_leader`.
- - let ptr = unsafe { *ptr::addr_of!((*self.0.get()).group_leader) };
- -
- - // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`,
- - // and given that a task has a reference to its group leader, we know it must be valid for
- - // the lifetime of the returned task reference.
- - unsafe { &*ptr.cast() }
- - }
- -
- - /// Returns the PID of the given task.
- - pub fn pid(&self) -> Pid {
- - // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always
- - // have a valid pid.
- - unsafe { *ptr::addr_of!((*self.0.get()).pid) }
- - }
- -
- - /// Determines whether the given task has pending signals.
- - pub fn signal_pending(&self) -> bool {
- - // SAFETY: By the type invariant, we know that `self.0` is valid.
- - unsafe { bindings::signal_pending(self.0.get()) != 0 }
- - }
- -
- - /// Wakes up the task.
- - pub fn wake_up(&self) {
- - // SAFETY: By the type invariant, we know that `self.0.get()` is non-null and valid.
- - // And `wake_up_process` is safe to be called for any valid task, even if the task is
- - // running.
- - unsafe { bindings::wake_up_process(self.0.get()) };
- - }
- -}
- -
- -// SAFETY: The type invariants guarantee that `Task` is always refcounted.
- -unsafe impl crate::types::AlwaysRefCounted for Task {
- - fn inc_ref(&self) {
- - // SAFETY: The existence of a shared reference means that the refcount is nonzero.
- - unsafe { bindings::get_task_struct(self.0.get()) };
- - }
- -
- - unsafe fn dec_ref(obj: ptr::NonNull<Self>) {
- - // SAFETY: The safety requirements guarantee that the refcount is nonzero.
- - unsafe { bindings::put_task_struct(obj.cast().as_ptr()) }
- - }
- -}
- diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs
- deleted file mode 100644
- --- a/rust/kernel/time.rs
- +++ /dev/null
- @@ -1,83 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Time related primitives.
- -//!
- -//! This module contains the kernel APIs related to time and timers that
- -//! have been ported or wrapped for usage by Rust code in the kernel.
- -//!
- -//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h).
- -//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h).
- -
- -/// The number of nanoseconds per millisecond.
- -pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64;
- -
- -/// The time unit of Linux kernel. One jiffy equals (1/HZ) second.
- -pub type Jiffies = core::ffi::c_ulong;
- -
- -/// The millisecond time unit.
- -pub type Msecs = core::ffi::c_uint;
- -
- -/// Converts milliseconds to jiffies.
- -#[inline]
- -pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies {
- - // SAFETY: The `__msecs_to_jiffies` function is always safe to call no
- - // matter what the argument is.
- - unsafe { bindings::__msecs_to_jiffies(msecs) }
- -}
- -
- -/// A Rust wrapper around a `ktime_t`.
- -#[repr(transparent)]
- -#[derive(Copy, Clone)]
- -pub struct Ktime {
- - inner: bindings::ktime_t,
- -}
- -
- -impl Ktime {
- - /// Create a `Ktime` from a raw `ktime_t`.
- - #[inline]
- - pub fn from_raw(inner: bindings::ktime_t) -> Self {
- - Self { inner }
- - }
- -
- - /// Get the current time using `CLOCK_MONOTONIC`.
- - #[inline]
- - pub fn ktime_get() -> Self {
- - // SAFETY: It is always safe to call `ktime_get` outside of NMI context.
- - Self::from_raw(unsafe { bindings::ktime_get() })
- - }
- -
- - /// Divide the number of nanoseconds by a compile-time constant.
- - #[inline]
- - fn divns_constant<const DIV: i64>(self) -> i64 {
- - self.to_ns() / DIV
- - }
- -
- - /// Returns the number of nanoseconds.
- - #[inline]
- - pub fn to_ns(self) -> i64 {
- - self.inner
- - }
- -
- - /// Returns the number of milliseconds.
- - #[inline]
- - pub fn to_ms(self) -> i64 {
- - self.divns_constant::<NSEC_PER_MSEC>()
- - }
- -}
- -
- -/// Returns the number of milliseconds between two ktimes.
- -#[inline]
- -pub fn ktime_ms_delta(later: Ktime, earlier: Ktime) -> i64 {
- - (later - earlier).to_ms()
- -}
- -
- -impl core::ops::Sub for Ktime {
- - type Output = Ktime;
- -
- - #[inline]
- - fn sub(self, other: Ktime) -> Ktime {
- - Self {
- - inner: self.inner - other.inner,
- - }
- - }
- -}
- diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs
- deleted file mode 100644
- --- a/rust/kernel/types.rs
- +++ /dev/null
- @@ -1,411 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Kernel types.
- -
- -use crate::init::{self, PinInit};
- -use alloc::boxed::Box;
- -use core::{
- - cell::UnsafeCell,
- - marker::{PhantomData, PhantomPinned},
- - mem::MaybeUninit,
- - ops::{Deref, DerefMut},
- - ptr::NonNull,
- -};
- -
- -/// Used to transfer ownership to and from foreign (non-Rust) languages.
- -///
- -/// Ownership is transferred from Rust to a foreign language by calling [`Self::into_foreign`] and
- -/// later may be transferred back to Rust by calling [`Self::from_foreign`].
- -///
- -/// This trait is meant to be used in cases when Rust objects are stored in C objects and
- -/// eventually "freed" back to Rust.
- -pub trait ForeignOwnable: Sized {
- - /// Type of values borrowed between calls to [`ForeignOwnable::into_foreign`] and
- - /// [`ForeignOwnable::from_foreign`].
- - type Borrowed<'a>;
- -
- - /// Converts a Rust-owned object to a foreign-owned one.
- - ///
- - /// The foreign representation is a pointer to void.
- - fn into_foreign(self) -> *const core::ffi::c_void;
- -
- - /// Borrows a foreign-owned object.
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must have been returned by a previous call to [`ForeignOwnable::into_foreign`] for
- - /// which a previous matching [`ForeignOwnable::from_foreign`] hasn't been called yet.
- - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Self::Borrowed<'a>;
- -
- - /// Converts a foreign-owned object back to a Rust-owned one.
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must have been returned by a previous call to [`ForeignOwnable::into_foreign`] for
- - /// which a previous matching [`ForeignOwnable::from_foreign`] hasn't been called yet.
- - /// Additionally, all instances (if any) of values returned by [`ForeignOwnable::borrow`] for
- - /// this object must have been dropped.
- - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self;
- -
- - /// Tries to convert a foreign-owned object back to a Rust-owned one.
- - ///
- - /// A convenience wrapper over [`ForeignOwnable::from_foreign`] that returns [`None`] if `ptr`
- - /// is null.
- - ///
- - /// # Safety
- - ///
- - /// `ptr` must either be null or satisfy the safety requirements for
- - /// [`ForeignOwnable::from_foreign`].
- - unsafe fn try_from_foreign(ptr: *const core::ffi::c_void) -> Option<Self> {
- - if ptr.is_null() {
- - None
- - } else {
- - // SAFETY: Since `ptr` is not null here, then `ptr` satisfies the safety requirements
- - // of `from_foreign` given the safety requirements of this function.
- - unsafe { Some(Self::from_foreign(ptr)) }
- - }
- - }
- -}
- -
- -impl<T: 'static> ForeignOwnable for Box<T> {
- - type Borrowed<'a> = &'a T;
- -
- - fn into_foreign(self) -> *const core::ffi::c_void {
- - Box::into_raw(self) as _
- - }
- -
- - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T {
- - // SAFETY: The safety requirements for this function ensure that the object is still alive,
- - // so it is safe to dereference the raw pointer.
- - // The safety requirements of `from_foreign` also ensure that the object remains alive for
- - // the lifetime of the returned value.
- - unsafe { &*ptr.cast() }
- - }
- -
- - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {
- - // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
- - // call to `Self::into_foreign`.
- - unsafe { Box::from_raw(ptr as _) }
- - }
- -}
- -
- -impl ForeignOwnable for () {
- - type Borrowed<'a> = ();
- -
- - fn into_foreign(self) -> *const core::ffi::c_void {
- - core::ptr::NonNull::dangling().as_ptr()
- - }
- -
- - unsafe fn borrow<'a>(_: *const core::ffi::c_void) -> Self::Borrowed<'a> {}
- -
- - unsafe fn from_foreign(_: *const core::ffi::c_void) -> Self {}
- -}
- -
- -/// Runs a cleanup function/closure when dropped.
- -///
- -/// The [`ScopeGuard::dismiss`] function prevents the cleanup function from running.
- -///
- -/// # Examples
- -///
- -/// In the example below, we have multiple exit paths and we want to log regardless of which one is
- -/// taken:
- -///
- -/// ```
- -/// # use kernel::types::ScopeGuard;
- -/// fn example1(arg: bool) {
- -/// let _log = ScopeGuard::new(|| pr_info!("example1 completed\n"));
- -///
- -/// if arg {
- -/// return;
- -/// }
- -///
- -/// pr_info!("Do something...\n");
- -/// }
- -///
- -/// # example1(false);
- -/// # example1(true);
- -/// ```
- -///
- -/// In the example below, we want to log the same message on all early exits but a different one on
- -/// the main exit path:
- -///
- -/// ```
- -/// # use kernel::types::ScopeGuard;
- -/// fn example2(arg: bool) {
- -/// let log = ScopeGuard::new(|| pr_info!("example2 returned early\n"));
- -///
- -/// if arg {
- -/// return;
- -/// }
- -///
- -/// // (Other early returns...)
- -///
- -/// log.dismiss();
- -/// pr_info!("example2 no early return\n");
- -/// }
- -///
- -/// # example2(false);
- -/// # example2(true);
- -/// ```
- -///
- -/// In the example below, we need a mutable object (the vector) to be accessible within the log
- -/// function, so we wrap it in the [`ScopeGuard`]:
- -///
- -/// ```
- -/// # use kernel::types::ScopeGuard;
- -/// fn example3(arg: bool) -> Result {
- -/// let mut vec =
- -/// ScopeGuard::new_with_data(Vec::new(), |v| pr_info!("vec had {} elements\n", v.len()));
- -///
- -/// vec.push(10u8, GFP_KERNEL)?;
- -/// if arg {
- -/// return Ok(());
- -/// }
- -/// vec.push(20u8, GFP_KERNEL)?;
- -/// Ok(())
- -/// }
- -///
- -/// # assert_eq!(example3(false), Ok(()));
- -/// # assert_eq!(example3(true), Ok(()));
- -/// ```
- -///
- -/// # Invariants
- -///
- -/// The value stored in the struct is nearly always `Some(_)`, except between
- -/// [`ScopeGuard::dismiss`] and [`ScopeGuard::drop`]: in this case, it will be `None` as the value
- -/// will have been returned to the caller. Since [`ScopeGuard::dismiss`] consumes the guard,
- -/// callers won't be able to use it anymore.
- -pub struct ScopeGuard<T, F: FnOnce(T)>(Option<(T, F)>);
- -
- -impl<T, F: FnOnce(T)> ScopeGuard<T, F> {
- - /// Creates a new guarded object wrapping the given data and with the given cleanup function.
- - pub fn new_with_data(data: T, cleanup_func: F) -> Self {
- - // INVARIANT: The struct is being initialised with `Some(_)`.
- - Self(Some((data, cleanup_func)))
- - }
- -
- - /// Prevents the cleanup function from running and returns the guarded data.
- - pub fn dismiss(mut self) -> T {
- - // INVARIANT: This is the exception case in the invariant; it is not visible to callers
- - // because this function consumes `self`.
- - self.0.take().unwrap().0
- - }
- -}
- -
- -impl ScopeGuard<(), fn(())> {
- - /// Creates a new guarded object with the given cleanup function.
- - pub fn new(cleanup: impl FnOnce()) -> ScopeGuard<(), impl FnOnce(())> {
- - ScopeGuard::new_with_data((), move |_| cleanup())
- - }
- -}
- -
- -impl<T, F: FnOnce(T)> Deref for ScopeGuard<T, F> {
- - type Target = T;
- -
- - fn deref(&self) -> &T {
- - // The type invariants guarantee that `unwrap` will succeed.
- - &self.0.as_ref().unwrap().0
- - }
- -}
- -
- -impl<T, F: FnOnce(T)> DerefMut for ScopeGuard<T, F> {
- - fn deref_mut(&mut self) -> &mut T {
- - // The type invariants guarantee that `unwrap` will succeed.
- - &mut self.0.as_mut().unwrap().0
- - }
- -}
- -
- -impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> {
- - fn drop(&mut self) {
- - // Run the cleanup function if one is still present.
- - if let Some((data, cleanup)) = self.0.take() {
- - cleanup(data)
- - }
- - }
- -}
- -
- -/// Stores an opaque value.
- -///
- -/// This is meant to be used with FFI objects that are never interpreted by Rust code.
- -#[repr(transparent)]
- -pub struct Opaque<T> {
- - value: UnsafeCell<MaybeUninit<T>>,
- - _pin: PhantomPinned,
- -}
- -
- -impl<T> Opaque<T> {
- - /// Creates a new opaque value.
- - pub const fn new(value: T) -> Self {
- - Self {
- - value: UnsafeCell::new(MaybeUninit::new(value)),
- - _pin: PhantomPinned,
- - }
- - }
- -
- - /// Creates an uninitialised value.
- - pub const fn uninit() -> Self {
- - Self {
- - value: UnsafeCell::new(MaybeUninit::uninit()),
- - _pin: PhantomPinned,
- - }
- - }
- -
- - /// Creates a pin-initializer from the given initializer closure.
- - ///
- - /// The returned initializer calls the given closure with the pointer to the inner `T` of this
- - /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it.
- - ///
- - /// This function is safe, because the `T` inside of an `Opaque` is allowed to be
- - /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs
- - /// to verify at that point that the inner value is valid.
- - pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit<Self> {
- - // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully
- - // initialize the `T`.
- - unsafe {
- - init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| {
- - init_func(Self::raw_get(slot));
- - Ok(())
- - })
- - }
- - }
- -
- - /// Returns a raw pointer to the opaque data.
- - pub const fn get(&self) -> *mut T {
- - UnsafeCell::get(&self.value).cast::<T>()
- - }
- -
- - /// Gets the value behind `this`.
- - ///
- - /// This function is useful to get access to the value without creating intermediate
- - /// references.
- - pub const fn raw_get(this: *const Self) -> *mut T {
- - UnsafeCell::raw_get(this.cast::<UnsafeCell<MaybeUninit<T>>>()).cast::<T>()
- - }
- -}
- -
- -/// Types that are _always_ reference counted.
- -///
- -/// It allows such types to define their own custom ref increment and decrement functions.
- -/// Additionally, it allows users to convert from a shared reference `&T` to an owned reference
- -/// [`ARef<T>`].
- -///
- -/// This is usually implemented by wrappers to existing structures on the C side of the code. For
- -/// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted
- -/// instances of a type.
- -///
- -/// # Safety
- -///
- -/// Implementers must ensure that increments to the reference count keep the object alive in memory
- -/// at least until matching decrements are performed.
- -///
- -/// Implementers must also ensure that all instances are reference-counted. (Otherwise they
- -/// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object
- -/// alive.)
- -pub unsafe trait AlwaysRefCounted {
- - /// Increments the reference count on the object.
- - fn inc_ref(&self);
- -
- - /// Decrements the reference count on the object.
- - ///
- - /// Frees the object when the count reaches zero.
- - ///
- - /// # Safety
- - ///
- - /// Callers must ensure that there was a previous matching increment to the reference count,
- - /// and that the object is no longer used after its reference count is decremented (as it may
- - /// result in the object being freed), unless the caller owns another increment on the refcount
- - /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls
- - /// [`AlwaysRefCounted::dec_ref`] once).
- - unsafe fn dec_ref(obj: NonNull<Self>);
- -}
- -
- -/// An owned reference to an always-reference-counted object.
- -///
- -/// The object's reference count is automatically decremented when an instance of [`ARef`] is
- -/// dropped. It is also automatically incremented when a new instance is created via
- -/// [`ARef::clone`].
- -///
- -/// # Invariants
- -///
- -/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In
- -/// particular, the [`ARef`] instance owns an increment on the underlying object's reference count.
- -pub struct ARef<T: AlwaysRefCounted> {
- - ptr: NonNull<T>,
- - _p: PhantomData<T>,
- -}
- -
- -// SAFETY: It is safe to send `ARef<T>` to another thread when the underlying `T` is `Sync` because
- -// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
- -// `T` to be `Send` because any thread that has an `ARef<T>` may ultimately access `T` using a
- -// mutable reference, for example, when the reference count reaches zero and `T` is dropped.
- -unsafe impl<T: AlwaysRefCounted + Sync + Send> Send for ARef<T> {}
- -
- -// SAFETY: It is safe to send `&ARef<T>` to another thread when the underlying `T` is `Sync`
- -// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
- -// it needs `T` to be `Send` because any thread that has a `&ARef<T>` may clone it and get an
- -// `ARef<T>` on that thread, so the thread may ultimately access `T` using a mutable reference, for
- -// example, when the reference count reaches zero and `T` is dropped.
- -unsafe impl<T: AlwaysRefCounted + Sync + Send> Sync for ARef<T> {}
- -
- -impl<T: AlwaysRefCounted> ARef<T> {
- - /// Creates a new instance of [`ARef`].
- - ///
- - /// It takes over an increment of the reference count on the underlying object.
- - ///
- - /// # Safety
- - ///
- - /// Callers must ensure that the reference count was incremented at least once, and that they
- - /// are properly relinquishing one increment. That is, if there is only one increment, callers
- - /// must not use the underlying object anymore -- it is only safe to do so via the newly
- - /// created [`ARef`].
- - pub unsafe fn from_raw(ptr: NonNull<T>) -> Self {
- - // INVARIANT: The safety requirements guarantee that the new instance now owns the
- - // increment on the refcount.
- - Self {
- - ptr,
- - _p: PhantomData,
- - }
- - }
- -}
- -
- -impl<T: AlwaysRefCounted> Clone for ARef<T> {
- - fn clone(&self) -> Self {
- - self.inc_ref();
- - // SAFETY: We just incremented the refcount above.
- - unsafe { Self::from_raw(self.ptr) }
- - }
- -}
- -
- -impl<T: AlwaysRefCounted> Deref for ARef<T> {
- - type Target = T;
- -
- - fn deref(&self) -> &Self::Target {
- - // SAFETY: The type invariants guarantee that the object is valid.
- - unsafe { self.ptr.as_ref() }
- - }
- -}
- -
- -impl<T: AlwaysRefCounted> From<&T> for ARef<T> {
- - fn from(b: &T) -> Self {
- - b.inc_ref();
- - // SAFETY: We just incremented the refcount above.
- - unsafe { Self::from_raw(NonNull::from(b)) }
- - }
- -}
- -
- -impl<T: AlwaysRefCounted> Drop for ARef<T> {
- - fn drop(&mut self) {
- - // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to
- - // decrement.
- - unsafe { T::dec_ref(self.ptr) };
- - }
- -}
- -
- -/// A sum type that always holds either a value of type `L` or `R`.
- -pub enum Either<L, R> {
- - /// Constructs an instance of [`Either`] containing a value of type `L`.
- - Left(L),
- -
- - /// Constructs an instance of [`Either`] containing a value of type `R`.
- - Right(R),
- -}
- diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs
- deleted file mode 100644
- --- a/rust/kernel/workqueue.rs
- +++ /dev/null
- @@ -1,683 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Work queues.
- -//!
- -//! This file has two components: The raw work item API, and the safe work item API.
- -//!
- -//! One pattern that is used in both APIs is the `ID` const generic, which exists to allow a single
- -//! type to define multiple `work_struct` fields. This is done by choosing an id for each field,
- -//! and using that id to specify which field you wish to use. (The actual value doesn't matter, as
- -//! long as you use different values for different fields of the same struct.) Since these IDs are
- -//! generic, they are used only at compile-time, so they shouldn't exist in the final binary.
- -//!
- -//! # The raw API
- -//!
- -//! The raw API consists of the [`RawWorkItem`] trait, where the work item needs to provide an
- -//! arbitrary function that knows how to enqueue the work item. It should usually not be used
- -//! directly, but if you want to, you can use it without using the pieces from the safe API.
- -//!
- -//! # The safe API
- -//!
- -//! The safe API is used via the [`Work`] struct and [`WorkItem`] traits. Furthermore, it also
- -//! includes a trait called [`WorkItemPointer`], which is usually not used directly by the user.
- -//!
- -//! * The [`Work`] struct is the Rust wrapper for the C `work_struct` type.
- -//! * The [`WorkItem`] trait is implemented for structs that can be enqueued to a workqueue.
- -//! * The [`WorkItemPointer`] trait is implemented for the pointer type that points at a something
- -//! that implements [`WorkItem`].
- -//!
- -//! ## Example
- -//!
- -//! This example defines a struct that holds an integer and can be scheduled on the workqueue. When
- -//! the struct is executed, it will print the integer. Since there is only one `work_struct` field,
- -//! we do not need to specify ids for the fields.
- -//!
- -//! ```
- -//! use kernel::sync::Arc;
- -//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem};
- -//!
- -//! #[pin_data]
- -//! struct MyStruct {
- -//! value: i32,
- -//! #[pin]
- -//! work: Work<MyStruct>,
- -//! }
- -//!
- -//! impl_has_work! {
- -//! impl HasWork<Self> for MyStruct { self.work }
- -//! }
- -//!
- -//! impl MyStruct {
- -//! fn new(value: i32) -> Result<Arc<Self>> {
- -//! Arc::pin_init(pin_init!(MyStruct {
- -//! value,
- -//! work <- new_work!("MyStruct::work"),
- -//! }), GFP_KERNEL)
- -//! }
- -//! }
- -//!
- -//! impl WorkItem for MyStruct {
- -//! type Pointer = Arc<MyStruct>;
- -//!
- -//! fn run(this: Arc<MyStruct>) {
- -//! pr_info!("The value is: {}", this.value);
- -//! }
- -//! }
- -//!
- -//! /// This method will enqueue the struct for execution on the system workqueue, where its value
- -//! /// will be printed.
- -//! fn print_later(val: Arc<MyStruct>) {
- -//! let _ = workqueue::system().enqueue(val);
- -//! }
- -//! ```
- -//!
- -//! The following example shows how multiple `work_struct` fields can be used:
- -//!
- -//! ```
- -//! use kernel::sync::Arc;
- -//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem};
- -//!
- -//! #[pin_data]
- -//! struct MyStruct {
- -//! value_1: i32,
- -//! value_2: i32,
- -//! #[pin]
- -//! work_1: Work<MyStruct, 1>,
- -//! #[pin]
- -//! work_2: Work<MyStruct, 2>,
- -//! }
- -//!
- -//! impl_has_work! {
- -//! impl HasWork<Self, 1> for MyStruct { self.work_1 }
- -//! impl HasWork<Self, 2> for MyStruct { self.work_2 }
- -//! }
- -//!
- -//! impl MyStruct {
- -//! fn new(value_1: i32, value_2: i32) -> Result<Arc<Self>> {
- -//! Arc::pin_init(pin_init!(MyStruct {
- -//! value_1,
- -//! value_2,
- -//! work_1 <- new_work!("MyStruct::work_1"),
- -//! work_2 <- new_work!("MyStruct::work_2"),
- -//! }), GFP_KERNEL)
- -//! }
- -//! }
- -//!
- -//! impl WorkItem<1> for MyStruct {
- -//! type Pointer = Arc<MyStruct>;
- -//!
- -//! fn run(this: Arc<MyStruct>) {
- -//! pr_info!("The value is: {}", this.value_1);
- -//! }
- -//! }
- -//!
- -//! impl WorkItem<2> for MyStruct {
- -//! type Pointer = Arc<MyStruct>;
- -//!
- -//! fn run(this: Arc<MyStruct>) {
- -//! pr_info!("The second value is: {}", this.value_2);
- -//! }
- -//! }
- -//!
- -//! fn print_1_later(val: Arc<MyStruct>) {
- -//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 1>(val);
- -//! }
- -//!
- -//! fn print_2_later(val: Arc<MyStruct>) {
- -//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val);
- -//! }
- -//! ```
- -//!
- -//! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h)
- -
- -use crate::alloc::{AllocError, Flags};
- -use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque};
- -use core::marker::PhantomData;
- -
- -/// Creates a [`Work`] initialiser with the given name and a newly-created lock class.
- -#[macro_export]
- -macro_rules! new_work {
- - ($($name:literal)?) => {
- - $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!())
- - };
- -}
- -pub use new_work;
- -
- -/// A kernel work queue.
- -///
- -/// Wraps the kernel's C `struct workqueue_struct`.
- -///
- -/// It allows work items to be queued to run on thread pools managed by the kernel. Several are
- -/// always available, for example, `system`, `system_highpri`, `system_long`, etc.
- -#[repr(transparent)]
- -pub struct Queue(Opaque<bindings::workqueue_struct>);
- -
- -// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe.
- -unsafe impl Send for Queue {}
- -// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe.
- -unsafe impl Sync for Queue {}
- -
- -impl Queue {
- - /// Use the provided `struct workqueue_struct` with Rust.
- - ///
- - /// # Safety
- - ///
- - /// The caller must ensure that the provided raw pointer is not dangling, that it points at a
- - /// valid workqueue, and that it remains valid until the end of `'a`.
- - pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue {
- - // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The
- - // caller promises that the pointer is not dangling.
- - unsafe { &*(ptr as *const Queue) }
- - }
- -
- - /// Enqueues a work item.
- - ///
- - /// This may fail if the work item is already enqueued in a workqueue.
- - ///
- - /// The work item will be submitted using `WORK_CPU_UNBOUND`.
- - pub fn enqueue<W, const ID: u64>(&self, w: W) -> W::EnqueueOutput
- - where
- - W: RawWorkItem<ID> + Send + 'static,
- - {
- - let queue_ptr = self.0.get();
- -
- - // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other
- - // `__enqueue` requirements are not relevant since `W` is `Send` and static.
- - //
- - // The call to `bindings::queue_work_on` will dereference the provided raw pointer, which
- - // is ok because `__enqueue` guarantees that the pointer is valid for the duration of this
- - // closure.
- - //
- - // Furthermore, if the C workqueue code accesses the pointer after this call to
- - // `__enqueue`, then the work item was successfully enqueued, and `bindings::queue_work_on`
- - // will have returned true. In this case, `__enqueue` promises that the raw pointer will
- - // stay valid until we call the function pointer in the `work_struct`, so the access is ok.
- - unsafe {
- - w.__enqueue(move |work_ptr| {
- - bindings::queue_work_on(
- - bindings::wq_misc_consts_WORK_CPU_UNBOUND as _,
- - queue_ptr,
- - work_ptr,
- - )
- - })
- - }
- - }
- -
- - /// Tries to spawn the given function or closure as a work item.
- - ///
- - /// This method can fail because it allocates memory to store the work item.
- - pub fn try_spawn<T: 'static + Send + FnOnce()>(
- - &self,
- - flags: Flags,
- - func: T,
- - ) -> Result<(), AllocError> {
- - let init = pin_init!(ClosureWork {
- - work <- new_work!("Queue::try_spawn"),
- - func: Some(func),
- - });
- -
- - self.enqueue(Box::pin_init(init, flags).map_err(|_| AllocError)?);
- - Ok(())
- - }
- -}
- -
- -/// A helper type used in [`try_spawn`].
- -///
- -/// [`try_spawn`]: Queue::try_spawn
- -#[pin_data]
- -struct ClosureWork<T> {
- - #[pin]
- - work: Work<ClosureWork<T>>,
- - func: Option<T>,
- -}
- -
- -impl<T> ClosureWork<T> {
- - fn project(self: Pin<&mut Self>) -> &mut Option<T> {
- - // SAFETY: The `func` field is not structurally pinned.
- - unsafe { &mut self.get_unchecked_mut().func }
- - }
- -}
- -
- -impl<T: FnOnce()> WorkItem for ClosureWork<T> {
- - type Pointer = Pin<Box<Self>>;
- -
- - fn run(mut this: Pin<Box<Self>>) {
- - if let Some(func) = this.as_mut().project().take() {
- - (func)()
- - }
- - }
- -}
- -
- -/// A raw work item.
- -///
- -/// This is the low-level trait that is designed for being as general as possible.
- -///
- -/// The `ID` parameter to this trait exists so that a single type can provide multiple
- -/// implementations of this trait. For example, if a struct has multiple `work_struct` fields, then
- -/// you will implement this trait once for each field, using a different id for each field. The
- -/// actual value of the id is not important as long as you use different ids for different fields
- -/// of the same struct. (Fields of different structs need not use different ids.)
- -///
- -/// Note that the id is used only to select the right method to call during compilation. It won't be
- -/// part of the final executable.
- -///
- -/// # Safety
- -///
- -/// Implementers must ensure that any pointers passed to a `queue_work_on` closure by [`__enqueue`]
- -/// remain valid for the duration specified in the guarantees section of the documentation for
- -/// [`__enqueue`].
- -///
- -/// [`__enqueue`]: RawWorkItem::__enqueue
- -pub unsafe trait RawWorkItem<const ID: u64> {
- - /// The return type of [`Queue::enqueue`].
- - type EnqueueOutput;
- -
- - /// Enqueues this work item on a queue using the provided `queue_work_on` method.
- - ///
- - /// # Guarantees
- - ///
- - /// If this method calls the provided closure, then the raw pointer is guaranteed to point at a
- - /// valid `work_struct` for the duration of the call to the closure. If the closure returns
- - /// true, then it is further guaranteed that the pointer remains valid until someone calls the
- - /// function pointer stored in the `work_struct`.
- - ///
- - /// # Safety
- - ///
- - /// The provided closure may only return `false` if the `work_struct` is already in a workqueue.
- - ///
- - /// If the work item type is annotated with any lifetimes, then you must not call the function
- - /// pointer after any such lifetime expires. (Never calling the function pointer is okay.)
- - ///
- - /// If the work item type is not [`Send`], then the function pointer must be called on the same
- - /// thread as the call to `__enqueue`.
- - unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput
- - where
- - F: FnOnce(*mut bindings::work_struct) -> bool;
- -}
- -
- -/// Defines the method that should be called directly when a work item is executed.
- -///
- -/// This trait is implemented by `Pin<Box<T>>` and [`Arc<T>`], and is mainly intended to be
- -/// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`]
- -/// instead. The [`run`] method on this trait will usually just perform the appropriate
- -/// `container_of` translation and then call into the [`run`][WorkItem::run] method from the
- -/// [`WorkItem`] trait.
- -///
- -/// This trait is used when the `work_struct` field is defined using the [`Work`] helper.
- -///
- -/// # Safety
- -///
- -/// Implementers must ensure that [`__enqueue`] uses a `work_struct` initialized with the [`run`]
- -/// method of this trait as the function pointer.
- -///
- -/// [`__enqueue`]: RawWorkItem::__enqueue
- -/// [`run`]: WorkItemPointer::run
- -pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> {
- - /// Run this work item.
- - ///
- - /// # Safety
- - ///
- - /// The provided `work_struct` pointer must originate from a previous call to [`__enqueue`]
- - /// where the `queue_work_on` closure returned true, and the pointer must still be valid.
- - ///
- - /// [`__enqueue`]: RawWorkItem::__enqueue
- - unsafe extern "C" fn run(ptr: *mut bindings::work_struct);
- -}
- -
- -/// Defines the method that should be called when this work item is executed.
- -///
- -/// This trait is used when the `work_struct` field is defined using the [`Work`] helper.
- -pub trait WorkItem<const ID: u64 = 0> {
- - /// The pointer type that this struct is wrapped in. This will typically be `Arc<Self>` or
- - /// `Pin<Box<Self>>`.
- - type Pointer: WorkItemPointer<ID>;
- -
- - /// The method that should be called when this work item is executed.
- - fn run(this: Self::Pointer);
- -}
- -
- -/// Links for a work item.
- -///
- -/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`]
- -/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue.
- -///
- -/// Wraps the kernel's C `struct work_struct`.
- -///
- -/// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it.
- -///
- -/// [`run`]: WorkItemPointer::run
- -#[pin_data]
- -#[repr(transparent)]
- -pub struct Work<T: ?Sized, const ID: u64 = 0> {
- - #[pin]
- - work: Opaque<bindings::work_struct>,
- - _inner: PhantomData<T>,
- -}
- -
- -// SAFETY: Kernel work items are usable from any thread.
- -//
- -// We do not need to constrain `T` since the work item does not actually contain a `T`.
- -unsafe impl<T: ?Sized, const ID: u64> Send for Work<T, ID> {}
- -// SAFETY: Kernel work items are usable from any thread.
- -//
- -// We do not need to constrain `T` since the work item does not actually contain a `T`.
- -unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {}
- -
- -impl<T: ?Sized, const ID: u64> Work<T, ID> {
- - /// Creates a new instance of [`Work`].
- - #[inline]
- - #[allow(clippy::new_ret_no_self)]
- - pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self>
- - where
- - T: WorkItem<ID>,
- - {
- - pin_init!(Self {
- - work <- Opaque::ffi_init(|slot| {
- - // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as
- - // the work item function.
- - unsafe {
- - bindings::init_work_with_key(
- - slot,
- - Some(T::Pointer::run),
- - false,
- - name.as_char_ptr(),
- - key.as_ptr(),
- - )
- - }
- - }),
- - _inner: PhantomData,
- - })
- - }
- -
- - /// Get a pointer to the inner `work_struct`.
- - ///
- - /// # Safety
- - ///
- - /// The provided pointer must not be dangling and must be properly aligned. (But the memory
- - /// need not be initialized.)
- - #[inline]
- - pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct {
- - // SAFETY: The caller promises that the pointer is aligned and not dangling.
- - //
- - // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that
- - // the compiler does not complain that the `work` field is unused.
- - unsafe { Opaque::raw_get(core::ptr::addr_of!((*ptr).work)) }
- - }
- -}
- -
- -/// Declares that a type has a [`Work<T, ID>`] field.
- -///
- -/// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro
- -/// like this:
- -///
- -/// ```no_run
- -/// use kernel::workqueue::{impl_has_work, Work};
- -///
- -/// struct MyWorkItem {
- -/// work_field: Work<MyWorkItem, 1>,
- -/// }
- -///
- -/// impl_has_work! {
- -/// impl HasWork<MyWorkItem, 1> for MyWorkItem { self.work_field }
- -/// }
- -/// ```
- -///
- -/// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct`
- -/// fields by using a different id for each one.
- -///
- -/// # Safety
- -///
- -/// The [`OFFSET`] constant must be the offset of a field in `Self` of type [`Work<T, ID>`]. The
- -/// methods on this trait must have exactly the behavior that the definitions given below have.
- -///
- -/// [`impl_has_work!`]: crate::impl_has_work
- -/// [`OFFSET`]: HasWork::OFFSET
- -pub unsafe trait HasWork<T, const ID: u64 = 0> {
- - /// The offset of the [`Work<T, ID>`] field.
- - const OFFSET: usize;
- -
- - /// Returns the offset of the [`Work<T, ID>`] field.
- - ///
- - /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not
- - /// [`Sized`].
- - ///
- - /// [`OFFSET`]: HasWork::OFFSET
- - #[inline]
- - fn get_work_offset(&self) -> usize {
- - Self::OFFSET
- - }
- -
- - /// Returns a pointer to the [`Work<T, ID>`] field.
- - ///
- - /// # Safety
- - ///
- - /// The provided pointer must point at a valid struct of type `Self`.
- - #[inline]
- - unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> {
- - // SAFETY: The caller promises that the pointer is valid.
- - unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut Work<T, ID> }
- - }
- -
- - /// Returns a pointer to the struct containing the [`Work<T, ID>`] field.
- - ///
- - /// # Safety
- - ///
- - /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`.
- - #[inline]
- - unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self
- - where
- - Self: Sized,
- - {
- - // SAFETY: The caller promises that the pointer points at a field of the right type in the
- - // right kind of struct.
- - unsafe { (ptr as *mut u8).sub(Self::OFFSET) as *mut Self }
- - }
- -}
- -
- -/// Used to safely implement the [`HasWork<T, ID>`] trait.
- -///
- -/// # Examples
- -///
- -/// ```
- -/// use kernel::sync::Arc;
- -/// use kernel::workqueue::{self, impl_has_work, Work};
- -///
- -/// struct MyStruct {
- -/// work_field: Work<MyStruct, 17>,
- -/// }
- -///
- -/// impl_has_work! {
- -/// impl HasWork<MyStruct, 17> for MyStruct { self.work_field }
- -/// }
- -/// ```
- -#[macro_export]
- -macro_rules! impl_has_work {
- - ($(impl$(<$($implarg:ident),*>)?
- - HasWork<$work_type:ty $(, $id:tt)?>
- - for $self:ident $(<$($selfarg:ident),*>)?
- - { self.$field:ident }
- - )*) => {$(
- - // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right
- - // type.
- - unsafe impl$(<$($implarg),*>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self $(<$($selfarg),*>)? {
- - const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize;
- -
- - #[inline]
- - unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> {
- - // SAFETY: The caller promises that the pointer is not dangling.
- - unsafe {
- - ::core::ptr::addr_of_mut!((*ptr).$field)
- - }
- - }
- - }
- - )*};
- -}
- -pub use impl_has_work;
- -
- -impl_has_work! {
- - impl<T> HasWork<Self> for ClosureWork<T> { self.work }
- -}
- -
- -unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T>
- -where
- - T: WorkItem<ID, Pointer = Self>,
- - T: HasWork<T, ID>,
- -{
- - unsafe extern "C" fn run(ptr: *mut bindings::work_struct) {
- - // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`.
- - let ptr = ptr as *mut Work<T, ID>;
- - // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`.
- - let ptr = unsafe { T::work_container_of(ptr) };
- - // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership.
- - let arc = unsafe { Arc::from_raw(ptr) };
- -
- - T::run(arc)
- - }
- -}
- -
- -unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T>
- -where
- - T: WorkItem<ID, Pointer = Self>,
- - T: HasWork<T, ID>,
- -{
- - type EnqueueOutput = Result<(), Self>;
- -
- - unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput
- - where
- - F: FnOnce(*mut bindings::work_struct) -> bool,
- - {
- - // Casting between const and mut is not a problem as long as the pointer is a raw pointer.
- - let ptr = Arc::into_raw(self).cast_mut();
- -
- - // SAFETY: Pointers into an `Arc` point at a valid value.
- - let work_ptr = unsafe { T::raw_get_work(ptr) };
- - // SAFETY: `raw_get_work` returns a pointer to a valid value.
- - let work_ptr = unsafe { Work::raw_get(work_ptr) };
- -
- - if queue_work_on(work_ptr) {
- - Ok(())
- - } else {
- - // SAFETY: The work queue has not taken ownership of the pointer.
- - Err(unsafe { Arc::from_raw(ptr) })
- - }
- - }
- -}
- -
- -unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<Box<T>>
- -where
- - T: WorkItem<ID, Pointer = Self>,
- - T: HasWork<T, ID>,
- -{
- - unsafe extern "C" fn run(ptr: *mut bindings::work_struct) {
- - // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`.
- - let ptr = ptr as *mut Work<T, ID>;
- - // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`.
- - let ptr = unsafe { T::work_container_of(ptr) };
- - // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership.
- - let boxed = unsafe { Box::from_raw(ptr) };
- - // SAFETY: The box was already pinned when it was enqueued.
- - let pinned = unsafe { Pin::new_unchecked(boxed) };
- -
- - T::run(pinned)
- - }
- -}
- -
- -unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<Box<T>>
- -where
- - T: WorkItem<ID, Pointer = Self>,
- - T: HasWork<T, ID>,
- -{
- - type EnqueueOutput = ();
- -
- - unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput
- - where
- - F: FnOnce(*mut bindings::work_struct) -> bool,
- - {
- - // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily
- - // remove the `Pin` wrapper.
- - let boxed = unsafe { Pin::into_inner_unchecked(self) };
- - let ptr = Box::into_raw(boxed);
- -
- - // SAFETY: Pointers into a `Box` point at a valid value.
- - let work_ptr = unsafe { T::raw_get_work(ptr) };
- - // SAFETY: `raw_get_work` returns a pointer to a valid value.
- - let work_ptr = unsafe { Work::raw_get(work_ptr) };
- -
- - if !queue_work_on(work_ptr) {
- - // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a
- - // workqueue.
- - unsafe { ::core::hint::unreachable_unchecked() }
- - }
- - }
- -}
- -
- -/// Returns the system work queue (`system_wq`).
- -///
- -/// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are
- -/// users which expect relatively short queue flush time.
- -///
- -/// Callers shouldn't queue work items which can run for too long.
- -pub fn system() -> &'static Queue {
- - // SAFETY: `system_wq` is a C global, always available.
- - unsafe { Queue::from_raw(bindings::system_wq) }
- -}
- -
- -/// Returns the system high-priority work queue (`system_highpri_wq`).
- -///
- -/// It is similar to the one returned by [`system`] but for work items which require higher
- -/// scheduling priority.
- -pub fn system_highpri() -> &'static Queue {
- - // SAFETY: `system_highpri_wq` is a C global, always available.
- - unsafe { Queue::from_raw(bindings::system_highpri_wq) }
- -}
- -
- -/// Returns the system work queue for potentially long-running work items (`system_long_wq`).
- -///
- -/// It is similar to the one returned by [`system`] but may host long running work items. Queue
- -/// flushing might take relatively long.
- -pub fn system_long() -> &'static Queue {
- - // SAFETY: `system_long_wq` is a C global, always available.
- - unsafe { Queue::from_raw(bindings::system_long_wq) }
- -}
- -
- -/// Returns the system unbound work queue (`system_unbound_wq`).
- -///
- -/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items
- -/// are executed immediately as long as `max_active` limit is not reached and resources are
- -/// available.
- -pub fn system_unbound() -> &'static Queue {
- - // SAFETY: `system_unbound_wq` is a C global, always available.
- - unsafe { Queue::from_raw(bindings::system_unbound_wq) }
- -}
- -
- -/// Returns the system freezable work queue (`system_freezable_wq`).
- -///
- -/// It is equivalent to the one returned by [`system`] except that it's freezable.
- -///
- -/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work
- -/// items on the workqueue are drained and no new work item starts execution until thawed.
- -pub fn system_freezable() -> &'static Queue {
- - // SAFETY: `system_freezable_wq` is a C global, always available.
- - unsafe { Queue::from_raw(bindings::system_freezable_wq) }
- -}
- -
- -/// Returns the system power-efficient work queue (`system_power_efficient_wq`).
- -///
- -/// It is inclined towards saving power and is converted to "unbound" variants if the
- -/// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one
- -/// returned by [`system`].
- -pub fn system_power_efficient() -> &'static Queue {
- - // SAFETY: `system_power_efficient_wq` is a C global, always available.
- - unsafe { Queue::from_raw(bindings::system_power_efficient_wq) }
- -}
- -
- -/// Returns the system freezable power-efficient work queue (`system_freezable_power_efficient_wq`).
- -///
- -/// It is similar to the one returned by [`system_power_efficient`] except that is freezable.
- -///
- -/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work
- -/// items on the workqueue are drained and no new work item starts execution until thawed.
- -pub fn system_freezable_power_efficient() -> &'static Queue {
- - // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available.
- - unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) }
- -}
- diff --git a/rust/macros/concat_idents.rs b/rust/macros/concat_idents.rs
- deleted file mode 100644
- --- a/rust/macros/concat_idents.rs
- +++ /dev/null
- @@ -1,23 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -use proc_macro::{token_stream, Ident, TokenStream, TokenTree};
- -
- -use crate::helpers::expect_punct;
- -
- -fn expect_ident(it: &mut token_stream::IntoIter) -> Ident {
- - if let Some(TokenTree::Ident(ident)) = it.next() {
- - ident
- - } else {
- - panic!("Expected Ident")
- - }
- -}
- -
- -pub(crate) fn concat_idents(ts: TokenStream) -> TokenStream {
- - let mut it = ts.into_iter();
- - let a = expect_ident(&mut it);
- - assert_eq!(expect_punct(&mut it), ',');
- - let b = expect_ident(&mut it);
- - assert!(it.next().is_none(), "only two idents can be concatenated");
- - let res = Ident::new(&format!("{a}{b}"), b.span());
- - TokenStream::from_iter([TokenTree::Ident(res)])
- -}
- diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs
- deleted file mode 100644
- --- a/rust/macros/helpers.rs
- +++ /dev/null
- @@ -1,217 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -use proc_macro::{token_stream, Group, TokenStream, TokenTree};
- -
- -pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option<String> {
- - if let Some(TokenTree::Ident(ident)) = it.next() {
- - Some(ident.to_string())
- - } else {
- - None
- - }
- -}
- -
- -pub(crate) fn try_literal(it: &mut token_stream::IntoIter) -> Option<String> {
- - if let Some(TokenTree::Literal(literal)) = it.next() {
- - Some(literal.to_string())
- - } else {
- - None
- - }
- -}
- -
- -pub(crate) fn try_string(it: &mut token_stream::IntoIter) -> Option<String> {
- - try_literal(it).and_then(|string| {
- - if string.starts_with('\"') && string.ends_with('\"') {
- - let content = &string[1..string.len() - 1];
- - if content.contains('\\') {
- - panic!("Escape sequences in string literals not yet handled");
- - }
- - Some(content.to_string())
- - } else if string.starts_with("r\"") {
- - panic!("Raw string literals are not yet handled");
- - } else {
- - None
- - }
- - })
- -}
- -
- -pub(crate) fn expect_ident(it: &mut token_stream::IntoIter) -> String {
- - try_ident(it).expect("Expected Ident")
- -}
- -
- -pub(crate) fn expect_punct(it: &mut token_stream::IntoIter) -> char {
- - if let TokenTree::Punct(punct) = it.next().expect("Reached end of token stream for Punct") {
- - punct.as_char()
- - } else {
- - panic!("Expected Punct");
- - }
- -}
- -
- -pub(crate) fn expect_string(it: &mut token_stream::IntoIter) -> String {
- - try_string(it).expect("Expected string")
- -}
- -
- -pub(crate) fn expect_string_ascii(it: &mut token_stream::IntoIter) -> String {
- - let string = try_string(it).expect("Expected string");
- - assert!(string.is_ascii(), "Expected ASCII string");
- - string
- -}
- -
- -pub(crate) fn expect_group(it: &mut token_stream::IntoIter) -> Group {
- - if let TokenTree::Group(group) = it.next().expect("Reached end of token stream for Group") {
- - group
- - } else {
- - panic!("Expected Group");
- - }
- -}
- -
- -pub(crate) fn expect_end(it: &mut token_stream::IntoIter) {
- - if it.next().is_some() {
- - panic!("Expected end");
- - }
- -}
- -
- -/// Parsed generics.
- -///
- -/// See the field documentation for an explanation what each of the fields represents.
- -///
- -/// # Examples
- -///
- -/// ```rust,ignore
- -/// # let input = todo!();
- -/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input);
- -/// quote! {
- -/// struct Foo<$($decl_generics)*> {
- -/// // ...
- -/// }
- -///
- -/// impl<$impl_generics> Foo<$ty_generics> {
- -/// fn foo() {
- -/// // ...
- -/// }
- -/// }
- -/// }
- -/// ```
- -pub(crate) struct Generics {
- - /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`).
- - ///
- - /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`).
- - pub(crate) decl_generics: Vec<TokenTree>,
- - /// The generics with bounds (e.g. `T: Clone, const N: usize`).
- - ///
- - /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`.
- - pub(crate) impl_generics: Vec<TokenTree>,
- - /// The generics without bounds and without default values (e.g. `T, N`).
- - ///
- - /// Use this when you use the type that is declared with these generics e.g.
- - /// `Foo<$ty_generics>`.
- - pub(crate) ty_generics: Vec<TokenTree>,
- -}
- -
- -/// Parses the given `TokenStream` into `Generics` and the rest.
- -///
- -/// The generics are not present in the rest, but a where clause might remain.
- -pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) {
- - // The generics with bounds and default values.
- - let mut decl_generics = vec![];
- - // `impl_generics`, the declared generics with their bounds.
- - let mut impl_generics = vec![];
- - // Only the names of the generics, without any bounds.
- - let mut ty_generics = vec![];
- - // Tokens not related to the generics e.g. the `where` token and definition.
- - let mut rest = vec![];
- - // The current level of `<`.
- - let mut nesting = 0;
- - let mut toks = input.into_iter();
- - // If we are at the beginning of a generic parameter.
- - let mut at_start = true;
- - let mut skip_until_comma = false;
- - while let Some(tt) = toks.next() {
- - if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') {
- - // Found the end of the generics.
- - break;
- - } else if nesting >= 1 {
- - decl_generics.push(tt.clone());
- - }
- - match tt.clone() {
- - TokenTree::Punct(p) if p.as_char() == '<' => {
- - if nesting >= 1 && !skip_until_comma {
- - // This is inside of the generics and part of some bound.
- - impl_generics.push(tt);
- - }
- - nesting += 1;
- - }
- - TokenTree::Punct(p) if p.as_char() == '>' => {
- - // This is a parsing error, so we just end it here.
- - if nesting == 0 {
- - break;
- - } else {
- - nesting -= 1;
- - if nesting >= 1 && !skip_until_comma {
- - // We are still inside of the generics and part of some bound.
- - impl_generics.push(tt);
- - }
- - }
- - }
- - TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => {
- - if nesting == 1 {
- - impl_generics.push(tt.clone());
- - impl_generics.push(tt);
- - skip_until_comma = false;
- - }
- - }
- - _ if !skip_until_comma => {
- - match nesting {
- - // If we haven't entered the generics yet, we still want to keep these tokens.
- - 0 => rest.push(tt),
- - 1 => {
- - // Here depending on the token, it might be a generic variable name.
- - match tt.clone() {
- - TokenTree::Ident(i) if at_start && i.to_string() == "const" => {
- - let Some(name) = toks.next() else {
- - // Parsing error.
- - break;
- - };
- - impl_generics.push(tt);
- - impl_generics.push(name.clone());
- - ty_generics.push(name.clone());
- - decl_generics.push(name);
- - at_start = false;
- - }
- - TokenTree::Ident(_) if at_start => {
- - impl_generics.push(tt.clone());
- - ty_generics.push(tt);
- - at_start = false;
- - }
- - TokenTree::Punct(p) if p.as_char() == ',' => {
- - impl_generics.push(tt.clone());
- - ty_generics.push(tt);
- - at_start = true;
- - }
- - // Lifetimes begin with `'`.
- - TokenTree::Punct(p) if p.as_char() == '\'' && at_start => {
- - impl_generics.push(tt.clone());
- - ty_generics.push(tt);
- - }
- - // Generics can have default values, we skip these.
- - TokenTree::Punct(p) if p.as_char() == '=' => {
- - skip_until_comma = true;
- - }
- - _ => impl_generics.push(tt),
- - }
- - }
- - _ => impl_generics.push(tt),
- - }
- - }
- - _ => {}
- - }
- - }
- - rest.extend(toks);
- - (
- - Generics {
- - impl_generics,
- - decl_generics,
- - ty_generics,
- - },
- - rest,
- - )
- -}
- diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs
- deleted file mode 100644
- --- a/rust/macros/lib.rs
- +++ /dev/null
- @@ -1,395 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! Crate for all kernel procedural macros.
- -
- -#[macro_use]
- -mod quote;
- -mod concat_idents;
- -mod helpers;
- -mod module;
- -mod paste;
- -mod pin_data;
- -mod pinned_drop;
- -mod vtable;
- -mod zeroable;
- -
- -use proc_macro::TokenStream;
- -
- -/// Declares a kernel module.
- -///
- -/// The `type` argument should be a type which implements the [`Module`]
- -/// trait. Also accepts various forms of kernel metadata.
- -///
- -/// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h)
- -///
- -/// [`Module`]: ../kernel/trait.Module.html
- -///
- -/// # Examples
- -///
- -/// ```ignore
- -/// use kernel::prelude::*;
- -///
- -/// module!{
- -/// type: MyModule,
- -/// name: "my_kernel_module",
- -/// author: "Rust for Linux Contributors",
- -/// description: "My very own kernel module!",
- -/// license: "GPL",
- -/// }
- -///
- -/// struct MyModule;
- -///
- -/// impl kernel::Module for MyModule {
- -/// fn init() -> Result<Self> {
- -/// // If the parameter is writeable, then the kparam lock must be
- -/// // taken to read the parameter:
- -/// {
- -/// let lock = THIS_MODULE.kernel_param_lock();
- -/// pr_info!("i32 param is: {}\n", writeable_i32.read(&lock));
- -/// }
- -/// // If the parameter is read only, it can be read without locking
- -/// // the kernel parameters:
- -/// pr_info!("i32 param is: {}\n", my_i32.read());
- -/// Ok(Self)
- -/// }
- -/// }
- -/// ```
- -///
- -/// # Supported argument types
- -/// - `type`: type which implements the [`Module`] trait (required).
- -/// - `name`: byte array of the name of the kernel module (required).
- -/// - `author`: byte array of the author of the kernel module.
- -/// - `description`: byte array of the description of the kernel module.
- -/// - `license`: byte array of the license of the kernel module (required).
- -/// - `alias`: byte array of alias name of the kernel module.
- -#[proc_macro]
- -pub fn module(ts: TokenStream) -> TokenStream {
- - module::module(ts)
- -}
- -
- -/// Declares or implements a vtable trait.
- -///
- -/// Linux's use of pure vtables is very close to Rust traits, but they differ
- -/// in how unimplemented functions are represented. In Rust, traits can provide
- -/// default implementation for all non-required methods (and the default
- -/// implementation could just return `Error::EINVAL`); Linux typically use C
- -/// `NULL` pointers to represent these functions.
- -///
- -/// This attribute closes that gap. A trait can be annotated with the
- -/// `#[vtable]` attribute. Implementers of the trait will then also have to
- -/// annotate the trait with `#[vtable]`. This attribute generates a `HAS_*`
- -/// associated constant bool for each method in the trait that is set to true if
- -/// the implementer has overridden the associated method.
- -///
- -/// For a trait method to be optional, it must have a default implementation.
- -/// This is also the case for traits annotated with `#[vtable]`, but in this
- -/// case the default implementation will never be executed. The reason for this
- -/// is that the functions will be called through function pointers installed in
- -/// C side vtables. When an optional method is not implemented on a `#[vtable]`
- -/// trait, a NULL entry is installed in the vtable. Thus the default
- -/// implementation is never called. Since these traits are not designed to be
- -/// used on the Rust side, it should not be possible to call the default
- -/// implementation. This is done to ensure that we call the vtable methods
- -/// through the C vtable, and not through the Rust vtable. Therefore, the
- -/// default implementation should call `kernel::build_error`, which prevents
- -/// calls to this function at compile time:
- -///
- -/// ```compile_fail
- -/// # use kernel::error::VTABLE_DEFAULT_ERROR;
- -/// kernel::build_error(VTABLE_DEFAULT_ERROR)
- -/// ```
- -///
- -/// Note that you might need to import [`kernel::error::VTABLE_DEFAULT_ERROR`].
- -///
- -/// This macro should not be used when all functions are required.
- -///
- -/// # Examples
- -///
- -/// ```ignore
- -/// use kernel::error::VTABLE_DEFAULT_ERROR;
- -/// use kernel::prelude::*;
- -///
- -/// // Declares a `#[vtable]` trait
- -/// #[vtable]
- -/// pub trait Operations: Send + Sync + Sized {
- -/// fn foo(&self) -> Result<()> {
- -/// kernel::build_error(VTABLE_DEFAULT_ERROR)
- -/// }
- -///
- -/// fn bar(&self) -> Result<()> {
- -/// kernel::build_error(VTABLE_DEFAULT_ERROR)
- -/// }
- -/// }
- -///
- -/// struct Foo;
- -///
- -/// // Implements the `#[vtable]` trait
- -/// #[vtable]
- -/// impl Operations for Foo {
- -/// fn foo(&self) -> Result<()> {
- -/// # Err(EINVAL)
- -/// // ...
- -/// }
- -/// }
- -///
- -/// assert_eq!(<Foo as Operations>::HAS_FOO, true);
- -/// assert_eq!(<Foo as Operations>::HAS_BAR, false);
- -/// ```
- -///
- -/// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html
- -#[proc_macro_attribute]
- -pub fn vtable(attr: TokenStream, ts: TokenStream) -> TokenStream {
- - vtable::vtable(attr, ts)
- -}
- -
- -/// Concatenate two identifiers.
- -///
- -/// This is useful in macros that need to declare or reference items with names
- -/// starting with a fixed prefix and ending in a user specified name. The resulting
- -/// identifier has the span of the second argument.
- -///
- -/// # Examples
- -///
- -/// ```ignore
- -/// use kernel::macro::concat_idents;
- -///
- -/// macro_rules! pub_no_prefix {
- -/// ($prefix:ident, $($newname:ident),+) => {
- -/// $(pub(crate) const $newname: u32 = kernel::macros::concat_idents!($prefix, $newname);)+
- -/// };
- -/// }
- -///
- -/// pub_no_prefix!(
- -/// binder_driver_return_protocol_,
- -/// BR_OK,
- -/// BR_ERROR,
- -/// BR_TRANSACTION,
- -/// BR_REPLY,
- -/// BR_DEAD_REPLY,
- -/// BR_TRANSACTION_COMPLETE,
- -/// BR_INCREFS,
- -/// BR_ACQUIRE,
- -/// BR_RELEASE,
- -/// BR_DECREFS,
- -/// BR_NOOP,
- -/// BR_SPAWN_LOOPER,
- -/// BR_DEAD_BINDER,
- -/// BR_CLEAR_DEATH_NOTIFICATION_DONE,
- -/// BR_FAILED_REPLY
- -/// );
- -///
- -/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK);
- -/// ```
- -#[proc_macro]
- -pub fn concat_idents(ts: TokenStream) -> TokenStream {
- - concat_idents::concat_idents(ts)
- -}
- -
- -/// Used to specify the pinning information of the fields of a struct.
- -///
- -/// This is somewhat similar in purpose as
- -/// [pin-project-lite](https://crates.io/crates/pin-project-lite).
- -/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each
- -/// field you want to structurally pin.
- -///
- -/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`,
- -/// then `#[pin]` directs the type of initializer that is required.
- -///
- -/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this
- -/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with
- -/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care.
- -///
- -/// # Examples
- -///
- -/// ```rust,ignore
- -/// #[pin_data]
- -/// struct DriverData {
- -/// #[pin]
- -/// queue: Mutex<Vec<Command>>,
- -/// buf: Box<[u8; 1024 * 1024]>,
- -/// }
- -/// ```
- -///
- -/// ```rust,ignore
- -/// #[pin_data(PinnedDrop)]
- -/// struct DriverData {
- -/// #[pin]
- -/// queue: Mutex<Vec<Command>>,
- -/// buf: Box<[u8; 1024 * 1024]>,
- -/// raw_info: *mut Info,
- -/// }
- -///
- -/// #[pinned_drop]
- -/// impl PinnedDrop for DriverData {
- -/// fn drop(self: Pin<&mut Self>) {
- -/// unsafe { bindings::destroy_info(self.raw_info) };
- -/// }
- -/// }
- -/// ```
- -///
- -/// [`pin_init!`]: ../kernel/macro.pin_init.html
- -// ^ cannot use direct link, since `kernel` is not a dependency of `macros`.
- -#[proc_macro_attribute]
- -pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream {
- - pin_data::pin_data(inner, item)
- -}
- -
- -/// Used to implement `PinnedDrop` safely.
- -///
- -/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`.
- -///
- -/// # Examples
- -///
- -/// ```rust,ignore
- -/// #[pin_data(PinnedDrop)]
- -/// struct DriverData {
- -/// #[pin]
- -/// queue: Mutex<Vec<Command>>,
- -/// buf: Box<[u8; 1024 * 1024]>,
- -/// raw_info: *mut Info,
- -/// }
- -///
- -/// #[pinned_drop]
- -/// impl PinnedDrop for DriverData {
- -/// fn drop(self: Pin<&mut Self>) {
- -/// unsafe { bindings::destroy_info(self.raw_info) };
- -/// }
- -/// }
- -/// ```
- -#[proc_macro_attribute]
- -pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream {
- - pinned_drop::pinned_drop(args, input)
- -}
- -
- -/// Paste identifiers together.
- -///
- -/// Within the `paste!` macro, identifiers inside `[<` and `>]` are concatenated together to form a
- -/// single identifier.
- -///
- -/// This is similar to the [`paste`] crate, but with pasting feature limited to identifiers and
- -/// literals (lifetimes and documentation strings are not supported). There is a difference in
- -/// supported modifiers as well.
- -///
- -/// # Example
- -///
- -/// ```ignore
- -/// use kernel::macro::paste;
- -///
- -/// macro_rules! pub_no_prefix {
- -/// ($prefix:ident, $($newname:ident),+) => {
- -/// paste! {
- -/// $(pub(crate) const $newname: u32 = [<$prefix $newname>];)+
- -/// }
- -/// };
- -/// }
- -///
- -/// pub_no_prefix!(
- -/// binder_driver_return_protocol_,
- -/// BR_OK,
- -/// BR_ERROR,
- -/// BR_TRANSACTION,
- -/// BR_REPLY,
- -/// BR_DEAD_REPLY,
- -/// BR_TRANSACTION_COMPLETE,
- -/// BR_INCREFS,
- -/// BR_ACQUIRE,
- -/// BR_RELEASE,
- -/// BR_DECREFS,
- -/// BR_NOOP,
- -/// BR_SPAWN_LOOPER,
- -/// BR_DEAD_BINDER,
- -/// BR_CLEAR_DEATH_NOTIFICATION_DONE,
- -/// BR_FAILED_REPLY
- -/// );
- -///
- -/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK);
- -/// ```
- -///
- -/// # Modifiers
- -///
- -/// For each identifier, it is possible to attach one or multiple modifiers to
- -/// it.
- -///
- -/// Currently supported modifiers are:
- -/// * `span`: change the span of concatenated identifier to the span of the specified token. By
- -/// default the span of the `[< >]` group is used.
- -/// * `lower`: change the identifier to lower case.
- -/// * `upper`: change the identifier to upper case.
- -///
- -/// ```ignore
- -/// use kernel::macro::paste;
- -///
- -/// macro_rules! pub_no_prefix {
- -/// ($prefix:ident, $($newname:ident),+) => {
- -/// kernel::macros::paste! {
- -/// $(pub(crate) const fn [<$newname:lower:span>]: u32 = [<$prefix $newname:span>];)+
- -/// }
- -/// };
- -/// }
- -///
- -/// pub_no_prefix!(
- -/// binder_driver_return_protocol_,
- -/// BR_OK,
- -/// BR_ERROR,
- -/// BR_TRANSACTION,
- -/// BR_REPLY,
- -/// BR_DEAD_REPLY,
- -/// BR_TRANSACTION_COMPLETE,
- -/// BR_INCREFS,
- -/// BR_ACQUIRE,
- -/// BR_RELEASE,
- -/// BR_DECREFS,
- -/// BR_NOOP,
- -/// BR_SPAWN_LOOPER,
- -/// BR_DEAD_BINDER,
- -/// BR_CLEAR_DEATH_NOTIFICATION_DONE,
- -/// BR_FAILED_REPLY
- -/// );
- -///
- -/// assert_eq!(br_ok(), binder_driver_return_protocol_BR_OK);
- -/// ```
- -///
- -/// # Literals
- -///
- -/// Literals can also be concatenated with other identifiers:
- -///
- -/// ```ignore
- -/// macro_rules! create_numbered_fn {
- -/// ($name:literal, $val:literal) => {
- -/// kernel::macros::paste! {
- -/// fn [<some_ $name _fn $val>]() -> u32 { $val }
- -/// }
- -/// };
- -/// }
- -///
- -/// create_numbered_fn!("foo", 100);
- -///
- -/// assert_eq!(some_foo_fn100(), 100)
- -/// ```
- -///
- -/// [`paste`]: https://docs.rs/paste/
- -#[proc_macro]
- -pub fn paste(input: TokenStream) -> TokenStream {
- - let mut tokens = input.into_iter().collect();
- - paste::expand(&mut tokens);
- - tokens.into_iter().collect()
- -}
- -
- -/// Derives the [`Zeroable`] trait for the given struct.
- -///
- -/// This can only be used for structs where every field implements the [`Zeroable`] trait.
- -///
- -/// # Examples
- -///
- -/// ```rust,ignore
- -/// #[derive(Zeroable)]
- -/// pub struct DriverData {
- -/// id: i64,
- -/// buf_ptr: *mut u8,
- -/// len: usize,
- -/// }
- -/// ```
- -#[proc_macro_derive(Zeroable)]
- -pub fn derive_zeroable(input: TokenStream) -> TokenStream {
- - zeroable::derive(input)
- -}
- diff --git a/rust/macros/module.rs b/rust/macros/module.rs
- deleted file mode 100644
- --- a/rust/macros/module.rs
- +++ /dev/null
- @@ -1,351 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -use crate::helpers::*;
- -use proc_macro::{token_stream, Delimiter, Literal, TokenStream, TokenTree};
- -use std::fmt::Write;
- -
- -fn expect_string_array(it: &mut token_stream::IntoIter) -> Vec<String> {
- - let group = expect_group(it);
- - assert_eq!(group.delimiter(), Delimiter::Bracket);
- - let mut values = Vec::new();
- - let mut it = group.stream().into_iter();
- -
- - while let Some(val) = try_string(&mut it) {
- - assert!(val.is_ascii(), "Expected ASCII string");
- - values.push(val);
- - match it.next() {
- - Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','),
- - None => break,
- - _ => panic!("Expected ',' or end of array"),
- - }
- - }
- - values
- -}
- -
- -struct ModInfoBuilder<'a> {
- - module: &'a str,
- - counter: usize,
- - buffer: String,
- -}
- -
- -impl<'a> ModInfoBuilder<'a> {
- - fn new(module: &'a str) -> Self {
- - ModInfoBuilder {
- - module,
- - counter: 0,
- - buffer: String::new(),
- - }
- - }
- -
- - fn emit_base(&mut self, field: &str, content: &str, builtin: bool) {
- - let string = if builtin {
- - // Built-in modules prefix their modinfo strings by `module.`.
- - format!(
- - "{module}.{field}={content}\0",
- - module = self.module,
- - field = field,
- - content = content
- - )
- - } else {
- - // Loadable modules' modinfo strings go as-is.
- - format!("{field}={content}\0", field = field, content = content)
- - };
- -
- - write!(
- - &mut self.buffer,
- - "
- - {cfg}
- - #[doc(hidden)]
- - #[link_section = \".modinfo\"]
- - #[used]
- - pub static __{module}_{counter}: [u8; {length}] = *{string};
- - ",
- - cfg = if builtin {
- - "#[cfg(not(MODULE))]"
- - } else {
- - "#[cfg(MODULE)]"
- - },
- - module = self.module.to_uppercase(),
- - counter = self.counter,
- - length = string.len(),
- - string = Literal::byte_string(string.as_bytes()),
- - )
- - .unwrap();
- -
- - self.counter += 1;
- - }
- -
- - fn emit_only_builtin(&mut self, field: &str, content: &str) {
- - self.emit_base(field, content, true)
- - }
- -
- - fn emit_only_loadable(&mut self, field: &str, content: &str) {
- - self.emit_base(field, content, false)
- - }
- -
- - fn emit(&mut self, field: &str, content: &str) {
- - self.emit_only_builtin(field, content);
- - self.emit_only_loadable(field, content);
- - }
- -}
- -
- -#[derive(Debug, Default)]
- -struct ModuleInfo {
- - type_: String,
- - license: String,
- - name: String,
- - author: Option<String>,
- - description: Option<String>,
- - alias: Option<Vec<String>>,
- -}
- -
- -impl ModuleInfo {
- - fn parse(it: &mut token_stream::IntoIter) -> Self {
- - let mut info = ModuleInfo::default();
- -
- - const EXPECTED_KEYS: &[&str] =
- - &["type", "name", "author", "description", "license", "alias"];
- - const REQUIRED_KEYS: &[&str] = &["type", "name", "license"];
- - let mut seen_keys = Vec::new();
- -
- - loop {
- - let key = match it.next() {
- - Some(TokenTree::Ident(ident)) => ident.to_string(),
- - Some(_) => panic!("Expected Ident or end"),
- - None => break,
- - };
- -
- - if seen_keys.contains(&key) {
- - panic!(
- - "Duplicated key \"{}\". Keys can only be specified once.",
- - key
- - );
- - }
- -
- - assert_eq!(expect_punct(it), ':');
- -
- - match key.as_str() {
- - "type" => info.type_ = expect_ident(it),
- - "name" => info.name = expect_string_ascii(it),
- - "author" => info.author = Some(expect_string(it)),
- - "description" => info.description = Some(expect_string(it)),
- - "license" => info.license = expect_string_ascii(it),
- - "alias" => info.alias = Some(expect_string_array(it)),
- - _ => panic!(
- - "Unknown key \"{}\". Valid keys are: {:?}.",
- - key, EXPECTED_KEYS
- - ),
- - }
- -
- - assert_eq!(expect_punct(it), ',');
- -
- - seen_keys.push(key);
- - }
- -
- - expect_end(it);
- -
- - for key in REQUIRED_KEYS {
- - if !seen_keys.iter().any(|e| e == key) {
- - panic!("Missing required key \"{}\".", key);
- - }
- - }
- -
- - let mut ordered_keys: Vec<&str> = Vec::new();
- - for key in EXPECTED_KEYS {
- - if seen_keys.iter().any(|e| e == key) {
- - ordered_keys.push(key);
- - }
- - }
- -
- - if seen_keys != ordered_keys {
- - panic!(
- - "Keys are not ordered as expected. Order them like: {:?}.",
- - ordered_keys
- - );
- - }
- -
- - info
- - }
- -}
- -
- -pub(crate) fn module(ts: TokenStream) -> TokenStream {
- - let mut it = ts.into_iter();
- -
- - let info = ModuleInfo::parse(&mut it);
- -
- - let mut modinfo = ModInfoBuilder::new(info.name.as_ref());
- - if let Some(author) = info.author {
- - modinfo.emit("author", &author);
- - }
- - if let Some(description) = info.description {
- - modinfo.emit("description", &description);
- - }
- - modinfo.emit("license", &info.license);
- - if let Some(aliases) = info.alias {
- - for alias in aliases {
- - modinfo.emit("alias", &alias);
- - }
- - }
- -
- - // Built-in modules also export the `file` modinfo string.
- - let file =
- - std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable");
- - modinfo.emit_only_builtin("file", &file);
- -
- - format!(
- - "
- - /// The module name.
- - ///
- - /// Used by the printing macros, e.g. [`info!`].
- - const __LOG_PREFIX: &[u8] = b\"{name}\\0\";
- -
- - // SAFETY: `__this_module` is constructed by the kernel at load time and will not be
- - // freed until the module is unloaded.
- - #[cfg(MODULE)]
- - static THIS_MODULE: kernel::ThisModule = unsafe {{
- - extern \"C\" {{
- - static __this_module: kernel::types::Opaque<kernel::bindings::module>;
- - }}
- -
- - kernel::ThisModule::from_ptr(__this_module.get())
- - }};
- - #[cfg(not(MODULE))]
- - static THIS_MODULE: kernel::ThisModule = unsafe {{
- - kernel::ThisModule::from_ptr(core::ptr::null_mut())
- - }};
- -
- - // Double nested modules, since then nobody can access the public items inside.
- - mod __module_init {{
- - mod __module_init {{
- - use super::super::{type_};
- -
- - /// The \"Rust loadable module\" mark.
- - //
- - // This may be best done another way later on, e.g. as a new modinfo
- - // key or a new section. For the moment, keep it simple.
- - #[cfg(MODULE)]
- - #[doc(hidden)]
- - #[used]
- - static __IS_RUST_MODULE: () = ();
- -
- - static mut __MOD: Option<{type_}> = None;
- -
- - // Loadable modules need to export the `{{init,cleanup}}_module` identifiers.
- - /// # Safety
- - ///
- - /// This function must not be called after module initialization, because it may be
- - /// freed after that completes.
- - #[cfg(MODULE)]
- - #[doc(hidden)]
- - #[no_mangle]
- - #[link_section = \".init.text\"]
- - pub unsafe extern \"C\" fn init_module() -> core::ffi::c_int {{
- - // SAFETY: This function is inaccessible to the outside due to the double
- - // module wrapping it. It is called exactly once by the C side via its
- - // unique name.
- - unsafe {{ __init() }}
- - }}
- -
- - #[cfg(MODULE)]
- - #[doc(hidden)]
- - #[no_mangle]
- - pub extern \"C\" fn cleanup_module() {{
- - // SAFETY:
- - // - This function is inaccessible to the outside due to the double
- - // module wrapping it. It is called exactly once by the C side via its
- - // unique name,
- - // - furthermore it is only called after `init_module` has returned `0`
- - // (which delegates to `__init`).
- - unsafe {{ __exit() }}
- - }}
- -
- - // Built-in modules are initialized through an initcall pointer
- - // and the identifiers need to be unique.
- - #[cfg(not(MODULE))]
- - #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))]
- - #[doc(hidden)]
- - #[link_section = \"{initcall_section}\"]
- - #[used]
- - pub static __{name}_initcall: extern \"C\" fn() -> core::ffi::c_int = __{name}_init;
- -
- - #[cfg(not(MODULE))]
- - #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)]
- - core::arch::global_asm!(
- - r#\".section \"{initcall_section}\", \"a\"
- - __{name}_initcall:
- - .long __{name}_init - .
- - .previous
- - \"#
- - );
- -
- - #[cfg(not(MODULE))]
- - #[doc(hidden)]
- - #[no_mangle]
- - pub extern \"C\" fn __{name}_init() -> core::ffi::c_int {{
- - // SAFETY: This function is inaccessible to the outside due to the double
- - // module wrapping it. It is called exactly once by the C side via its
- - // placement above in the initcall section.
- - unsafe {{ __init() }}
- - }}
- -
- - #[cfg(not(MODULE))]
- - #[doc(hidden)]
- - #[no_mangle]
- - pub extern \"C\" fn __{name}_exit() {{
- - // SAFETY:
- - // - This function is inaccessible to the outside due to the double
- - // module wrapping it. It is called exactly once by the C side via its
- - // unique name,
- - // - furthermore it is only called after `__{name}_init` has returned `0`
- - // (which delegates to `__init`).
- - unsafe {{ __exit() }}
- - }}
- -
- - /// # Safety
- - ///
- - /// This function must only be called once.
- - unsafe fn __init() -> core::ffi::c_int {{
- - match <{type_} as kernel::Module>::init(&super::super::THIS_MODULE) {{
- - Ok(m) => {{
- - // SAFETY: No data race, since `__MOD` can only be accessed by this
- - // module and there only `__init` and `__exit` access it. These
- - // functions are only called once and `__exit` cannot be called
- - // before or during `__init`.
- - unsafe {{
- - __MOD = Some(m);
- - }}
- - return 0;
- - }}
- - Err(e) => {{
- - return e.to_errno();
- - }}
- - }}
- - }}
- -
- - /// # Safety
- - ///
- - /// This function must
- - /// - only be called once,
- - /// - be called after `__init` has been called and returned `0`.
- - unsafe fn __exit() {{
- - // SAFETY: No data race, since `__MOD` can only be accessed by this module
- - // and there only `__init` and `__exit` access it. These functions are only
- - // called once and `__init` was already called.
- - unsafe {{
- - // Invokes `drop()` on `__MOD`, which should be used for cleanup.
- - __MOD = None;
- - }}
- - }}
- -
- - {modinfo}
- - }}
- - }}
- - ",
- - type_ = info.type_,
- - name = info.name,
- - modinfo = modinfo.buffer,
- - initcall_section = ".initcall6.init"
- - )
- - .parse()
- - .expect("Error parsing formatted string into token stream.")
- -}
- diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs
- deleted file mode 100644
- --- a/rust/macros/paste.rs
- +++ /dev/null
- @@ -1,104 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -use proc_macro::{Delimiter, Group, Ident, Spacing, Span, TokenTree};
- -
- -fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree {
- - let mut tokens = tokens.iter();
- - let mut segments = Vec::new();
- - let mut span = None;
- - loop {
- - match tokens.next() {
- - None => break,
- - Some(TokenTree::Literal(lit)) => {
- - // Allow us to concat string literals by stripping quotes
- - let mut value = lit.to_string();
- - if value.starts_with('"') && value.ends_with('"') {
- - value.remove(0);
- - value.pop();
- - }
- - segments.push((value, lit.span()));
- - }
- - Some(TokenTree::Ident(ident)) => {
- - let mut value = ident.to_string();
- - if value.starts_with("r#") {
- - value.replace_range(0..2, "");
- - }
- - segments.push((value, ident.span()));
- - }
- - Some(TokenTree::Punct(p)) if p.as_char() == ':' => {
- - let Some(TokenTree::Ident(ident)) = tokens.next() else {
- - panic!("expected identifier as modifier");
- - };
- -
- - let (mut value, sp) = segments.pop().expect("expected identifier before modifier");
- - match ident.to_string().as_str() {
- - // Set the overall span of concatenated token as current span
- - "span" => {
- - assert!(
- - span.is_none(),
- - "span modifier should only appear at most once"
- - );
- - span = Some(sp);
- - }
- - "lower" => value = value.to_lowercase(),
- - "upper" => value = value.to_uppercase(),
- - v => panic!("unknown modifier `{v}`"),
- - };
- - segments.push((value, sp));
- - }
- - _ => panic!("unexpected token in paste segments"),
- - };
- - }
- -
- - let pasted: String = segments.into_iter().map(|x| x.0).collect();
- - TokenTree::Ident(Ident::new(&pasted, span.unwrap_or(group_span)))
- -}
- -
- -pub(crate) fn expand(tokens: &mut Vec<TokenTree>) {
- - for token in tokens.iter_mut() {
- - if let TokenTree::Group(group) = token {
- - let delimiter = group.delimiter();
- - let span = group.span();
- - let mut stream: Vec<_> = group.stream().into_iter().collect();
- - // Find groups that looks like `[< A B C D >]`
- - if delimiter == Delimiter::Bracket
- - && stream.len() >= 3
- - && matches!(&stream[0], TokenTree::Punct(p) if p.as_char() == '<')
- - && matches!(&stream[stream.len() - 1], TokenTree::Punct(p) if p.as_char() == '>')
- - {
- - // Replace the group with concatenated token
- - *token = concat(&stream[1..stream.len() - 1], span);
- - } else {
- - // Recursively expand tokens inside the group
- - expand(&mut stream);
- - let mut group = Group::new(delimiter, stream.into_iter().collect());
- - group.set_span(span);
- - *token = TokenTree::Group(group);
- - }
- - }
- - }
- -
- - // Path segments cannot contain invisible delimiter group, so remove them if any.
- - for i in (0..tokens.len().saturating_sub(3)).rev() {
- - // Looking for a double colon
- - if matches!(
- - (&tokens[i + 1], &tokens[i + 2]),
- - (TokenTree::Punct(a), TokenTree::Punct(b))
- - if a.as_char() == ':' && a.spacing() == Spacing::Joint && b.as_char() == ':'
- - ) {
- - match &tokens[i + 3] {
- - TokenTree::Group(group) if group.delimiter() == Delimiter::None => {
- - tokens.splice(i + 3..i + 4, group.stream());
- - }
- - _ => (),
- - }
- -
- - match &tokens[i] {
- - TokenTree::Group(group) if group.delimiter() == Delimiter::None => {
- - tokens.splice(i..i + 1, group.stream());
- - }
- - _ => (),
- - }
- - }
- - }
- -}
- diff --git a/rust/macros/pin_data.rs b/rust/macros/pin_data.rs
- deleted file mode 100644
- --- a/rust/macros/pin_data.rs
- +++ /dev/null
- @@ -1,129 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -use crate::helpers::{parse_generics, Generics};
- -use proc_macro::{Group, Punct, Spacing, TokenStream, TokenTree};
- -
- -pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream {
- - // This proc-macro only does some pre-parsing and then delegates the actual parsing to
- - // `kernel::__pin_data!`.
- -
- - let (
- - Generics {
- - impl_generics,
- - decl_generics,
- - ty_generics,
- - },
- - rest,
- - ) = parse_generics(input);
- - // The struct definition might contain the `Self` type. Since `__pin_data!` will define a new
- - // type with the same generics and bounds, this poses a problem, since `Self` will refer to the
- - // new type as opposed to this struct definition. Therefore we have to replace `Self` with the
- - // concrete name.
- -
- - // Errors that occur when replacing `Self` with `struct_name`.
- - let mut errs = TokenStream::new();
- - // The name of the struct with ty_generics.
- - let struct_name = rest
- - .iter()
- - .skip_while(|tt| !matches!(tt, TokenTree::Ident(i) if i.to_string() == "struct"))
- - .nth(1)
- - .and_then(|tt| match tt {
- - TokenTree::Ident(_) => {
- - let tt = tt.clone();
- - let mut res = vec![tt];
- - if !ty_generics.is_empty() {
- - // We add this, so it is maximally compatible with e.g. `Self::CONST` which
- - // will be replaced by `StructName::<$generics>::CONST`.
- - res.push(TokenTree::Punct(Punct::new(':', Spacing::Joint)));
- - res.push(TokenTree::Punct(Punct::new(':', Spacing::Alone)));
- - res.push(TokenTree::Punct(Punct::new('<', Spacing::Alone)));
- - res.extend(ty_generics.iter().cloned());
- - res.push(TokenTree::Punct(Punct::new('>', Spacing::Alone)));
- - }
- - Some(res)
- - }
- - _ => None,
- - })
- - .unwrap_or_else(|| {
- - // If we did not find the name of the struct then we will use `Self` as the replacement
- - // and add a compile error to ensure it does not compile.
- - errs.extend(
- - "::core::compile_error!(\"Could not locate type name.\");"
- - .parse::<TokenStream>()
- - .unwrap(),
- - );
- - "Self".parse::<TokenStream>().unwrap().into_iter().collect()
- - });
- - let impl_generics = impl_generics
- - .into_iter()
- - .flat_map(|tt| replace_self_and_deny_type_defs(&struct_name, tt, &mut errs))
- - .collect::<Vec<_>>();
- - let mut rest = rest
- - .into_iter()
- - .flat_map(|tt| {
- - // We ignore top level `struct` tokens, since they would emit a compile error.
- - if matches!(&tt, TokenTree::Ident(i) if i.to_string() == "struct") {
- - vec![tt]
- - } else {
- - replace_self_and_deny_type_defs(&struct_name, tt, &mut errs)
- - }
- - })
- - .collect::<Vec<_>>();
- - // This should be the body of the struct `{...}`.
- - let last = rest.pop();
- - let mut quoted = quote!(::kernel::__pin_data! {
- - parse_input:
- - @args(#args),
- - @sig(#(#rest)*),
- - @impl_generics(#(#impl_generics)*),
- - @ty_generics(#(#ty_generics)*),
- - @decl_generics(#(#decl_generics)*),
- - @body(#last),
- - });
- - quoted.extend(errs);
- - quoted
- -}
- -
- -/// Replaces `Self` with `struct_name` and errors on `enum`, `trait`, `struct` `union` and `impl`
- -/// keywords.
- -///
- -/// The error is appended to `errs` to allow normal parsing to continue.
- -fn replace_self_and_deny_type_defs(
- - struct_name: &Vec<TokenTree>,
- - tt: TokenTree,
- - errs: &mut TokenStream,
- -) -> Vec<TokenTree> {
- - match tt {
- - TokenTree::Ident(ref i)
- - if i.to_string() == "enum"
- - || i.to_string() == "trait"
- - || i.to_string() == "struct"
- - || i.to_string() == "union"
- - || i.to_string() == "impl" =>
- - {
- - errs.extend(
- - format!(
- - "::core::compile_error!(\"Cannot use `{i}` inside of struct definition with \
- - `#[pin_data]`.\");"
- - )
- - .parse::<TokenStream>()
- - .unwrap()
- - .into_iter()
- - .map(|mut tok| {
- - tok.set_span(tt.span());
- - tok
- - }),
- - );
- - vec![tt]
- - }
- - TokenTree::Ident(i) if i.to_string() == "Self" => struct_name.clone(),
- - TokenTree::Literal(_) | TokenTree::Punct(_) | TokenTree::Ident(_) => vec![tt],
- - TokenTree::Group(g) => vec![TokenTree::Group(Group::new(
- - g.delimiter(),
- - g.stream()
- - .into_iter()
- - .flat_map(|tt| replace_self_and_deny_type_defs(struct_name, tt, errs))
- - .collect(),
- - ))],
- - }
- -}
- diff --git a/rust/macros/pinned_drop.rs b/rust/macros/pinned_drop.rs
- deleted file mode 100644
- --- a/rust/macros/pinned_drop.rs
- +++ /dev/null
- @@ -1,49 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -use proc_macro::{TokenStream, TokenTree};
- -
- -pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream {
- - let mut toks = input.into_iter().collect::<Vec<_>>();
- - assert!(!toks.is_empty());
- - // Ensure that we have an `impl` item.
- - assert!(matches!(&toks[0], TokenTree::Ident(i) if i.to_string() == "impl"));
- - // Ensure that we are implementing `PinnedDrop`.
- - let mut nesting: usize = 0;
- - let mut pinned_drop_idx = None;
- - for (i, tt) in toks.iter().enumerate() {
- - match tt {
- - TokenTree::Punct(p) if p.as_char() == '<' => {
- - nesting += 1;
- - }
- - TokenTree::Punct(p) if p.as_char() == '>' => {
- - nesting = nesting.checked_sub(1).unwrap();
- - continue;
- - }
- - _ => {}
- - }
- - if i >= 1 && nesting == 0 {
- - // Found the end of the generics, this should be `PinnedDrop`.
- - assert!(
- - matches!(tt, TokenTree::Ident(i) if i.to_string() == "PinnedDrop"),
- - "expected 'PinnedDrop', found: '{:?}'",
- - tt
- - );
- - pinned_drop_idx = Some(i);
- - break;
- - }
- - }
- - let idx = pinned_drop_idx
- - .unwrap_or_else(|| panic!("Expected an `impl` block implementing `PinnedDrop`."));
- - // Fully qualify the `PinnedDrop`, as to avoid any tampering.
- - toks.splice(idx..idx, quote!(::kernel::init::));
- - // Take the `{}` body and call the declarative macro.
- - if let Some(TokenTree::Group(last)) = toks.pop() {
- - let last = last.stream();
- - quote!(::kernel::__pinned_drop! {
- - @impl_sig(#(#toks)*),
- - @impl_body(#last),
- - })
- - } else {
- - TokenStream::from_iter(toks)
- - }
- -}
- diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs
- deleted file mode 100644
- --- a/rust/macros/quote.rs
- +++ /dev/null
- @@ -1,157 +0,0 @@
- -// SPDX-License-Identifier: Apache-2.0 OR MIT
- -
- -use proc_macro::{TokenStream, TokenTree};
- -
- -pub(crate) trait ToTokens {
- - fn to_tokens(&self, tokens: &mut TokenStream);
- -}
- -
- -impl<T: ToTokens> ToTokens for Option<T> {
- - fn to_tokens(&self, tokens: &mut TokenStream) {
- - if let Some(v) = self {
- - v.to_tokens(tokens);
- - }
- - }
- -}
- -
- -impl ToTokens for proc_macro::Group {
- - fn to_tokens(&self, tokens: &mut TokenStream) {
- - tokens.extend([TokenTree::from(self.clone())]);
- - }
- -}
- -
- -impl ToTokens for TokenTree {
- - fn to_tokens(&self, tokens: &mut TokenStream) {
- - tokens.extend([self.clone()]);
- - }
- -}
- -
- -impl ToTokens for TokenStream {
- - fn to_tokens(&self, tokens: &mut TokenStream) {
- - tokens.extend(self.clone());
- - }
- -}
- -
- -/// Converts tokens into [`proc_macro::TokenStream`] and performs variable interpolations with
- -/// the given span.
- -///
- -/// This is a similar to the
- -/// [`quote_spanned!`](https://docs.rs/quote/latest/quote/macro.quote_spanned.html) macro from the
- -/// `quote` crate but provides only just enough functionality needed by the current `macros` crate.
- -macro_rules! quote_spanned {
- - ($span:expr => $($tt:tt)*) => {{
- - let mut tokens;
- - #[allow(clippy::vec_init_then_push)]
- - {
- - tokens = ::std::vec::Vec::new();
- - let span = $span;
- - quote_spanned!(@proc tokens span $($tt)*);
- - }
- - ::proc_macro::TokenStream::from_iter(tokens)
- - }};
- - (@proc $v:ident $span:ident) => {};
- - (@proc $v:ident $span:ident #$id:ident $($tt:tt)*) => {
- - let mut ts = ::proc_macro::TokenStream::new();
- - $crate::quote::ToTokens::to_tokens(&$id, &mut ts);
- - $v.extend(ts);
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident #(#$id:ident)* $($tt:tt)*) => {
- - for token in $id {
- - let mut ts = ::proc_macro::TokenStream::new();
- - $crate::quote::ToTokens::to_tokens(&token, &mut ts);
- - $v.extend(ts);
- - }
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident ( $($inner:tt)* ) $($tt:tt)*) => {
- - let mut tokens = ::std::vec::Vec::new();
- - quote_spanned!(@proc tokens $span $($inner)*);
- - $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new(
- - ::proc_macro::Delimiter::Parenthesis,
- - ::proc_macro::TokenStream::from_iter(tokens)
- - )));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident [ $($inner:tt)* ] $($tt:tt)*) => {
- - let mut tokens = ::std::vec::Vec::new();
- - quote_spanned!(@proc tokens $span $($inner)*);
- - $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new(
- - ::proc_macro::Delimiter::Bracket,
- - ::proc_macro::TokenStream::from_iter(tokens)
- - )));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident { $($inner:tt)* } $($tt:tt)*) => {
- - let mut tokens = ::std::vec::Vec::new();
- - quote_spanned!(@proc tokens $span $($inner)*);
- - $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new(
- - ::proc_macro::Delimiter::Brace,
- - ::proc_macro::TokenStream::from_iter(tokens)
- - )));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident :: $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Joint)
- - ));
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Alone)
- - ));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident : $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Alone)
- - ));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident , $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new(',', ::proc_macro::Spacing::Alone)
- - ));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident @ $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new('@', ::proc_macro::Spacing::Alone)
- - ));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident ! $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new('!', ::proc_macro::Spacing::Alone)
- - ));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident ; $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new(';', ::proc_macro::Spacing::Alone)
- - ));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident + $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Punct(
- - ::proc_macro::Punct::new('+', ::proc_macro::Spacing::Alone)
- - ));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- - (@proc $v:ident $span:ident $id:ident $($tt:tt)*) => {
- - $v.push(::proc_macro::TokenTree::Ident(::proc_macro::Ident::new(stringify!($id), $span)));
- - quote_spanned!(@proc $v $span $($tt)*);
- - };
- -}
- -
- -/// Converts tokens into [`proc_macro::TokenStream`] and performs variable interpolations with
- -/// mixed site span ([`Span::mixed_site()`]).
- -///
- -/// This is a similar to the [`quote!`](https://docs.rs/quote/latest/quote/macro.quote.html) macro
- -/// from the `quote` crate but provides only just enough functionality needed by the current
- -/// `macros` crate.
- -///
- -/// [`Span::mixed_site()`]: https://doc.rust-lang.org/proc_macro/struct.Span.html#method.mixed_site
- -macro_rules! quote {
- - ($($tt:tt)*) => {
- - quote_spanned!(::proc_macro::Span::mixed_site() => $($tt)*)
- - }
- -}
- diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs
- deleted file mode 100644
- --- a/rust/macros/vtable.rs
- +++ /dev/null
- @@ -1,96 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -use proc_macro::{Delimiter, Group, TokenStream, TokenTree};
- -use std::collections::HashSet;
- -use std::fmt::Write;
- -
- -pub(crate) fn vtable(_attr: TokenStream, ts: TokenStream) -> TokenStream {
- - let mut tokens: Vec<_> = ts.into_iter().collect();
- -
- - // Scan for the `trait` or `impl` keyword.
- - let is_trait = tokens
- - .iter()
- - .find_map(|token| match token {
- - TokenTree::Ident(ident) => match ident.to_string().as_str() {
- - "trait" => Some(true),
- - "impl" => Some(false),
- - _ => None,
- - },
- - _ => None,
- - })
- - .expect("#[vtable] attribute should only be applied to trait or impl block");
- -
- - // Retrieve the main body. The main body should be the last token tree.
- - let body = match tokens.pop() {
- - Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::Brace => group,
- - _ => panic!("cannot locate main body of trait or impl block"),
- - };
- -
- - let mut body_it = body.stream().into_iter();
- - let mut functions = Vec::new();
- - let mut consts = HashSet::new();
- - while let Some(token) = body_it.next() {
- - match token {
- - TokenTree::Ident(ident) if ident.to_string() == "fn" => {
- - let fn_name = match body_it.next() {
- - Some(TokenTree::Ident(ident)) => ident.to_string(),
- - // Possibly we've encountered a fn pointer type instead.
- - _ => continue,
- - };
- - functions.push(fn_name);
- - }
- - TokenTree::Ident(ident) if ident.to_string() == "const" => {
- - let const_name = match body_it.next() {
- - Some(TokenTree::Ident(ident)) => ident.to_string(),
- - // Possibly we've encountered an inline const block instead.
- - _ => continue,
- - };
- - consts.insert(const_name);
- - }
- - _ => (),
- - }
- - }
- -
- - let mut const_items;
- - if is_trait {
- - const_items = "
- - /// A marker to prevent implementors from forgetting to use [`#[vtable]`](vtable)
- - /// attribute when implementing this trait.
- - const USE_VTABLE_ATTR: ();
- - "
- - .to_owned();
- -
- - for f in functions {
- - let gen_const_name = format!("HAS_{}", f.to_uppercase());
- - // Skip if it's declared already -- this allows user override.
- - if consts.contains(&gen_const_name) {
- - continue;
- - }
- - // We don't know on the implementation-site whether a method is required or provided
- - // so we have to generate a const for all methods.
- - write!(
- - const_items,
- - "/// Indicates if the `{f}` method is overridden by the implementor.
- - const {gen_const_name}: bool = false;",
- - )
- - .unwrap();
- - consts.insert(gen_const_name);
- - }
- - } else {
- - const_items = "const USE_VTABLE_ATTR: () = ();".to_owned();
- -
- - for f in functions {
- - let gen_const_name = format!("HAS_{}", f.to_uppercase());
- - if consts.contains(&gen_const_name) {
- - continue;
- - }
- - write!(const_items, "const {gen_const_name}: bool = true;").unwrap();
- - }
- - }
- -
- - let new_body = vec![const_items.parse().unwrap(), body.stream()]
- - .into_iter()
- - .collect();
- - tokens.push(TokenTree::Group(Group::new(Delimiter::Brace, new_body)));
- - tokens.into_iter().collect()
- -}
- diff --git a/rust/macros/zeroable.rs b/rust/macros/zeroable.rs
- deleted file mode 100644
- --- a/rust/macros/zeroable.rs
- +++ /dev/null
- @@ -1,73 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -use crate::helpers::{parse_generics, Generics};
- -use proc_macro::{TokenStream, TokenTree};
- -
- -pub(crate) fn derive(input: TokenStream) -> TokenStream {
- - let (
- - Generics {
- - impl_generics,
- - decl_generics: _,
- - ty_generics,
- - },
- - mut rest,
- - ) = parse_generics(input);
- - // This should be the body of the struct `{...}`.
- - let last = rest.pop();
- - // Now we insert `Zeroable` as a bound for every generic parameter in `impl_generics`.
- - let mut new_impl_generics = Vec::with_capacity(impl_generics.len());
- - // Are we inside of a generic where we want to add `Zeroable`?
- - let mut in_generic = !impl_generics.is_empty();
- - // Have we already inserted `Zeroable`?
- - let mut inserted = false;
- - // Level of `<>` nestings.
- - let mut nested = 0;
- - for tt in impl_generics {
- - match &tt {
- - // If we find a `,`, then we have finished a generic/constant/lifetime parameter.
- - TokenTree::Punct(p) if nested == 0 && p.as_char() == ',' => {
- - if in_generic && !inserted {
- - new_impl_generics.extend(quote! { : ::kernel::init::Zeroable });
- - }
- - in_generic = true;
- - inserted = false;
- - new_impl_generics.push(tt);
- - }
- - // If we find `'`, then we are entering a lifetime.
- - TokenTree::Punct(p) if nested == 0 && p.as_char() == '\'' => {
- - in_generic = false;
- - new_impl_generics.push(tt);
- - }
- - TokenTree::Punct(p) if nested == 0 && p.as_char() == ':' => {
- - new_impl_generics.push(tt);
- - if in_generic {
- - new_impl_generics.extend(quote! { ::kernel::init::Zeroable + });
- - inserted = true;
- - }
- - }
- - TokenTree::Punct(p) if p.as_char() == '<' => {
- - nested += 1;
- - new_impl_generics.push(tt);
- - }
- - TokenTree::Punct(p) if p.as_char() == '>' => {
- - assert!(nested > 0);
- - nested -= 1;
- - new_impl_generics.push(tt);
- - }
- - _ => new_impl_generics.push(tt),
- - }
- - }
- - assert_eq!(nested, 0);
- - if in_generic && !inserted {
- - new_impl_generics.extend(quote! { : ::kernel::init::Zeroable });
- - }
- - quote! {
- - ::kernel::__derive_zeroable!(
- - parse_input:
- - @sig(#(#rest)*),
- - @impl_generics(#(#new_impl_generics)*),
- - @ty_generics(#(#ty_generics)*),
- - @body(#last),
- - );
- - }
- -}
- diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs
- deleted file mode 100644
- --- a/rust/uapi/lib.rs
- +++ /dev/null
- @@ -1,26 +0,0 @@
- -// SPDX-License-Identifier: GPL-2.0
- -
- -//! UAPI Bindings.
- -//!
- -//! Contains the bindings generated by `bindgen` for UAPI interfaces.
- -//!
- -//! This crate may be used directly by drivers that need to interact with
- -//! userspace APIs.
- -
- -#![no_std]
- -// See <https://github.com/rust-lang/rust-bindgen/issues/1651>.
- -#![cfg_attr(test, allow(deref_nullptr))]
- -#![cfg_attr(test, allow(unaligned_references))]
- -#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))]
- -#![allow(
- - clippy::all,
- - missing_docs,
- - non_camel_case_types,
- - non_upper_case_globals,
- - non_snake_case,
- - improper_ctypes,
- - unreachable_pub,
- - unsafe_op_in_unsafe_fn
- -)]
- -
- -include!(concat!(env!("OBJTREE"), "/rust/uapi/uapi_generated.rs"));
- diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h
- deleted file mode 100644
- --- a/rust/uapi/uapi_helper.h
- +++ /dev/null
- @@ -1,11 +0,0 @@
- -/* SPDX-License-Identifier: GPL-2.0 */
- -/*
- - * Header that contains the headers for which Rust UAPI bindings
- - * will be automatically generated by `bindgen`.
- - *
- - * Sorted alphabetically.
- - */
- -
- -#include <uapi/asm-generic/ioctl.h>
- -#include <uapi/linux/mii.h>
- -#include <uapi/linux/ethtool.h>
|