1
0

edma3.c 59 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606
  1. /*++
  2. Copyright (c) 2016 Minoca Corp. All Rights Reserved
  3. Module Name:
  4. edma3.c
  5. Abstract:
  6. This module implements the TI EDMA3 controller driver.
  7. Author:
  8. Evan Green 2-Feb-2016
  9. Environment:
  10. Kernel
  11. --*/
  12. //
  13. // ------------------------------------------------------------------- Includes
  14. //
  15. #include <minoca/kernel/driver.h>
  16. #include <minoca/dma/dmahost.h>
  17. #include <minoca/dma/edma3.h>
  18. //
  19. // --------------------------------------------------------------------- Macros
  20. //
  21. //
  22. // These macros read and write to registers in the global region.
  23. //
  24. #define EDMA_READ(_Controller, _Register) \
  25. HlReadRegister32((_Controller)->ControllerBase + (_Register))
  26. #define EDMA_WRITE(_Controller, _Register, _Value) \
  27. HlWriteRegister32((_Controller)->ControllerBase + (_Register), (_Value))
  28. #define EDMA_READ64(_Controller, _Register) \
  29. ((ULONGLONG)(EDMA_READ((_Controller), (_Register))) | \
  30. ((ULONGLONG)(EDMA_READ((_Controller), (_Register) + 4)) << 32))
  31. #define EDMA_WRITE64(_Controller, _Register, _Value) \
  32. EDMA_WRITE((_Controller), (_Register), (ULONG)(_Value)), \
  33. EDMA_WRITE((_Controller), (_Register) + 4, (ULONG)((_Value) >> 32))
  34. //
  35. // These macros read and write to registers in the shadow regions.
  36. //
  37. #define EDMA_REGION_READ(_Controller, _Register) \
  38. EDMA_READ((_Controller), \
  39. ((_Register) + 0x1000 + (0x200 * (_Controller)->Region)))
  40. #define EDMA_REGION_WRITE(_Controller, _Register, _Value) \
  41. EDMA_WRITE((_Controller), \
  42. ((_Register) + 0x1000 + (0x200 * (_Controller)->Region)), \
  43. (_Value))
  44. #define EDMA_REGION_READ64(_Controller, _Register) \
  45. ((ULONGLONG)(EDMA_REGION_READ((_Controller), (_Register))) | \
  46. ((ULONGLONG)(EDMA_REGION_READ((_Controller), (_Register) + 4)) << 32))
  47. #define EDMA_REGION_WRITE64(_Controller, _Register, _Value) \
  48. EDMA_REGION_WRITE((_Controller), (_Register), (ULONG)(_Value)), \
  49. EDMA_REGION_WRITE((_Controller), (_Register) + 4, (ULONG)((_Value) >> 32))
  50. //
  51. // ---------------------------------------------------------------- Definitions
  52. //
  53. #define EDMA_ALLOCATION_TAG 0x616D4445
  54. #define EDMA_PARAM_WORDS (EDMA_PARAM_COUNT / (sizeof(UINTN) * BITS_PER_BYTE))
  55. #define EDMA_TRANSFER_PARAMS 32
  56. //
  57. // ------------------------------------------------------ Data Type Definitions
  58. //
  59. /*++
  60. Structure Description:
  61. This structure defines the context for an EDMA3 transfer.
  62. Members:
  63. Transfer - Stores a pointer to the DMA transfer.
  64. Params - Stores the array of PaRAM slots allocated for this transfer.
  65. ParamCount - Stores the number of valid entries in the PaRAMs array.
  66. BytesPending - Stores the size of the currently outstanding request.
  67. --*/
  68. typedef struct _EDMA_TRANSFER {
  69. PDMA_TRANSFER Transfer;
  70. UCHAR Params[EDMA_TRANSFER_PARAMS];
  71. ULONG ParamCount;
  72. UINTN BytesPending;
  73. } EDMA_TRANSFER, *PEDMA_TRANSFER;
  74. /*++
  75. Structure Description:
  76. This structure defines the set of pending interrupts in the controller.
  77. Members:
  78. CompletionLow - Stores the pending completion interrupts for first 32
  79. channels.
  80. CompletionHigh - Stores the pending completion interrupts for the upper
  81. 32 channels.
  82. MissedLow - Stores the pending missed event interrupts for the lower 32
  83. channels.
  84. MissedHigh - Stores the pending missed event interrupts for the upper
  85. 32 channels.
  86. MissedQuick - Stores the pending missed quick event DMA interrupts.
  87. Error - Stores the pending error interrupts.
  88. --*/
  89. typedef struct _EDMA_PENDING_INTERRUPTS {
  90. ULONG CompletionLow;
  91. ULONG CompletionHigh;
  92. ULONG MissedLow;
  93. ULONG MissedHigh;
  94. ULONG MissedQuick;
  95. ULONG Error;
  96. } EDMA_PENDING_INTERRUPTS, *PEDMA_PENDING_INTERRUPTS;
  97. /*++
  98. Structure Description:
  99. This structure defines the context for an EDMA3 controller.
  100. Members:
  101. OsDevice - Stores a pointer to the OS device object.
  102. CompletionInterruptLine - Stores the interrupt line that this controller's
  103. completion interrupt comes in on.
  104. CompletionInterruptVector - Stores the interrupt vector that this
  105. controller's completion interrupt comes in on.
  106. CompletionInterruptHandle - Stores a pointer to the handle received when the
  107. completion interrupt was connected.
  108. ErrorInterruptLine - Stores the interrupt line that this controller's
  109. error interrupt comes in on.
  110. ErrorInterruptVector - Stores the interrupt vector that this controller's
  111. error interrupt comes in on.
  112. ErrorInterruptHandle - Stores a pointer to the handle received when the
  113. error interrupt was connected.
  114. ControllerBase - Stores the virtual address of the memory mapping to the
  115. EDMA3 registers.
  116. DmaController - Stores a pointer to the library DMA controller.
  117. Lock - Stores the lock serializing access to the sensitive parts of the
  118. structure.
  119. TransferList - Stores the head of the list of transfers.
  120. FreeList - Stores the head of the list of transfer structures that are
  121. allocated but not currently used.
  122. Params - Stores the bitmap of allocated PaRAM entries.
  123. Pending - Stores the pending interrupt flags.
  124. Region - Stores the shadow region identifier that the processor is
  125. connected to.
  126. Transfers - Stores an array of points to EDMA transfers. One for each
  127. channel.
  128. --*/
  129. typedef struct _EDMA_CONTROLLER {
  130. PDEVICE OsDevice;
  131. ULONGLONG CompletionInterruptLine;
  132. ULONGLONG CompletionInterruptVector;
  133. HANDLE CompletionInterruptHandle;
  134. ULONGLONG ErrorInterruptLine;
  135. ULONGLONG ErrorInterruptVector;
  136. HANDLE ErrorInterruptHandle;
  137. PVOID ControllerBase;
  138. PDMA_CONTROLLER DmaController;
  139. KSPIN_LOCK Lock;
  140. UINTN Params[EDMA_PARAM_WORDS];
  141. EDMA_PENDING_INTERRUPTS Pending;
  142. UCHAR Region;
  143. PEDMA_TRANSFER Transfers[EDMA_CHANNEL_COUNT];
  144. } EDMA_CONTROLLER, *PEDMA_CONTROLLER;
  145. //
  146. // ----------------------------------------------- Internal Function Prototypes
  147. //
  148. KSTATUS
  149. EdmaAddDevice (
  150. PVOID Driver,
  151. PSTR DeviceId,
  152. PSTR ClassId,
  153. PSTR CompatibleIds,
  154. PVOID DeviceToken
  155. );
  156. VOID
  157. EdmaDispatchStateChange (
  158. PIRP Irp,
  159. PVOID DeviceContext,
  160. PVOID IrpContext
  161. );
  162. VOID
  163. EdmaDispatchOpen (
  164. PIRP Irp,
  165. PVOID DeviceContext,
  166. PVOID IrpContext
  167. );
  168. VOID
  169. EdmaDispatchClose (
  170. PIRP Irp,
  171. PVOID DeviceContext,
  172. PVOID IrpContext
  173. );
  174. VOID
  175. EdmaDispatchIo (
  176. PIRP Irp,
  177. PVOID DeviceContext,
  178. PVOID IrpContext
  179. );
  180. VOID
  181. EdmaDispatchSystemControl (
  182. PIRP Irp,
  183. PVOID DeviceContext,
  184. PVOID IrpContext
  185. );
  186. INTERRUPT_STATUS
  187. EdmaCompletionInterruptService (
  188. PVOID Context
  189. );
  190. INTERRUPT_STATUS
  191. EdmaErrorInterruptService (
  192. PVOID Context
  193. );
  194. INTERRUPT_STATUS
  195. EdmaInterruptServiceDispatch (
  196. PVOID Context
  197. );
  198. KSTATUS
  199. EdmaProcessResourceRequirements (
  200. PIRP Irp
  201. );
  202. KSTATUS
  203. EdmaStartDevice (
  204. PIRP Irp,
  205. PEDMA_CONTROLLER Device
  206. );
  207. KSTATUS
  208. EdmaSubmit (
  209. PVOID Context,
  210. PDMA_TRANSFER Transfer
  211. );
  212. KSTATUS
  213. EdmaCancel (
  214. PVOID Context,
  215. PDMA_TRANSFER Transfer
  216. );
  217. VOID
  218. EdmapControllerReset (
  219. PEDMA_CONTROLLER Controller
  220. );
  221. KSTATUS
  222. EdmapPrepareAndSubmitTransfer (
  223. PEDMA_CONTROLLER Controller,
  224. PEDMA_TRANSFER Transfer
  225. );
  226. KSTATUS
  227. EdmapPrepareTransfer (
  228. PEDMA_CONTROLLER Controller,
  229. PEDMA_TRANSFER Transfer
  230. );
  231. KSTATUS
  232. EdmapSubmitTransfer (
  233. PEDMA_CONTROLLER Controller,
  234. PEDMA_TRANSFER Transfer
  235. );
  236. KSTATUS
  237. EdmapSetupParam (
  238. PEDMA_CONTROLLER Controller,
  239. PEDMA_TRANSFER Transfer,
  240. UCHAR ParamIndex,
  241. PHYSICAL_ADDRESS MemoryAddress,
  242. PHYSICAL_ADDRESS DeviceAddress,
  243. ULONG Size,
  244. BOOL LastOne
  245. );
  246. VOID
  247. EdmapProcessCompletedTransfer (
  248. PEDMA_CONTROLLER Controller,
  249. ULONG Channel,
  250. BOOL MissedEvent
  251. );
  252. VOID
  253. EdmapTearDownChannel (
  254. PEDMA_CONTROLLER Controller,
  255. ULONG Channel
  256. );
  257. VOID
  258. EdmapResetTransfer (
  259. PEDMA_CONTROLLER Controller,
  260. PEDMA_TRANSFER Transfer
  261. );
  262. UCHAR
  263. EdmapAllocateParam (
  264. PEDMA_CONTROLLER Controller
  265. );
  266. VOID
  267. EdmapFreeParam (
  268. PEDMA_CONTROLLER Controller,
  269. UCHAR Param
  270. );
  271. VOID
  272. EdmapGetParam (
  273. PEDMA_CONTROLLER Controller,
  274. UCHAR Index,
  275. PEDMA_PARAM Param
  276. );
  277. VOID
  278. EdmapSetParam (
  279. PEDMA_CONTROLLER Controller,
  280. UCHAR Index,
  281. PEDMA_PARAM Param
  282. );
  283. VOID
  284. EdmapClearMissEvent (
  285. PEDMA_CONTROLLER Controller,
  286. ULONG Channel
  287. );
  288. RUNLEVEL
  289. EdmapAcquireLock (
  290. PEDMA_CONTROLLER Controller
  291. );
  292. VOID
  293. EdmapReleaseLock (
  294. PEDMA_CONTROLLER Controller,
  295. RUNLEVEL OldRunLevel
  296. );
  297. //
  298. // -------------------------------------------------------------------- Globals
  299. //
  300. PDRIVER EdmaDriver = NULL;
  301. DMA_FUNCTION_TABLE EdmaFunctionTableTemplate = {
  302. EdmaSubmit,
  303. EdmaCancel,
  304. NULL
  305. };
  306. DMA_INFORMATION EdmaInformationTemplate = {
  307. DMA_INFORMATION_VERSION,
  308. UUID_EDMA_CONTROLLER,
  309. 0,
  310. NULL,
  311. 0,
  312. EDMA_CHANNEL_COUNT,
  313. 0,
  314. 0xFFFFFFFF
  315. };
  316. //
  317. // ------------------------------------------------------------------ Functions
  318. //
  319. KSTATUS
  320. DriverEntry (
  321. PDRIVER Driver
  322. )
  323. /*++
  324. Routine Description:
  325. This routine is the entry point for the RK32 GPIO driver. It registers
  326. its other dispatch functions, and performs driver-wide initialization.
  327. Arguments:
  328. Driver - Supplies a pointer to the driver object.
  329. Return Value:
  330. STATUS_SUCCESS on success.
  331. Failure code on error.
  332. --*/
  333. {
  334. DRIVER_FUNCTION_TABLE FunctionTable;
  335. KSTATUS Status;
  336. EdmaDriver = Driver;
  337. RtlZeroMemory(&FunctionTable, sizeof(DRIVER_FUNCTION_TABLE));
  338. FunctionTable.Version = DRIVER_FUNCTION_TABLE_VERSION;
  339. FunctionTable.AddDevice = EdmaAddDevice;
  340. FunctionTable.DispatchStateChange = EdmaDispatchStateChange;
  341. FunctionTable.DispatchOpen = EdmaDispatchOpen;
  342. FunctionTable.DispatchClose = EdmaDispatchClose;
  343. FunctionTable.DispatchIo = EdmaDispatchIo;
  344. FunctionTable.DispatchSystemControl = EdmaDispatchSystemControl;
  345. Status = IoRegisterDriverFunctions(Driver, &FunctionTable);
  346. return Status;
  347. }
  348. KSTATUS
  349. EdmaAddDevice (
  350. PVOID Driver,
  351. PSTR DeviceId,
  352. PSTR ClassId,
  353. PSTR CompatibleIds,
  354. PVOID DeviceToken
  355. )
  356. /*++
  357. Routine Description:
  358. This routine is called when a device is detected for which this driver
  359. acts as the function driver. The driver will attach itself to the stack.
  360. Arguments:
  361. Driver - Supplies a pointer to the driver being called.
  362. DeviceId - Supplies a pointer to a string with the device ID.
  363. ClassId - Supplies a pointer to a string containing the device's class ID.
  364. CompatibleIds - Supplies a pointer to a string containing device IDs
  365. that would be compatible with this device.
  366. DeviceToken - Supplies an opaque token that the driver can use to identify
  367. the device in the system. This token should be used when attaching to
  368. the stack.
  369. Return Value:
  370. STATUS_SUCCESS on success.
  371. Failure code if the driver was unsuccessful in attaching itself.
  372. --*/
  373. {
  374. PEDMA_CONTROLLER Controller;
  375. KSTATUS Status;
  376. Controller = MmAllocateNonPagedPool(sizeof(EDMA_CONTROLLER),
  377. EDMA_ALLOCATION_TAG);
  378. if (Controller == NULL) {
  379. return STATUS_INSUFFICIENT_RESOURCES;
  380. }
  381. RtlZeroMemory(Controller, sizeof(EDMA_CONTROLLER));
  382. Controller->OsDevice = DeviceToken;
  383. Controller->CompletionInterruptHandle = INVALID_HANDLE;
  384. Controller->ErrorInterruptHandle = INVALID_HANDLE;
  385. KeInitializeSpinLock(&(Controller->Lock));
  386. //
  387. // PaRAM zero is reserved for a null entry at all times.
  388. //
  389. Controller->Params[0] = 1;
  390. Status = IoAttachDriverToDevice(Driver, DeviceToken, Controller);
  391. if (!KSUCCESS(Status)) {
  392. goto AddDeviceEnd;
  393. }
  394. AddDeviceEnd:
  395. if (!KSUCCESS(Status)) {
  396. if (Controller != NULL) {
  397. MmFreeNonPagedPool(Controller);
  398. }
  399. }
  400. return Status;
  401. }
  402. VOID
  403. EdmaDispatchStateChange (
  404. PIRP Irp,
  405. PVOID DeviceContext,
  406. PVOID IrpContext
  407. )
  408. /*++
  409. Routine Description:
  410. This routine handles State Change IRPs.
  411. Arguments:
  412. Irp - Supplies a pointer to the I/O request packet.
  413. DeviceContext - Supplies the context pointer supplied by the driver when it
  414. attached itself to the driver stack. Presumably this pointer contains
  415. driver-specific device context.
  416. IrpContext - Supplies the context pointer supplied by the driver when
  417. the IRP was created.
  418. Return Value:
  419. None.
  420. --*/
  421. {
  422. KSTATUS Status;
  423. ASSERT(Irp->MajorCode == IrpMajorStateChange);
  424. if (Irp->Direction == IrpUp) {
  425. switch (Irp->MinorCode) {
  426. case IrpMinorQueryResources:
  427. Status = EdmaProcessResourceRequirements(Irp);
  428. if (!KSUCCESS(Status)) {
  429. IoCompleteIrp(EdmaDriver, Irp, Status);
  430. }
  431. break;
  432. case IrpMinorStartDevice:
  433. Status = EdmaStartDevice(Irp, DeviceContext);
  434. if (!KSUCCESS(Status)) {
  435. IoCompleteIrp(EdmaDriver, Irp, Status);
  436. }
  437. break;
  438. default:
  439. break;
  440. }
  441. }
  442. return;
  443. }
  444. VOID
  445. EdmaDispatchOpen (
  446. PIRP Irp,
  447. PVOID DeviceContext,
  448. PVOID IrpContext
  449. )
  450. /*++
  451. Routine Description:
  452. This routine handles Open IRPs.
  453. Arguments:
  454. Irp - Supplies a pointer to the I/O request packet.
  455. DeviceContext - Supplies the context pointer supplied by the driver when it
  456. attached itself to the driver stack. Presumably this pointer contains
  457. driver-specific device context.
  458. IrpContext - Supplies the context pointer supplied by the driver when
  459. the IRP was created.
  460. Return Value:
  461. None.
  462. --*/
  463. {
  464. return;
  465. }
  466. VOID
  467. EdmaDispatchClose (
  468. PIRP Irp,
  469. PVOID DeviceContext,
  470. PVOID IrpContext
  471. )
  472. /*++
  473. Routine Description:
  474. This routine handles Close IRPs.
  475. Arguments:
  476. Irp - Supplies a pointer to the I/O request packet.
  477. DeviceContext - Supplies the context pointer supplied by the driver when it
  478. attached itself to the driver stack. Presumably this pointer contains
  479. driver-specific device context.
  480. IrpContext - Supplies the context pointer supplied by the driver when
  481. the IRP was created.
  482. Return Value:
  483. None.
  484. --*/
  485. {
  486. return;
  487. }
  488. VOID
  489. EdmaDispatchIo (
  490. PIRP Irp,
  491. PVOID DeviceContext,
  492. PVOID IrpContext
  493. )
  494. /*++
  495. Routine Description:
  496. This routine handles I/O IRPs.
  497. Arguments:
  498. Irp - Supplies a pointer to the I/O request packet.
  499. DeviceContext - Supplies the context pointer supplied by the driver when it
  500. attached itself to the driver stack. Presumably this pointer contains
  501. driver-specific device context.
  502. IrpContext - Supplies the context pointer supplied by the driver when
  503. the IRP was created.
  504. Return Value:
  505. None.
  506. --*/
  507. {
  508. return;
  509. }
  510. VOID
  511. EdmaDispatchSystemControl (
  512. PIRP Irp,
  513. PVOID DeviceContext,
  514. PVOID IrpContext
  515. )
  516. /*++
  517. Routine Description:
  518. This routine handles System Control IRPs.
  519. Arguments:
  520. Irp - Supplies a pointer to the I/O request packet.
  521. DeviceContext - Supplies the context pointer supplied by the driver when it
  522. attached itself to the driver stack. Presumably this pointer contains
  523. driver-specific device context.
  524. IrpContext - Supplies the context pointer supplied by the driver when
  525. the IRP was created.
  526. Return Value:
  527. None.
  528. --*/
  529. {
  530. ASSERT(Irp->MajorCode == IrpMajorSystemControl);
  531. //
  532. // Do no processing on any IRPs. Let them flow.
  533. //
  534. return;
  535. }
  536. INTERRUPT_STATUS
  537. EdmaCompletionInterruptService (
  538. PVOID Context
  539. )
  540. /*++
  541. Routine Description:
  542. This routine processes a transfer completion interrupt.
  543. Arguments:
  544. Context - Supplies the context supplied when this interrupt was initially
  545. connected.
  546. Return Value:
  547. Returns an interrupt status indicating if this ISR is claiming the
  548. interrupt, not claiming the interrupt, or needs the interrupt to be
  549. masked temporarily.
  550. --*/
  551. {
  552. PEDMA_CONTROLLER Controller;
  553. INTERRUPT_STATUS Status;
  554. ULONG Value;
  555. Controller = Context;
  556. Status = InterruptStatusNotClaimed;
  557. Value = EDMA_REGION_READ(Controller, EdmaInterruptPendingLow);
  558. if (Value != 0) {
  559. RtlAtomicOr32(&(Controller->Pending.CompletionLow), Value);
  560. EDMA_REGION_WRITE(Controller, EdmaInterruptClearLow, Value);
  561. Status = InterruptStatusClaimed;
  562. }
  563. Value = EDMA_REGION_READ(Controller, EdmaInterruptPendingHigh);
  564. if (Value != 0) {
  565. RtlAtomicOr32(&(Controller->Pending.CompletionHigh), Value);
  566. EDMA_REGION_WRITE(Controller, EdmaInterruptClearHigh, Value);
  567. Status = InterruptStatusClaimed;
  568. }
  569. return Status;
  570. }
  571. INTERRUPT_STATUS
  572. EdmaErrorInterruptService (
  573. PVOID Context
  574. )
  575. /*++
  576. Routine Description:
  577. This routine processes a transfer error interrupt.
  578. Arguments:
  579. Context - Supplies the context supplied when this interrupt was initially
  580. connected.
  581. Return Value:
  582. Returns an interrupt status indicating if this ISR is claiming the
  583. interrupt, not claiming the interrupt, or needs the interrupt to be
  584. masked temporarily.
  585. --*/
  586. {
  587. PEDMA_CONTROLLER Controller;
  588. INTERRUPT_STATUS Status;
  589. ULONG Value;
  590. Controller = Context;
  591. Status = InterruptStatusNotClaimed;
  592. Value = EDMA_READ(Controller, EdmaEventMissedLow);
  593. if (Value != 0) {
  594. RtlAtomicOr32(&(Controller->Pending.MissedLow), Value);
  595. EDMA_WRITE(Controller, EdmaEventMissedClearLow, Value);
  596. EDMA_WRITE(Controller, EdmaSecondaryEventClearLow, Value);
  597. Status = InterruptStatusClaimed;
  598. }
  599. Value = EDMA_READ(Controller, EdmaEventMissedHigh);
  600. if (Value != 0) {
  601. RtlAtomicOr32(&(Controller->Pending.MissedHigh), Value);
  602. EDMA_WRITE(Controller, EdmaEventMissedClearHigh, Value);
  603. EDMA_WRITE(Controller, EdmaSecondaryEventClearHigh, Value);
  604. Status = InterruptStatusClaimed;
  605. }
  606. Value = EDMA_READ(Controller, EdmaQDmaEventMissed);
  607. if (Value != 0) {
  608. RtlAtomicOr32(&(Controller->Pending.MissedQuick), Value);
  609. EDMA_WRITE(Controller, EdmaQDmaEventMissedClear, Value);
  610. EDMA_WRITE(Controller, EdmaQDmaSecondaryEventClear, Value);
  611. Status = InterruptStatusClaimed;
  612. }
  613. Value = EDMA_READ(Controller, EdmaCcError);
  614. if (Value != 0) {
  615. RtlAtomicOr32(&(Controller->Pending.Error), Value);
  616. EDMA_WRITE(Controller, EdmaCcErrorClear, Value);
  617. Status = InterruptStatusClaimed;
  618. }
  619. if (Status == InterruptStatusClaimed) {
  620. RtlDebugPrint("EDMA: Error %x %x %x %x\n",
  621. Controller->Pending.MissedLow,
  622. Controller->Pending.MissedHigh,
  623. Controller->Pending.MissedQuick,
  624. Controller->Pending.Error);
  625. EDMA_WRITE(Controller, EdmaErrorEvaluate, 1);
  626. }
  627. return Status;
  628. }
  629. INTERRUPT_STATUS
  630. EdmaInterruptServiceDispatch (
  631. PVOID Context
  632. )
  633. /*++
  634. Routine Description:
  635. This routine handles interrupts for the EDMA controller at dispatch level.
  636. Arguments:
  637. Context - Supplies the context supplied when this interrupt was initially
  638. connected.
  639. Return Value:
  640. Returns an interrupt status indicating if this ISR is claiming the
  641. interrupt, not claiming the interrupt, or needs the interrupt to be
  642. masked temporarily.
  643. --*/
  644. {
  645. ULONGLONG Bits;
  646. ULONG Channel;
  647. PEDMA_CONTROLLER Controller;
  648. ULONG Value;
  649. Controller = Context;
  650. ASSERT(KeGetRunLevel() == RunLevelDispatch);
  651. //
  652. // Handle completion interrupts.
  653. //
  654. KeAcquireSpinLock(&(Controller->Lock));
  655. Bits = RtlAtomicExchange32(&(Controller->Pending.CompletionLow), 0);
  656. Value = RtlAtomicExchange32(&(Controller->Pending.CompletionHigh), 0);
  657. Bits |= (ULONGLONG)Value << 32;
  658. while (Bits != 0) {
  659. Channel = RtlCountTrailingZeros64(Bits);
  660. Bits &= ~(1ULL << Channel);
  661. EdmapProcessCompletedTransfer(Controller, Channel, FALSE);
  662. }
  663. if ((Controller->Pending.MissedLow != 0) ||
  664. (Controller->Pending.MissedHigh != 0)) {
  665. Bits = RtlAtomicExchange32(&(Controller->Pending.MissedLow), 0);
  666. Value = RtlAtomicExchange32(&(Controller->Pending.MissedHigh), 0);
  667. Bits |= (ULONGLONG)Value << 32;
  668. while (Bits != 0) {
  669. Channel = RtlCountTrailingZeros64(Bits);
  670. Bits &= ~(1ULL << Channel);
  671. EdmapProcessCompletedTransfer(Controller, Channel, TRUE);
  672. }
  673. }
  674. if ((Controller->Pending.MissedQuick != 0) ||
  675. (Controller->Pending.Error != 0)) {
  676. Value = RtlAtomicExchange32(&(Controller->Pending.MissedQuick), 0);
  677. if (Value != 0) {
  678. RtlDebugPrint("EDMA: Missed quick DMA events 0x%x\n", Value);
  679. }
  680. Value = RtlAtomicExchange32(&(Controller->Pending.Error), 0);
  681. if (Value != 0) {
  682. RtlDebugPrint("EDMA: Error event 0x%x\n", Value);
  683. }
  684. }
  685. KeReleaseSpinLock(&(Controller->Lock));
  686. return InterruptStatusClaimed;
  687. }
  688. //
  689. // --------------------------------------------------------- Internal Functions
  690. //
  691. KSTATUS
  692. EdmaProcessResourceRequirements (
  693. PIRP Irp
  694. )
  695. /*++
  696. Routine Description:
  697. This routine filters through the resource requirements presented by the
  698. bus for an RK32 GPIO controller. It adds an interrupt vector requirement
  699. for any interrupt line requested.
  700. Arguments:
  701. Irp - Supplies a pointer to the I/O request packet.
  702. Return Value:
  703. Status code.
  704. --*/
  705. {
  706. PRESOURCE_CONFIGURATION_LIST Requirements;
  707. KSTATUS Status;
  708. RESOURCE_REQUIREMENT VectorRequirement;
  709. ASSERT((Irp->MajorCode == IrpMajorStateChange) &&
  710. (Irp->MinorCode == IrpMinorQueryResources));
  711. //
  712. // Initialize a nice interrupt vector requirement in preparation.
  713. //
  714. RtlZeroMemory(&VectorRequirement, sizeof(RESOURCE_REQUIREMENT));
  715. VectorRequirement.Type = ResourceTypeInterruptVector;
  716. VectorRequirement.Minimum = 0;
  717. VectorRequirement.Maximum = -1;
  718. VectorRequirement.Length = 1;
  719. //
  720. // Loop through all configuration lists, creating a vector for each line.
  721. //
  722. Requirements = Irp->U.QueryResources.ResourceRequirements;
  723. Status = IoCreateAndAddInterruptVectorsForLines(Requirements,
  724. &VectorRequirement);
  725. if (!KSUCCESS(Status)) {
  726. goto ProcessResourceRequirementsEnd;
  727. }
  728. ProcessResourceRequirementsEnd:
  729. return Status;
  730. }
  731. KSTATUS
  732. EdmaStartDevice (
  733. PIRP Irp,
  734. PEDMA_CONTROLLER Device
  735. )
  736. /*++
  737. Routine Description:
  738. This routine starts the RK32 GPIO device.
  739. Arguments:
  740. Irp - Supplies a pointer to the start IRP.
  741. Device - Supplies a pointer to the device information.
  742. Return Value:
  743. Status code.
  744. --*/
  745. {
  746. ULONG AlignmentOffset;
  747. PRESOURCE_ALLOCATION Allocation;
  748. PRESOURCE_ALLOCATION_LIST AllocationList;
  749. PRESOURCE_ALLOCATION CompletionInterrupt;
  750. IO_CONNECT_INTERRUPT_PARAMETERS Connect;
  751. PRESOURCE_ALLOCATION ControllerBase;
  752. PHYSICAL_ADDRESS EndAddress;
  753. PRESOURCE_ALLOCATION ErrorInterrupt;
  754. ULONG PageSize;
  755. PHYSICAL_ADDRESS PhysicalAddress;
  756. PRESOURCE_ALLOCATION ProtectionInterrupt;
  757. DMA_CONTROLLER_INFORMATION Registration;
  758. ULONG Size;
  759. KSTATUS Status;
  760. ControllerBase = NULL;
  761. CompletionInterrupt = NULL;
  762. ErrorInterrupt = NULL;
  763. ProtectionInterrupt = NULL;
  764. Size = 0;
  765. //
  766. // Loop through the allocated resources to get the controller base and the
  767. // interrupt.
  768. //
  769. AllocationList = Irp->U.StartDevice.ProcessorLocalResources;
  770. Allocation = IoGetNextResourceAllocation(AllocationList, NULL);
  771. while (Allocation != NULL) {
  772. //
  773. // If the resource is an interrupt vector, then it should have an
  774. // owning interrupt line allocation.
  775. //
  776. if (Allocation->Type == ResourceTypeInterruptVector) {
  777. ASSERT(Allocation->OwningAllocation != NULL);
  778. if (CompletionInterrupt == NULL) {
  779. CompletionInterrupt = Allocation;
  780. } else if (ProtectionInterrupt == NULL) {
  781. ProtectionInterrupt = Allocation;
  782. } else if (ErrorInterrupt == NULL) {
  783. ErrorInterrupt = Allocation;
  784. }
  785. //
  786. // Look for the first physical address reservation, the registers.
  787. //
  788. } else if (Allocation->Type == ResourceTypePhysicalAddressSpace) {
  789. if (ControllerBase == NULL) {
  790. ControllerBase = Allocation;
  791. }
  792. }
  793. //
  794. // Get the next allocation in the list.
  795. //
  796. Allocation = IoGetNextResourceAllocation(AllocationList, Allocation);
  797. }
  798. //
  799. // Fail to start if the controller base was not found.
  800. //
  801. if ((ControllerBase == NULL) ||
  802. (CompletionInterrupt == NULL) ||
  803. (ErrorInterrupt == NULL)) {
  804. Status = STATUS_INVALID_CONFIGURATION;
  805. goto StartDeviceEnd;
  806. }
  807. //
  808. // Map the controller.
  809. //
  810. if (Device->ControllerBase == NULL) {
  811. //
  812. // Page align the mapping request.
  813. //
  814. PageSize = MmPageSize();
  815. PhysicalAddress = ControllerBase->Allocation;
  816. EndAddress = PhysicalAddress + ControllerBase->Length;
  817. PhysicalAddress = ALIGN_RANGE_DOWN(PhysicalAddress, PageSize);
  818. AlignmentOffset = ControllerBase->Allocation - PhysicalAddress;
  819. EndAddress = ALIGN_RANGE_UP(EndAddress, PageSize);
  820. Size = (ULONG)(EndAddress - PhysicalAddress);
  821. Device->ControllerBase = MmMapPhysicalAddress(PhysicalAddress,
  822. Size,
  823. TRUE,
  824. FALSE,
  825. TRUE);
  826. if (Device->ControllerBase == NULL) {
  827. Status = STATUS_NO_MEMORY;
  828. goto StartDeviceEnd;
  829. }
  830. Device->ControllerBase += AlignmentOffset;
  831. }
  832. ASSERT(Device->ControllerBase != NULL);
  833. //
  834. // Allocate the controller structures.
  835. //
  836. if (Device->DmaController == NULL) {
  837. RtlZeroMemory(&Registration, sizeof(DMA_CONTROLLER_INFORMATION));
  838. Registration.Version = DMA_CONTROLLER_INFORMATION_VERSION;
  839. Registration.Context = Device;
  840. Registration.Device = Device->OsDevice;
  841. RtlCopyMemory(&(Registration.Information),
  842. &EdmaInformationTemplate,
  843. sizeof(DMA_INFORMATION));
  844. RtlCopyMemory(&(Registration.FunctionTable),
  845. &EdmaFunctionTableTemplate,
  846. sizeof(DMA_FUNCTION_TABLE));
  847. Status = DmaCreateController(&Registration, &(Device->DmaController));
  848. if (!KSUCCESS(Status)) {
  849. goto StartDeviceEnd;
  850. }
  851. }
  852. EdmapControllerReset(Device);
  853. //
  854. // Start up the controller.
  855. //
  856. Status = DmaStartController(Device->DmaController);
  857. if (!KSUCCESS(Status)) {
  858. goto StartDeviceEnd;
  859. }
  860. //
  861. // Connect the completion interrupt.
  862. //
  863. if (Device->CompletionInterruptHandle == INVALID_HANDLE) {
  864. Device->CompletionInterruptVector = CompletionInterrupt->Allocation;
  865. Device->CompletionInterruptLine =
  866. CompletionInterrupt->OwningAllocation->Allocation;
  867. RtlZeroMemory(&Connect, sizeof(IO_CONNECT_INTERRUPT_PARAMETERS));
  868. Connect.Version = IO_CONNECT_INTERRUPT_PARAMETERS_VERSION;
  869. Connect.Device = Irp->Device;
  870. Connect.LineNumber = Device->CompletionInterruptLine;
  871. Connect.Vector = Device->CompletionInterruptVector;
  872. Connect.InterruptServiceRoutine = EdmaCompletionInterruptService;
  873. Connect.DispatchServiceRoutine = EdmaInterruptServiceDispatch;
  874. Connect.Context = Device;
  875. Connect.Interrupt = &(Device->CompletionInterruptHandle);
  876. Status = IoConnectInterrupt(&Connect);
  877. if (!KSUCCESS(Status)) {
  878. return Status;
  879. }
  880. }
  881. //
  882. // Connect the error interrupt.
  883. //
  884. if (Device->ErrorInterruptHandle == INVALID_HANDLE) {
  885. Device->ErrorInterruptVector = ErrorInterrupt->Allocation;
  886. Device->ErrorInterruptLine =
  887. ErrorInterrupt->OwningAllocation->Allocation;
  888. RtlZeroMemory(&Connect, sizeof(IO_CONNECT_INTERRUPT_PARAMETERS));
  889. Connect.Version = IO_CONNECT_INTERRUPT_PARAMETERS_VERSION;
  890. Connect.Device = Irp->Device;
  891. Connect.LineNumber = Device->ErrorInterruptLine;
  892. Connect.Vector = Device->ErrorInterruptVector;
  893. Connect.InterruptServiceRoutine = EdmaErrorInterruptService;
  894. Connect.DispatchServiceRoutine = EdmaInterruptServiceDispatch;
  895. Connect.Context = Device;
  896. Connect.Interrupt = &(Device->ErrorInterruptHandle);
  897. Status = IoConnectInterrupt(&Connect);
  898. if (!KSUCCESS(Status)) {
  899. return Status;
  900. }
  901. }
  902. StartDeviceEnd:
  903. if (!KSUCCESS(Status)) {
  904. if (Device->CompletionInterruptHandle != INVALID_HANDLE) {
  905. IoDisconnectInterrupt(Device->CompletionInterruptHandle);
  906. Device->CompletionInterruptHandle = INVALID_HANDLE;
  907. }
  908. if (Device->ErrorInterruptHandle != INVALID_HANDLE) {
  909. IoDisconnectInterrupt(Device->ErrorInterruptHandle);
  910. Device->ErrorInterruptHandle = INVALID_HANDLE;
  911. }
  912. if (Device->ControllerBase != NULL) {
  913. MmUnmapAddress(Device->ControllerBase, Size);
  914. Device->ControllerBase = NULL;
  915. }
  916. if (Device->DmaController != NULL) {
  917. DmaDestroyController(Device->DmaController);
  918. Device->DmaController = NULL;
  919. }
  920. }
  921. return Status;
  922. }
  923. KSTATUS
  924. EdmaSubmit (
  925. PVOID Context,
  926. PDMA_TRANSFER Transfer
  927. )
  928. /*++
  929. Routine Description:
  930. This routine is called to execute a transfer on the EDMA3 controller.
  931. Arguments:
  932. Context - Supplies the host controller context.
  933. Transfer - Supplies a pointer to the transfer to begin executing. The
  934. controller can return immediately, and should call
  935. DmaProcessCompletedTransfer when the transfer completes.
  936. Return Value:
  937. Status code indicating whether or not the transfer was successfully
  938. started.
  939. --*/
  940. {
  941. ULONG Channel;
  942. PEDMA_CONTROLLER Controller;
  943. PEDMA_TRANSFER EdmaTransfer;
  944. BOOL LockHeld;
  945. RUNLEVEL OldRunLevel;
  946. KSTATUS Status;
  947. Controller = Context;
  948. LockHeld = FALSE;
  949. //
  950. // Allocate a transfer for this channel if necessary. This is serialized by
  951. // the DMA core that only submits one transfer to a channel at a time.
  952. //
  953. Channel = Transfer->Allocation->Allocation;
  954. EdmaTransfer = Controller->Transfers[Channel];
  955. if (EdmaTransfer == NULL) {
  956. EdmaTransfer = MmAllocateNonPagedPool(sizeof(EDMA_TRANSFER),
  957. EDMA_ALLOCATION_TAG);
  958. if (EdmaTransfer == NULL) {
  959. Status = STATUS_INSUFFICIENT_RESOURCES;
  960. goto SubmitEnd;
  961. }
  962. EdmaTransfer->Transfer = NULL;
  963. EdmaTransfer->ParamCount = 0;
  964. Controller->Transfers[Channel] = EdmaTransfer;
  965. }
  966. OldRunLevel = EdmapAcquireLock(Controller);
  967. LockHeld = TRUE;
  968. ASSERT(EdmaTransfer->Transfer == NULL);
  969. EdmaTransfer->Transfer = Transfer;
  970. Status = EdmapPrepareAndSubmitTransfer(Controller, EdmaTransfer);
  971. if (!KSUCCESS(Status)) {
  972. goto SubmitEnd;
  973. }
  974. Status = STATUS_SUCCESS;
  975. SubmitEnd:
  976. if (!KSUCCESS(Status)) {
  977. if (EdmaTransfer != NULL) {
  978. EdmapResetTransfer(Controller, EdmaTransfer);
  979. }
  980. }
  981. if (LockHeld != FALSE) {
  982. EdmapReleaseLock(Controller, OldRunLevel);
  983. }
  984. return Status;
  985. }
  986. KSTATUS
  987. EdmaCancel (
  988. PVOID Context,
  989. PDMA_TRANSFER Transfer
  990. )
  991. /*++
  992. Routine Description:
  993. This routine is called to cancel an in-progress transfer. Once this routine
  994. returns, the transfer should be all the way out of the DMA controller and
  995. the controller should no longer interrupt because of this transfer. This
  996. routine is called at dispatch level.
  997. Arguments:
  998. Context - Supplies the host controller context.
  999. Transfer - Supplies a pointer to the transfer to cancel.
  1000. Return Value:
  1001. STATUS_SUCCESS on success.
  1002. STATUS_TOO_LATE if the transfer is already complete.
  1003. Other errors on other failures.
  1004. --*/
  1005. {
  1006. ULONG Channel;
  1007. PEDMA_CONTROLLER Controller;
  1008. RUNLEVEL OldRunLevel;
  1009. KSTATUS Status;
  1010. Controller = Context;
  1011. Channel = Transfer->Allocation->Allocation;
  1012. //
  1013. // If there is no transfer for this channel, then something is wrong.
  1014. //
  1015. if (Controller->Transfers[Channel] == NULL) {
  1016. return STATUS_INVALID_PARAMETER;
  1017. }
  1018. //
  1019. // Do a quick check to see if the transfer is still in the channel. If it
  1020. // is not, then it's too late.
  1021. //
  1022. if (Controller->Transfers[Channel]->Transfer != Transfer) {
  1023. return STATUS_TOO_LATE;
  1024. }
  1025. //
  1026. // Grab the lock to synchronize with completion, and then look again.
  1027. //
  1028. OldRunLevel = EdmapAcquireLock(Controller);
  1029. if (Controller->Transfers[Channel]->Transfer != Transfer) {
  1030. Status = STATUS_TOO_LATE;
  1031. goto CancelEnd;
  1032. }
  1033. //
  1034. // Tear down the channel to stop any transfer that might be in progress.
  1035. //
  1036. EdmapTearDownChannel(Controller, Channel);
  1037. EdmapResetTransfer(Controller, Controller->Transfers[Channel]);
  1038. Status = STATUS_SUCCESS;
  1039. CancelEnd:
  1040. EdmapReleaseLock(Controller, OldRunLevel);
  1041. return Status;
  1042. }
  1043. VOID
  1044. EdmapControllerReset (
  1045. PEDMA_CONTROLLER Controller
  1046. )
  1047. /*++
  1048. Routine Description:
  1049. This routine resets and initializes the EDMA controller.
  1050. Arguments:
  1051. Controller - Supplies a pointer to the controller.
  1052. Return Value:
  1053. None.
  1054. --*/
  1055. {
  1056. ULONG Channel;
  1057. EDMA_PARAM Param;
  1058. EDMA_WRITE64(Controller, EdmaEventMissedClearLow, -1ULL);
  1059. EDMA_WRITE(Controller, EdmaQDmaEventMissedClear, -1);
  1060. EDMA_WRITE(Controller, EdmaCcErrorClear, -1);
  1061. //
  1062. // Create a null entry. PaRAM slot zero is reserved to always be a null
  1063. // entry.
  1064. //
  1065. RtlZeroMemory(&Param, sizeof(EDMA_PARAM));
  1066. EdmapSetParam(Controller, 0, &Param);
  1067. //
  1068. // Initially set all events to point at the null entry.
  1069. //
  1070. for (Channel = 0; Channel < EDMA_CHANNEL_COUNT; Channel += 1) {
  1071. EDMA_WRITE(Controller, EDMA_DMA_CHANNEL_MAP(Channel), 0);
  1072. }
  1073. //
  1074. // Enable all DMA channels in this controller's region.
  1075. //
  1076. EDMA_WRITE64(Controller,
  1077. EDMA_DMA_REGION_ACCESS(Controller->Region),
  1078. -1ULL);
  1079. EDMA_WRITE64(Controller,
  1080. EDMA_QDMA_REGION_ACCESS(Controller->Region),
  1081. -1ULL);
  1082. //
  1083. // Disable all interrupts.
  1084. //
  1085. EDMA_REGION_WRITE64(Controller, EdmaInterruptEnableClearLow, -1ULL);
  1086. return;
  1087. }
  1088. KSTATUS
  1089. EdmapPrepareAndSubmitTransfer (
  1090. PEDMA_CONTROLLER Controller,
  1091. PEDMA_TRANSFER Transfer
  1092. )
  1093. /*++
  1094. Routine Description:
  1095. This routine prepares and submits an EDMA transfer.
  1096. Arguments:
  1097. Controller - Supplies a pointer to the controller.
  1098. Transfer - Supplies a pointer to the transfer to prepare and submit.
  1099. Return Value:
  1100. Status code.
  1101. --*/
  1102. {
  1103. KSTATUS Status;
  1104. Status = EdmapPrepareTransfer(Controller, Transfer);
  1105. if (!KSUCCESS(Status)) {
  1106. goto PrepareAndSubmitTransfer;
  1107. }
  1108. Status = EdmapSubmitTransfer(Controller, Transfer);
  1109. if (!KSUCCESS(Status)) {
  1110. goto PrepareAndSubmitTransfer;
  1111. }
  1112. PrepareAndSubmitTransfer:
  1113. return Status;
  1114. }
  1115. KSTATUS
  1116. EdmapPrepareTransfer (
  1117. PEDMA_CONTROLLER Controller,
  1118. PEDMA_TRANSFER Transfer
  1119. )
  1120. /*++
  1121. Routine Description:
  1122. This routine prepares for a DMA transfer, filling out as many PaRAM
  1123. entries as possible.
  1124. Arguments:
  1125. Controller - Supplies a pointer to the controller.
  1126. Transfer - Supplies a pointer to the transfer to set up.
  1127. Return Value:
  1128. Status code.
  1129. --*/
  1130. {
  1131. UINTN BytesThisRound;
  1132. PEDMA_CONFIGURATION Configuration;
  1133. PHYSICAL_ADDRESS DeviceAddress;
  1134. PDMA_TRANSFER DmaTransfer;
  1135. PIO_BUFFER_FRAGMENT Fragment;
  1136. ULONG FragmentIndex;
  1137. UINTN FragmentOffset;
  1138. PIO_BUFFER IoBuffer;
  1139. UINTN IoBufferOffset;
  1140. PHYSICAL_ADDRESS MemoryAddress;
  1141. ULONG ParamIndex;
  1142. UINTN ParamSize;
  1143. PHYSICAL_ADDRESS PreviousAddress;
  1144. UINTN Remaining;
  1145. KSTATUS Status;
  1146. DmaTransfer = Transfer->Transfer;
  1147. ParamIndex = 0;
  1148. IoBuffer = DmaTransfer->Memory;
  1149. if (DmaTransfer->Completed >= DmaTransfer->Size) {
  1150. Status = STATUS_SUCCESS;
  1151. goto PrepareTransferEnd;
  1152. }
  1153. //
  1154. // Memory to memory transfers require some reorganization of the loop in
  1155. // this function.
  1156. //
  1157. ASSERT(DmaTransfer->Direction != DmaTransferMemoryToMemory);
  1158. DeviceAddress = DmaTransfer->Device.Address;
  1159. if ((DmaTransfer->Flags & DMA_TRANSFER_ADVANCE_DEVICE) != 0) {
  1160. DeviceAddress += DmaTransfer->Completed;
  1161. }
  1162. //
  1163. // Get past the already completed portion.
  1164. //
  1165. IoBufferOffset = MmGetIoBufferCurrentOffset(IoBuffer) +
  1166. DmaTransfer->Completed;
  1167. FragmentIndex = 0;
  1168. FragmentOffset = 0;
  1169. while (IoBufferOffset != 0) {
  1170. ASSERT(FragmentIndex < IoBuffer->FragmentCount);
  1171. Fragment = &(IoBuffer->Fragment[FragmentIndex]);
  1172. if (IoBufferOffset < Fragment->Size) {
  1173. FragmentOffset = IoBufferOffset;
  1174. break;
  1175. }
  1176. IoBufferOffset -= Fragment->Size;
  1177. FragmentIndex += 1;
  1178. }
  1179. //
  1180. // Now loop filling out PaRAM entries.
  1181. //
  1182. Transfer->BytesPending = 0;
  1183. Remaining = DmaTransfer->Size - DmaTransfer->Completed;
  1184. PreviousAddress = IoBuffer->Fragment[FragmentIndex].PhysicalAddress +
  1185. FragmentOffset;
  1186. MemoryAddress = PreviousAddress;
  1187. ParamSize = 0;
  1188. while ((Remaining != 0) && (ParamIndex + 1 < EDMA_TRANSFER_PARAMS)) {
  1189. ASSERT(FragmentIndex < IoBuffer->FragmentCount);
  1190. Fragment = &(IoBuffer->Fragment[FragmentIndex]);
  1191. //
  1192. // If the last address is not contiguous, or the current run is too
  1193. // big, start a new PaRAM.
  1194. //
  1195. if ((Fragment->PhysicalAddress + FragmentOffset != PreviousAddress) ||
  1196. (ParamSize == EDMA_MAX_TRANSFER_SIZE)) {
  1197. Status = EdmapSetupParam(Controller,
  1198. Transfer,
  1199. ParamIndex,
  1200. MemoryAddress,
  1201. DeviceAddress,
  1202. ParamSize,
  1203. FALSE);
  1204. if (!KSUCCESS(Status)) {
  1205. goto PrepareTransferEnd;
  1206. }
  1207. Transfer->BytesPending += ParamSize;
  1208. ParamIndex += 1;
  1209. ParamSize = 0;
  1210. MemoryAddress = Fragment->PhysicalAddress + FragmentOffset;
  1211. PreviousAddress = MemoryAddress;
  1212. if ((DmaTransfer->Flags & DMA_TRANSFER_ADVANCE_DEVICE) != 0) {
  1213. DeviceAddress += ParamSize;
  1214. }
  1215. }
  1216. BytesThisRound = Fragment->Size - FragmentOffset;
  1217. if (BytesThisRound > Remaining) {
  1218. BytesThisRound = Remaining;
  1219. }
  1220. if (BytesThisRound > EDMA_MAX_TRANSFER_SIZE - ParamSize) {
  1221. BytesThisRound = EDMA_MAX_TRANSFER_SIZE - ParamSize;
  1222. }
  1223. FragmentOffset += BytesThisRound;
  1224. ASSERT(FragmentOffset <= Fragment->Size);
  1225. if (FragmentOffset == Fragment->Size) {
  1226. FragmentIndex += 1;
  1227. FragmentOffset = 0;
  1228. }
  1229. ParamSize += BytesThisRound;
  1230. Remaining -= BytesThisRound;
  1231. PreviousAddress += BytesThisRound;
  1232. }
  1233. if (ParamSize != 0) {
  1234. Status = EdmapSetupParam(Controller,
  1235. Transfer,
  1236. ParamIndex,
  1237. MemoryAddress,
  1238. DeviceAddress,
  1239. ParamSize,
  1240. TRUE);
  1241. if (!KSUCCESS(Status)) {
  1242. goto PrepareTransferEnd;
  1243. }
  1244. Transfer->BytesPending += ParamSize;
  1245. ParamIndex += 1;
  1246. }
  1247. //
  1248. // If this is an event based transaction, limit the DMA transfer to what
  1249. // could be achieved this round. Otherwise, the caller may set up a larger
  1250. // transfer, resulting in missed events.
  1251. //
  1252. Configuration = DmaTransfer->Configuration;
  1253. if ((Configuration != NULL) &&
  1254. (DmaTransfer->ConfigurationSize >= sizeof(EDMA_CONFIGURATION))) {
  1255. if (Configuration->Mode == EdmaTriggerModeEvent) {
  1256. Transfer->Transfer->Size = Transfer->BytesPending +
  1257. Transfer->Transfer->Completed;
  1258. }
  1259. }
  1260. Status = STATUS_SUCCESS;
  1261. PrepareTransferEnd:
  1262. return Status;
  1263. }
  1264. KSTATUS
  1265. EdmapSubmitTransfer (
  1266. PEDMA_CONTROLLER Controller,
  1267. PEDMA_TRANSFER Transfer
  1268. )
  1269. /*++
  1270. Routine Description:
  1271. This routine submits a transfer to the EDMA controller. It assumes all
  1272. PaRAMs are set up and ready to go.
  1273. Arguments:
  1274. Controller - Supplies a pointer to the controller.
  1275. Transfer - Supplies a pointer to the EDMA transfer.
  1276. Return Value:
  1277. Status code.
  1278. --*/
  1279. {
  1280. ULONG Channel;
  1281. ULONG ChannelMask;
  1282. PEDMA_CONFIGURATION Configuration;
  1283. PDMA_TRANSFER DmaTransfer;
  1284. EDMA3_TRIGGER_MODE Mode;
  1285. ULONG Offset;
  1286. ULONG Queue;
  1287. ULONG Register;
  1288. ULONG Shift;
  1289. ULONG Value;
  1290. DmaTransfer = Transfer->Transfer;
  1291. Configuration = DmaTransfer->Configuration;
  1292. if (DmaTransfer->ConfigurationSize < sizeof(EDMA_CONFIGURATION)) {
  1293. Configuration = NULL;
  1294. }
  1295. Channel = DmaTransfer->Allocation->Allocation;
  1296. Offset = 0;
  1297. if (Channel >= 32) {
  1298. ChannelMask = 1 << (Channel - 32);
  1299. Offset = 4;
  1300. } else {
  1301. ChannelMask = 1 << Channel;
  1302. }
  1303. ASSERT(Transfer->ParamCount != 0);
  1304. //
  1305. // Set the channel to the first PaRAM entry.
  1306. //
  1307. Register = EDMA_DMA_CHANNEL_MAP(Channel);
  1308. EDMA_WRITE(Controller, Register, Transfer->Params[0] * sizeof(EDMA_PARAM));
  1309. //
  1310. // Shove everything on queue zero unless the caller wants something
  1311. // different.
  1312. //
  1313. Queue = 0;
  1314. if (Configuration != NULL) {
  1315. Queue = Configuration->Queue;
  1316. }
  1317. Shift = EDMA_CHANNEL_QUEUE_SHIFT(Channel);
  1318. Register = EDMA_CHANNEL_QUEUE_REGISTER(Channel);
  1319. Value = EDMA_READ(Controller, Register);
  1320. Value &= ~(EDMA_QUEUE_NUMBER_MASK << Shift);
  1321. Value |= Queue << Shift;
  1322. EDMA_WRITE(Controller, Register, Value);
  1323. //
  1324. // Enable the channel interrupt.
  1325. //
  1326. EDMA_REGION_WRITE(Controller,
  1327. EdmaInterruptEnableSetLow + Offset,
  1328. ChannelMask);
  1329. //
  1330. // Kick off the transfer.
  1331. //
  1332. Mode = EdmaTriggerModeManual;
  1333. if (Configuration != NULL) {
  1334. Mode = Configuration->Mode;
  1335. }
  1336. switch (Mode) {
  1337. //
  1338. // For manual mode, just set the event.
  1339. //
  1340. case EdmaTriggerModeManual:
  1341. EDMA_REGION_WRITE(Controller, EdmaEventSetLow + Offset, ChannelMask);
  1342. break;
  1343. //
  1344. // For event mode, clear the secondary event and event miss registers, then
  1345. // enable the event.
  1346. //
  1347. case EdmaTriggerModeEvent:
  1348. EdmapClearMissEvent(Controller, Channel);
  1349. EDMA_REGION_WRITE(Controller,
  1350. EdmaEventEnableSetLow + Offset,
  1351. ChannelMask);
  1352. break;
  1353. default:
  1354. return STATUS_INVALID_CONFIGURATION;
  1355. }
  1356. return STATUS_SUCCESS;
  1357. }
  1358. KSTATUS
  1359. EdmapSetupParam (
  1360. PEDMA_CONTROLLER Controller,
  1361. PEDMA_TRANSFER Transfer,
  1362. UCHAR ParamIndex,
  1363. PHYSICAL_ADDRESS MemoryAddress,
  1364. PHYSICAL_ADDRESS DeviceAddress,
  1365. ULONG Size,
  1366. BOOL LastOne
  1367. )
  1368. /*++
  1369. Routine Description:
  1370. This routine fills out a PaRAM entry.
  1371. Arguments:
  1372. Controller - Supplies a pointer to the controller.
  1373. Transfer - Supplies a pointer to the EDMA transfer.
  1374. ParamIndex - Supplies the index to fill out.
  1375. MemoryAddress - Supplies the physical memory address to set.
  1376. DeviceAddress - Supplies the physical device address to set.
  1377. Size - Supplies the size of the PaRAM transfer in bytes.
  1378. LastOne - Supplies a boolean indicating if this is the last transfer or not.
  1379. Return Value:
  1380. STATUS_SUCCESS on success.
  1381. STATUS_INSUFFICIENT_RESOURCES if a PaRAM entry could not be allocated.
  1382. STATUS_INVALID_CONFIGURATION if the settings in the existing configuration
  1383. conflict with the request.
  1384. --*/
  1385. {
  1386. ULONG BlockSize;
  1387. ULONG Channel;
  1388. PEDMA_CONFIGURATION Configuration;
  1389. PDMA_TRANSFER DmaTransfer;
  1390. EDMA_PARAM Param;
  1391. DmaTransfer = Transfer->Transfer;
  1392. ASSERT(ParamIndex <= Transfer->ParamCount);
  1393. ASSERT(ParamIndex < EDMA_TRANSFER_PARAMS);
  1394. if (ParamIndex == Transfer->ParamCount) {
  1395. Transfer->Params[ParamIndex] = EdmapAllocateParam(Controller);
  1396. if (Transfer->Params[ParamIndex] == 0) {
  1397. return STATUS_INSUFFICIENT_RESOURCES;
  1398. }
  1399. Transfer->ParamCount += 1;
  1400. }
  1401. //
  1402. // Use the supplied configuration if there is one.
  1403. //
  1404. Configuration = DmaTransfer->Configuration;
  1405. if ((Configuration != NULL) &&
  1406. (DmaTransfer->ConfigurationSize >= sizeof(EDMA_CONFIGURATION))) {
  1407. RtlCopyMemory(&Param, &(Configuration->Param), sizeof(EDMA_PARAM));
  1408. //
  1409. // Figure out how many blocks are in this transfer depending on whether
  1410. // or not there's a third dimension set.
  1411. //
  1412. BlockSize = Param.SourceCIndex;
  1413. if (BlockSize < Param.DestinationCIndex) {
  1414. BlockSize = Param.DestinationCIndex;
  1415. }
  1416. if (BlockSize != 0) {
  1417. //
  1418. // If there's a stride in the third dimension, there had better
  1419. // be a count in the second.
  1420. //
  1421. ASSERT(Param.BCount != 0);
  1422. Param.CCount = Size / BlockSize;
  1423. if ((Size % BlockSize) != 0) {
  1424. ASSERT(FALSE);
  1425. return STATUS_INVALID_CONFIGURATION;
  1426. }
  1427. } else {
  1428. //
  1429. // If there's no stride in the third dimension, there had better
  1430. // not be a count either.
  1431. //
  1432. ASSERT(Param.CCount == 0);
  1433. BlockSize = Param.SourceBIndex;
  1434. if (BlockSize < Param.DestinationBIndex) {
  1435. BlockSize = Param.DestinationBIndex;
  1436. }
  1437. if (BlockSize != 0) {
  1438. Param.BCount = Size / BlockSize;
  1439. }
  1440. }
  1441. } else {
  1442. Configuration = NULL;
  1443. RtlZeroMemory(&Param, sizeof(EDMA_PARAM));
  1444. Param.ACount = DmaTransfer->Width / BITS_PER_BYTE;
  1445. if ((Param.ACount == 0) ||
  1446. ((DmaTransfer->Width % BITS_PER_BYTE) != 0)) {
  1447. ASSERT(FALSE);
  1448. return STATUS_INVALID_CONFIGURATION;
  1449. }
  1450. Param.BCount = Size / Param.ACount;
  1451. if ((Param.BCount == 0) || ((Size % Param.ACount) != 0)) {
  1452. ASSERT(FALSE);
  1453. return STATUS_INVALID_CONFIGURATION;
  1454. }
  1455. Param.SourceBIndex = Param.ACount;
  1456. Param.DestinationBIndex = Param.ACount;
  1457. Channel = DmaTransfer->Allocation->Allocation;
  1458. Param.Options = EDMA_TRANSFER_AB_SYNCHRONIZED;
  1459. Param.Options |= (Channel << EDMA_TRANSFER_COMPLETION_CODE_SHIFT) &
  1460. EDMA_TRANSFER_COMPLETION_CODE_MASK;
  1461. if ((DmaTransfer->Flags & DMA_TRANSFER_ADVANCE_DEVICE) == 0) {
  1462. switch (DmaTransfer->Width) {
  1463. case 256:
  1464. Param.Options |= EDMA_TRANSFER_FIFO_WIDTH_256;
  1465. break;
  1466. case 128:
  1467. Param.Options |= EDMA_TRANSFER_FIFO_WIDTH_128;
  1468. break;
  1469. case 64:
  1470. Param.Options |= EDMA_TRANSFER_FIFO_WIDTH_64;
  1471. break;
  1472. case 32:
  1473. Param.Options |= EDMA_TRANSFER_FIFO_WIDTH_32;
  1474. break;
  1475. case 16:
  1476. Param.Options |= EDMA_TRANSFER_FIFO_WIDTH_16;
  1477. break;
  1478. case 8:
  1479. Param.Options |= EDMA_TRANSFER_FIFO_WIDTH_8;
  1480. break;
  1481. default:
  1482. ASSERT(FALSE);
  1483. return STATUS_INVALID_CONFIGURATION;
  1484. }
  1485. }
  1486. }
  1487. //
  1488. // Link to the next PaRAM entry.
  1489. //
  1490. if (LastOne != FALSE) {
  1491. Param.Link = EDMA_LINK_TERMINATE;
  1492. Param.Options |= EDMA_TRANSFER_COMPLETION_INTERRUPT;
  1493. } else {
  1494. ASSERT(ParamIndex + 1 <= Transfer->ParamCount);
  1495. ASSERT(ParamIndex + 1 < EDMA_TRANSFER_PARAMS);
  1496. if (ParamIndex + 1 == Transfer->ParamCount) {
  1497. Transfer->Params[ParamIndex + 1] = EdmapAllocateParam(Controller);
  1498. if (Transfer->Params[ParamIndex + 1] == 0) {
  1499. return STATUS_INSUFFICIENT_RESOURCES;
  1500. }
  1501. Transfer->ParamCount += 1;
  1502. }
  1503. Param.Link = Transfer->Params[ParamIndex + 1] * sizeof(EDMA_PARAM);
  1504. }
  1505. ASSERT((ULONG)DeviceAddress == DeviceAddress);
  1506. ASSERT((ULONG)MemoryAddress == MemoryAddress);
  1507. if (DmaTransfer->Direction == DmaTransferFromDevice) {
  1508. Param.Source = DeviceAddress;
  1509. Param.Destination = MemoryAddress;
  1510. if (Configuration == NULL) {
  1511. if ((DmaTransfer->Flags & DMA_TRANSFER_ADVANCE_DEVICE) == 0) {
  1512. Param.SourceBIndex = 0;
  1513. Param.Options |= EDMA_TRANSFER_SOURCE_FIFO;
  1514. }
  1515. }
  1516. } else {
  1517. ASSERT((DmaTransfer->Direction == DmaTransferToDevice) ||
  1518. (DmaTransfer->Direction == DmaTransferMemoryToMemory));
  1519. Param.Source = MemoryAddress;
  1520. Param.Destination = DeviceAddress;
  1521. if (Configuration == NULL) {
  1522. if ((DmaTransfer->Flags & DMA_TRANSFER_ADVANCE_DEVICE) == 0) {
  1523. Param.DestinationBIndex = 0;
  1524. Param.Options |= EDMA_TRANSFER_DESTINATION_FIFO;
  1525. }
  1526. }
  1527. }
  1528. EdmapSetParam(Controller, Transfer->Params[ParamIndex], &Param);
  1529. return STATUS_SUCCESS;
  1530. }
  1531. VOID
  1532. EdmapProcessCompletedTransfer (
  1533. PEDMA_CONTROLLER Controller,
  1534. ULONG Channel,
  1535. BOOL MissedEvent
  1536. )
  1537. /*++
  1538. Routine Description:
  1539. This routine processes a completed transfer.
  1540. Arguments:
  1541. Controller - Supplies a pointer to the controller.
  1542. Channel - Supplies the channel that completed.
  1543. MissedEvent - Supplies a boolean indicating whether the transfer missed an
  1544. an event or not.
  1545. Return Value:
  1546. None.
  1547. --*/
  1548. {
  1549. BOOL CompleteTransfer;
  1550. PDMA_TRANSFER DmaTransfer;
  1551. EDMA_PARAM Param;
  1552. KSTATUS Status;
  1553. PEDMA_TRANSFER Transfer;
  1554. //
  1555. // If the channel does not have a transfer allocated, then there is nothing
  1556. // that can be done for this interrupt.
  1557. //
  1558. if (Controller->Transfers[Channel] == NULL) {
  1559. return;
  1560. }
  1561. //
  1562. // If the transfer is gone, ignore it. It may have come in while a transfer
  1563. // was being canceled.
  1564. //
  1565. Transfer = Controller->Transfers[Channel];
  1566. if (Transfer->Transfer == NULL) {
  1567. return;
  1568. }
  1569. //
  1570. // Read the channel's current PaRAM to make sure the transfer is actually
  1571. // complete. When a NULL link is encountered, the NULL PaRAM set is written
  1572. // to the current PaRAM set. A NULL PaRAM set has all three count fields
  1573. // set to 0 and the NULL link set.
  1574. //
  1575. EdmapGetParam(Controller, Transfer->Params[0], &Param);
  1576. if ((Param.Link != EDMA_LINK_TERMINATE) ||
  1577. (Param.ACount != 0) ||
  1578. (Param.BCount != 0) ||
  1579. (Param.CCount != 0)) {
  1580. return;
  1581. }
  1582. DmaTransfer = Transfer->Transfer;
  1583. //
  1584. // Tear down the channel, since either way this transfer is over.
  1585. //
  1586. EdmapTearDownChannel(Controller, Channel);
  1587. CompleteTransfer = TRUE;
  1588. if (MissedEvent != FALSE) {
  1589. Status = STATUS_DEVICE_IO_ERROR;
  1590. goto ProcessCompletedTransferEnd;
  1591. }
  1592. DmaTransfer->Completed += Transfer->BytesPending;
  1593. ASSERT((Transfer->BytesPending != 0) &&
  1594. (DmaTransfer->Completed <= DmaTransfer->Size));
  1595. //
  1596. // Continue the DMA transfer if there's more to do.
  1597. //
  1598. if (DmaTransfer->Completed < DmaTransfer->Size) {
  1599. Status = EdmapPrepareAndSubmitTransfer(Controller, Transfer);
  1600. if (!KSUCCESS(Status)) {
  1601. goto ProcessCompletedTransferEnd;
  1602. }
  1603. CompleteTransfer = FALSE;
  1604. } else {
  1605. Status = STATUS_SUCCESS;
  1606. }
  1607. ProcessCompletedTransferEnd:
  1608. if (CompleteTransfer != FALSE) {
  1609. DmaTransfer->Status = Status;
  1610. EdmapResetTransfer(Controller, Transfer);
  1611. KeReleaseSpinLock(&(Controller->Lock));
  1612. DmaTransfer = DmaTransferCompletion(Controller->DmaController,
  1613. DmaTransfer);
  1614. KeAcquireSpinLock(&(Controller->Lock));
  1615. if (DmaTransfer != NULL) {
  1616. Transfer->Transfer = DmaTransfer;
  1617. EdmapPrepareAndSubmitTransfer(Controller, Transfer);
  1618. }
  1619. }
  1620. return;
  1621. }
  1622. VOID
  1623. EdmapTearDownChannel (
  1624. PEDMA_CONTROLLER Controller,
  1625. ULONG Channel
  1626. )
  1627. /*++
  1628. Routine Description:
  1629. This routine tears down an initialized DMA channel.
  1630. Arguments:
  1631. Controller - Supplies a pointer to the controller.
  1632. Channel - Supplies the channel/event number to tear down.
  1633. Return Value:
  1634. None.
  1635. --*/
  1636. {
  1637. ULONG ChannelMask;
  1638. ULONG Offset;
  1639. ChannelMask = 1 << Channel;
  1640. Offset = 0;
  1641. if (Channel >= 32) {
  1642. ChannelMask = 1 << (Channel - 32);
  1643. Offset = 4;
  1644. }
  1645. EDMA_REGION_WRITE(Controller,
  1646. EdmaInterruptEnableClearLow + Offset,
  1647. ChannelMask);
  1648. EDMA_REGION_WRITE(Controller,
  1649. EdmaEventEnableClearLow + Offset,
  1650. ChannelMask);
  1651. EDMA_REGION_WRITE(Controller,
  1652. EdmaSecondaryEventClearLow + Offset,
  1653. ChannelMask);
  1654. EdmapClearMissEvent(Controller, Channel);
  1655. EDMA_REGION_WRITE(Controller, EdmaEventClearLow + Offset, ChannelMask);
  1656. //
  1657. // Set the PaRAM address back to the null entry.
  1658. //
  1659. EDMA_WRITE(Controller, EDMA_DMA_CHANNEL_MAP(Channel), 0);
  1660. return;
  1661. }
  1662. VOID
  1663. EdmapResetTransfer (
  1664. PEDMA_CONTROLLER Controller,
  1665. PEDMA_TRANSFER Transfer
  1666. )
  1667. /*++
  1668. Routine Description:
  1669. This routine resets an EDMA transfer.
  1670. Arguments:
  1671. Controller - Supplies a pointer to the controller.
  1672. Transfer - Supplies a pointer to the EDMA transfer.
  1673. Return Value:
  1674. None.
  1675. --*/
  1676. {
  1677. ULONG ParamIndex;
  1678. for (ParamIndex = 0; ParamIndex < Transfer->ParamCount; ParamIndex += 1) {
  1679. EdmapFreeParam(Controller, Transfer->Params[ParamIndex]);
  1680. }
  1681. Transfer->ParamCount = 0;
  1682. Transfer->Transfer = NULL;
  1683. return;
  1684. }
  1685. UCHAR
  1686. EdmapAllocateParam (
  1687. PEDMA_CONTROLLER Controller
  1688. )
  1689. /*++
  1690. Routine Description:
  1691. This routine allocates a PaRAM entry.
  1692. Arguments:
  1693. Controller - Supplies a pointer to the controller to allocate from.
  1694. Return Value:
  1695. Returns the PaRAM index on success.
  1696. 0 on failure (0 is reserved for a permanently null entry).
  1697. --*/
  1698. {
  1699. ULONG BitIndex;
  1700. UINTN Block;
  1701. ULONG BlockIndex;
  1702. for (BlockIndex = 0; BlockIndex < EDMA_PARAM_WORDS; BlockIndex += 1) {
  1703. Block = ~(Controller->Params[BlockIndex]);
  1704. if (Block == 0) {
  1705. continue;
  1706. }
  1707. BitIndex = RtlCountTrailingZeros(Block);
  1708. Controller->Params[BlockIndex] |= 1 << BitIndex;
  1709. return (BlockIndex * (sizeof(UINTN) * BITS_PER_BYTE)) + BitIndex;
  1710. }
  1711. return 0;
  1712. }
  1713. VOID
  1714. EdmapFreeParam (
  1715. PEDMA_CONTROLLER Controller,
  1716. UCHAR Param
  1717. )
  1718. /*++
  1719. Routine Description:
  1720. This routine frees a PaRAM entry.
  1721. Arguments:
  1722. Controller - Supplies a pointer to the controller to free to.
  1723. Param - Supplies a pointer to the param to free. This must not be zero.
  1724. Return Value:
  1725. None.
  1726. --*/
  1727. {
  1728. ULONG BitIndex;
  1729. ULONG BlockIndex;
  1730. ULONG Mask;
  1731. ASSERT(Param != 0);
  1732. BlockIndex = Param / (sizeof(UINTN) * BITS_PER_BYTE);
  1733. BitIndex = Param % (sizeof(UINTN) * BITS_PER_BYTE);
  1734. Mask = 1 << BitIndex;
  1735. ASSERT((Controller->Params[BlockIndex] & Mask) != 0);
  1736. Controller->Params[BlockIndex] &= ~Mask;
  1737. return;
  1738. }
  1739. VOID
  1740. EdmapGetParam (
  1741. PEDMA_CONTROLLER Controller,
  1742. UCHAR Index,
  1743. PEDMA_PARAM Param
  1744. )
  1745. /*++
  1746. Routine Description:
  1747. This routine reads an entry from PaRAM.
  1748. Arguments:
  1749. Controller - Supplies a pointer to the controller.
  1750. Index - Supplies the PaRAM number to get.
  1751. Param - Supplies a pointer where the PaRAM will be returned on success.
  1752. Return Value:
  1753. None.
  1754. --*/
  1755. {
  1756. ULONG Register;
  1757. ULONG WordIndex;
  1758. PULONG Words;
  1759. Register = EDMA_GET_PARAM(Controller, Index);
  1760. Words = (PULONG)Param;
  1761. for (WordIndex = 0;
  1762. WordIndex < (sizeof(EDMA_PARAM) / sizeof(ULONG));
  1763. WordIndex += 1) {
  1764. *Words = EDMA_READ(Controller, Register);
  1765. Words += 1;
  1766. Register += sizeof(ULONG);
  1767. }
  1768. return;
  1769. }
  1770. VOID
  1771. EdmapSetParam (
  1772. PEDMA_CONTROLLER Controller,
  1773. UCHAR Index,
  1774. PEDMA_PARAM Param
  1775. )
  1776. /*++
  1777. Routine Description:
  1778. This routine writes an entry to PaRAM.
  1779. Arguments:
  1780. Controller - Supplies a pointer to the controller.
  1781. Index - Supplies the PaRAM number to set.
  1782. Param - Supplies a pointer to the PaRAM data to set.
  1783. Return Value:
  1784. None.
  1785. --*/
  1786. {
  1787. ULONG Register;
  1788. ULONG WordIndex;
  1789. PULONG Words;
  1790. Register = EDMA_GET_PARAM(Controller, Index);
  1791. Words = (PULONG)Param;
  1792. for (WordIndex = 0;
  1793. WordIndex < (sizeof(EDMA_PARAM) / sizeof(ULONG));
  1794. WordIndex += 1) {
  1795. EDMA_WRITE(Controller, Register, *Words);
  1796. Words += 1;
  1797. Register += sizeof(ULONG);
  1798. }
  1799. return;
  1800. }
  1801. VOID
  1802. EdmapClearMissEvent (
  1803. PEDMA_CONTROLLER Controller,
  1804. ULONG Channel
  1805. )
  1806. /*++
  1807. Routine Description:
  1808. This routine clears any missed events in the controller for a particular
  1809. channel.
  1810. Arguments:
  1811. Controller - Supplies a pointer to the controller.
  1812. Channel - Supplies the channel to clear.
  1813. Return Value:
  1814. None.
  1815. --*/
  1816. {
  1817. ULONG Offset;
  1818. Offset = 0;
  1819. if (Channel >= 32) {
  1820. Channel -= 32;
  1821. Offset = 4;
  1822. }
  1823. EDMA_REGION_WRITE(Controller,
  1824. EdmaSecondaryEventClearLow + Offset,
  1825. 1 << Channel);
  1826. EDMA_WRITE(Controller, EdmaEventMissedClearLow + Offset, 1 << Channel);
  1827. return;
  1828. }
  1829. RUNLEVEL
  1830. EdmapAcquireLock (
  1831. PEDMA_CONTROLLER Controller
  1832. )
  1833. /*++
  1834. Routine Description:
  1835. This routine raises to dispatch and acquires the DMA controller's lock.
  1836. Arguments:
  1837. Controller - Supplies a pointer to the controller to lock.
  1838. Return Value:
  1839. Returns the previous runlevel, which should be passed into the release
  1840. function.
  1841. --*/
  1842. {
  1843. RUNLEVEL OldRunLevel;
  1844. OldRunLevel = KeRaiseRunLevel(RunLevelDispatch);
  1845. KeAcquireSpinLock(&(Controller->Lock));
  1846. return OldRunLevel;
  1847. }
  1848. VOID
  1849. EdmapReleaseLock (
  1850. PEDMA_CONTROLLER Controller,
  1851. RUNLEVEL OldRunLevel
  1852. )
  1853. /*++
  1854. Routine Description:
  1855. This routine releases the DMA controller's lock and lowers to the runlevel
  1856. the system was at before the acquire.
  1857. Arguments:
  1858. Controller - Supplies a pointer to the controller to unlock.
  1859. OldRunLevel - Supplies the runlevel returned by the acquire function.
  1860. Return Value:
  1861. None.
  1862. --*/
  1863. {
  1864. KeReleaseSpinLock(&(Controller->Lock));
  1865. KeLowerRunLevel(OldRunLevel);
  1866. return;
  1867. }