taslock.c 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337
  1. /*
  2. * This file is part of the UCB release of Plan 9. It is subject to the license
  3. * terms in the LICENSE file found in the top-level directory of this
  4. * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
  5. * part of the UCB release of Plan 9, including this file, may be copied,
  6. * modified, propagated, or distributed except according to the terms contained
  7. * in the LICENSE file.
  8. */
  9. #include "u.h"
  10. #include "../port/lib.h"
  11. #include "mem.h"
  12. #include "dat.h"
  13. #include "fns.h"
  14. #include "../port/edf.h"
  15. /*
  16. * measure max lock cycles and max lock waiting time.
  17. */
  18. #define LOCKCYCLES 0
  19. uint64_t maxlockcycles;
  20. uint64_t maxilockcycles;
  21. uintptr_t maxlockpc;
  22. uintptr_t maxilockpc;
  23. Lockstats lockstats;
  24. Waitstats waitstats;
  25. Lock waitstatslk;
  26. static void
  27. newwaitstats(void)
  28. {
  29. if(waitstats.pcs != nil)
  30. return;
  31. waitstats.pcs = malloc(NWstats * sizeof waitstats.pcs[0]);
  32. waitstats.ns = malloc(NWstats * sizeof waitstats.ns[0]);
  33. waitstats.wait = malloc(NWstats * sizeof waitstats.wait[0]);
  34. waitstats.total = malloc(NWstats * sizeof waitstats.total[0]);
  35. waitstats.type = malloc(NWstats * sizeof waitstats.type[0]);
  36. }
  37. void
  38. startwaitstats(int on)
  39. {
  40. newwaitstats();
  41. mfence();
  42. waitstats.on = on;
  43. print("lockstats %s\n", on?"on":"off");
  44. }
  45. void
  46. clearwaitstats(void)
  47. {
  48. newwaitstats();
  49. memset(waitstats.ns, 0, NWstats * sizeof(int));
  50. memset(waitstats.wait, 0, NWstats * sizeof(uint64_t));
  51. memset(waitstats.total, 0, NWstats * sizeof(uint64_t));
  52. }
  53. void
  54. addwaitstat(uintptr_t pc, uint64_t t0, int type)
  55. {
  56. uint i;
  57. uint64_t w;
  58. if(waitstats.on == 0)
  59. return;
  60. cycles(&w);
  61. w -= t0;
  62. mfence();
  63. for(i = 0; i < NWstats; i++)
  64. if(waitstats.pcs[i] == pc){
  65. ainc(&waitstats.ns[i]);
  66. if(w > waitstats.wait[i])
  67. waitstats.wait[i] = w; /* race but ok */
  68. waitstats.total[i] += w; /* race but ok */
  69. return;
  70. }
  71. if(!canlock(&waitstatslk))
  72. return;
  73. for(i = 0; i < NWstats; i++)
  74. if(waitstats.pcs[i] == pc){
  75. ainc(&waitstats.ns[i]);
  76. if(w > waitstats.wait[i])
  77. waitstats.wait[i] = w; /* race but ok */
  78. waitstats.total[i] += w;
  79. unlock(&waitstatslk);
  80. return;
  81. }
  82. for(i = 0; i < NWstats; i++)
  83. if(waitstats.pcs[i] == 0){
  84. waitstats.ns[i] = 1;
  85. waitstats.type[i] = type;
  86. waitstats.wait[i] = w;
  87. waitstats.total[i] = w;
  88. mfence();
  89. waitstats.pcs[i] = pc;
  90. waitstats.npcs++;
  91. break;
  92. }
  93. unlock(&waitstatslk);
  94. }
  95. void
  96. lockloop(Lock *l, uintptr_t pc)
  97. {
  98. Proc *up = externup();
  99. Proc *p;
  100. p = l->p;
  101. print("lock %#p loop key %#x pc %#p held by pc %#p proc %d\n",
  102. l, l->key, pc, l->_pc, p ? p->pid : 0);
  103. dumpaproc(up);
  104. if(p != nil)
  105. dumpaproc(p);
  106. }
  107. int
  108. lock(Lock *l)
  109. {
  110. Proc *up = externup();
  111. int i;
  112. uintptr_t pc;
  113. uint64_t t0;
  114. pc = getcallerpc();
  115. lockstats.locks++;
  116. if(up)
  117. ainc(&up->nlocks); /* prevent being scheded */
  118. if(TAS(&l->key) == 0){
  119. if(up)
  120. up->lastlock = l;
  121. l->_pc = pc;
  122. l->p = up;
  123. l->isilock = 0;
  124. if(LOCKCYCLES)
  125. cycles(&l->lockcycles);
  126. return 0;
  127. }
  128. if(up)
  129. adec(&up->nlocks);
  130. cycles(&t0);
  131. lockstats.glare++;
  132. for(;;){
  133. lockstats.inglare++;
  134. i = 0;
  135. while(l->key){
  136. if(sys->nmach < 2 && up && up->edf && (up->edf->flags & Admitted)){
  137. /*
  138. * Priority inversion, yield on a uniprocessor; on a
  139. * multiprocessor, the other processor will unlock
  140. */
  141. print("inversion %#p pc %#p proc %d held by pc %#p proc %d\n",
  142. l, pc, up ? up->pid : 0, l->_pc, l->p ? l->p->pid : 0);
  143. up->edf->d = todget(nil); /* yield to process with lock */
  144. }
  145. if(i++ > 100000000){
  146. i = 0;
  147. lockloop(l, pc);
  148. }
  149. }
  150. if(up)
  151. ainc(&up->nlocks);
  152. if(TAS(&l->key) == 0){
  153. if(up)
  154. up->lastlock = l;
  155. l->_pc = pc;
  156. l->p = up;
  157. l->isilock = 0;
  158. if(LOCKCYCLES)
  159. cycles(&l->lockcycles);
  160. if(l != &waitstatslk)
  161. addwaitstat(pc, t0, WSlock);
  162. return 1;
  163. }
  164. if(up)
  165. adec(&up->nlocks);
  166. }
  167. }
  168. void
  169. ilock(Lock *l)
  170. {
  171. Proc *up = externup();
  172. Mpl pl;
  173. uintptr_t pc;
  174. uint64_t t0;
  175. pc = getcallerpc();
  176. lockstats.locks++;
  177. pl = splhi();
  178. if(TAS(&l->key) != 0){
  179. cycles(&t0);
  180. lockstats.glare++;
  181. /*
  182. * Cannot also check l->pc, l->m, or l->isilock here
  183. * because they might just not be set yet, or
  184. * (for pc and m) the lock might have just been unlocked.
  185. */
  186. for(;;){
  187. lockstats.inglare++;
  188. splx(pl);
  189. while(l->key)
  190. ;
  191. pl = splhi();
  192. if(TAS(&l->key) == 0){
  193. if(l != &waitstatslk)
  194. addwaitstat(pc, t0, WSlock);
  195. goto acquire;
  196. }
  197. }
  198. }
  199. acquire:
  200. machp()->ilockdepth++;
  201. if(up)
  202. up->lastilock = l;
  203. l->pl = pl;
  204. l->_pc = pc;
  205. l->p = up;
  206. l->isilock = 1;
  207. l->m = machp();
  208. if(LOCKCYCLES)
  209. cycles(&l->lockcycles);
  210. }
  211. int
  212. canlock(Lock *l)
  213. {
  214. Proc *up = externup();
  215. if(up)
  216. ainc(&up->nlocks);
  217. if(TAS(&l->key)){
  218. if(up)
  219. adec(&up->nlocks);
  220. return 0;
  221. }
  222. if(up)
  223. up->lastlock = l;
  224. l->_pc = getcallerpc();
  225. l->p = up;
  226. l->isilock = 0;
  227. if(LOCKCYCLES)
  228. cycles(&l->lockcycles);
  229. return 1;
  230. }
  231. void
  232. unlock(Lock *l)
  233. {
  234. Proc *up = externup();
  235. uint64_t x;
  236. if(LOCKCYCLES){
  237. cycles(&x);
  238. l->lockcycles = x - l->lockcycles;
  239. if(l->lockcycles > maxlockcycles){
  240. maxlockcycles = l->lockcycles;
  241. maxlockpc = l->_pc;
  242. }
  243. }
  244. if(l->key == 0)
  245. print("unlock: not locked: pc %#p\n", getcallerpc());
  246. if(l->isilock)
  247. print("unlock of ilock: pc %#p, held by %#p\n", getcallerpc(), l->_pc);
  248. if(l->p != up)
  249. print("unlock: up changed: pc %#p, acquired at pc %#p, lock p %#p, unlock up %#p\n", getcallerpc(), l->_pc, l->p, up);
  250. l->m = nil;
  251. l->key = 0;
  252. coherence();
  253. if(up && adec(&up->nlocks) == 0 && up->delaysched && islo()){
  254. /*
  255. * Call sched if the need arose while locks were held
  256. * But, don't do it from interrupt routines, hence the islo() test
  257. */
  258. sched();
  259. }
  260. }
  261. void
  262. iunlock(Lock *l)
  263. {
  264. Proc *up = externup();
  265. Mpl pl;
  266. uint64_t x;
  267. if(LOCKCYCLES){
  268. cycles(&x);
  269. l->lockcycles = x - l->lockcycles;
  270. if(l->lockcycles > maxilockcycles){
  271. maxilockcycles = l->lockcycles;
  272. maxilockpc = l->_pc;
  273. }
  274. }
  275. if(l->key == 0)
  276. print("iunlock: not locked: pc %#p\n", getcallerpc());
  277. if(!l->isilock)
  278. print("iunlock of lock: pc %#p, held by %#p\n", getcallerpc(), l->_pc);
  279. if(islo())
  280. print("iunlock while lo: pc %#p, held by %#p\n", getcallerpc(), l->_pc);
  281. if(l->m != machp()){
  282. print("iunlock by cpu%d, locked by cpu%d: pc %#p, held by %#p\n",
  283. machp()->machno, l->m->machno, getcallerpc(), l->_pc);
  284. }
  285. pl = l->pl;
  286. l->m = nil;
  287. l->key = 0;
  288. coherence();
  289. machp()->ilockdepth--;
  290. if(up)
  291. up->lastilock = nil;
  292. splx(pl);
  293. }
  294. void
  295. portmwait(void *value)
  296. {
  297. while (*(void**)value == nil)
  298. ;
  299. }
  300. void (*mwait)(void *) = portmwait;