2
0

dynlink.c 66 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438
  1. #define _GNU_SOURCE
  2. #define SYSCALL_NO_TLS 1
  3. #include <stdlib.h>
  4. #include <stdarg.h>
  5. #include <stddef.h>
  6. #include <string.h>
  7. #include <unistd.h>
  8. #include <stdint.h>
  9. #include <elf.h>
  10. #include <sys/mman.h>
  11. #include <limits.h>
  12. #include <fcntl.h>
  13. #include <sys/stat.h>
  14. #include <errno.h>
  15. #include <link.h>
  16. #include <setjmp.h>
  17. #include <pthread.h>
  18. #include <ctype.h>
  19. #include <dlfcn.h>
  20. #include <semaphore.h>
  21. #include <sys/membarrier.h>
  22. #include "pthread_impl.h"
  23. #include "fork_impl.h"
  24. #include "libc.h"
  25. #include "dynlink.h"
  26. static size_t ldso_page_size;
  27. /* libc.h may have defined a macro for dynamic PAGE_SIZE already, but
  28. * PAGESIZE is only defined if it's constant for the arch. */
  29. #ifndef PAGESIZE
  30. #undef PAGE_SIZE
  31. #define PAGE_SIZE ldso_page_size
  32. #endif
  33. #define malloc __libc_malloc
  34. #define calloc __libc_calloc
  35. #define realloc __libc_realloc
  36. #define free __libc_free
  37. static void error_impl(const char *, ...);
  38. static void error_noop(const char *, ...);
  39. static void (*error)(const char *, ...) = error_noop;
  40. #define MAXP2(a,b) (-(-(a)&-(b)))
  41. #define ALIGN(x,y) ((x)+(y)-1 & -(y))
  42. #define container_of(p,t,m) ((t*)((char *)(p)-offsetof(t,m)))
  43. #define countof(a) ((sizeof (a))/(sizeof (a)[0]))
  44. struct debug {
  45. int ver;
  46. void *head;
  47. void (*bp)(void);
  48. int state;
  49. void *base;
  50. };
  51. struct td_index {
  52. size_t args[2];
  53. struct td_index *next;
  54. };
  55. struct dso {
  56. #if DL_FDPIC
  57. struct fdpic_loadmap *loadmap;
  58. #else
  59. unsigned char *base;
  60. #endif
  61. char *name;
  62. size_t *dynv;
  63. struct dso *next, *prev;
  64. Phdr *phdr;
  65. int phnum;
  66. size_t phentsize;
  67. Sym *syms;
  68. Elf_Symndx *hashtab;
  69. uint32_t *ghashtab;
  70. int16_t *versym;
  71. char *strings;
  72. struct dso *syms_next, *lazy_next;
  73. size_t *lazy, lazy_cnt;
  74. unsigned char *map;
  75. size_t map_len;
  76. dev_t dev;
  77. ino_t ino;
  78. char relocated;
  79. char constructed;
  80. char kernel_mapped;
  81. char mark;
  82. char bfs_built;
  83. char runtime_loaded;
  84. struct dso **deps, *needed_by;
  85. size_t ndeps_direct;
  86. size_t next_dep;
  87. pthread_t ctor_visitor;
  88. char *rpath_orig, *rpath;
  89. struct tls_module tls;
  90. size_t tls_id;
  91. size_t relro_start, relro_end;
  92. uintptr_t *new_dtv;
  93. unsigned char *new_tls;
  94. struct td_index *td_index;
  95. struct dso *fini_next;
  96. char *shortname;
  97. #if DL_FDPIC
  98. unsigned char *base;
  99. #else
  100. struct fdpic_loadmap *loadmap;
  101. #endif
  102. struct funcdesc {
  103. void *addr;
  104. size_t *got;
  105. } *funcdescs;
  106. size_t *got;
  107. char buf[];
  108. };
  109. struct symdef {
  110. Sym *sym;
  111. struct dso *dso;
  112. };
  113. typedef void (*stage3_func)(size_t *, size_t *);
  114. static struct builtin_tls {
  115. char c;
  116. struct pthread pt;
  117. void *space[16];
  118. } builtin_tls[1];
  119. #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
  120. #define ADDEND_LIMIT 4096
  121. static size_t *saved_addends, *apply_addends_to;
  122. static struct dso ldso;
  123. static struct dso *head, *tail, *fini_head, *syms_tail, *lazy_head;
  124. static char *env_path, *sys_path;
  125. static unsigned long long gencnt;
  126. static int runtime;
  127. static int ldd_mode;
  128. static int ldso_fail;
  129. static int noload;
  130. static int shutting_down;
  131. static jmp_buf *rtld_fail;
  132. static pthread_rwlock_t lock;
  133. static struct debug debug;
  134. static struct tls_module *tls_tail;
  135. static size_t tls_cnt, tls_offset, tls_align = MIN_TLS_ALIGN;
  136. static size_t static_tls_cnt;
  137. static pthread_mutex_t init_fini_lock;
  138. static pthread_cond_t ctor_cond;
  139. static struct dso *builtin_deps[2];
  140. static struct dso *const no_deps[1];
  141. static struct dso *builtin_ctor_queue[4];
  142. static struct dso **main_ctor_queue;
  143. static struct fdpic_loadmap *app_loadmap;
  144. static struct fdpic_dummy_loadmap app_dummy_loadmap;
  145. struct debug *_dl_debug_addr = &debug;
  146. extern weak hidden char __ehdr_start[];
  147. extern hidden int __malloc_replaced;
  148. hidden void (*const __init_array_start)(void)=0, (*const __fini_array_start)(void)=0;
  149. extern hidden void (*const __init_array_end)(void), (*const __fini_array_end)(void);
  150. weak_alias(__init_array_start, __init_array_end);
  151. weak_alias(__fini_array_start, __fini_array_end);
  152. static int dl_strcmp(const char *l, const char *r)
  153. {
  154. for (; *l==*r && *l; l++, r++);
  155. return *(unsigned char *)l - *(unsigned char *)r;
  156. }
  157. #define strcmp(l,r) dl_strcmp(l,r)
  158. /* Compute load address for a virtual address in a given dso. */
  159. #if DL_FDPIC
  160. static void *laddr(const struct dso *p, size_t v)
  161. {
  162. size_t j=0;
  163. if (!p->loadmap) return p->base + v;
  164. for (j=0; v-p->loadmap->segs[j].p_vaddr >= p->loadmap->segs[j].p_memsz; j++);
  165. return (void *)(v - p->loadmap->segs[j].p_vaddr + p->loadmap->segs[j].addr);
  166. }
  167. static void *laddr_pg(const struct dso *p, size_t v)
  168. {
  169. size_t j=0;
  170. size_t pgsz = PAGE_SIZE;
  171. if (!p->loadmap) return p->base + v;
  172. for (j=0; ; j++) {
  173. size_t a = p->loadmap->segs[j].p_vaddr;
  174. size_t b = a + p->loadmap->segs[j].p_memsz;
  175. a &= -pgsz;
  176. b += pgsz-1;
  177. b &= -pgsz;
  178. if (v-a<b-a) break;
  179. }
  180. return (void *)(v - p->loadmap->segs[j].p_vaddr + p->loadmap->segs[j].addr);
  181. }
  182. static void (*fdbarrier(void *p))()
  183. {
  184. void (*fd)();
  185. __asm__("" : "=r"(fd) : "0"(p));
  186. return fd;
  187. }
  188. #define fpaddr(p, v) fdbarrier((&(struct funcdesc){ \
  189. laddr(p, v), (p)->got }))
  190. #else
  191. #define laddr(p, v) (void *)((p)->base + (v))
  192. #define laddr_pg(p, v) laddr(p, v)
  193. #define fpaddr(p, v) ((void (*)())laddr(p, v))
  194. #endif
  195. static void decode_vec(size_t *v, size_t *a, size_t cnt)
  196. {
  197. size_t i;
  198. for (i=0; i<cnt; i++) a[i] = 0;
  199. for (; v[0]; v+=2) if (v[0]-1<cnt-1) {
  200. if (v[0] < 8*sizeof(long))
  201. a[0] |= 1UL<<v[0];
  202. a[v[0]] = v[1];
  203. }
  204. }
  205. static int search_vec(size_t *v, size_t *r, size_t key)
  206. {
  207. for (; v[0]!=key; v+=2)
  208. if (!v[0]) return 0;
  209. *r = v[1];
  210. return 1;
  211. }
  212. static uint32_t sysv_hash(const char *s0)
  213. {
  214. const unsigned char *s = (void *)s0;
  215. uint_fast32_t h = 0;
  216. while (*s) {
  217. h = 16*h + *s++;
  218. h ^= h>>24 & 0xf0;
  219. }
  220. return h & 0xfffffff;
  221. }
  222. static uint32_t gnu_hash(const char *s0)
  223. {
  224. const unsigned char *s = (void *)s0;
  225. uint_fast32_t h = 5381;
  226. for (; *s; s++)
  227. h += h*32 + *s;
  228. return h;
  229. }
  230. static Sym *sysv_lookup(const char *s, uint32_t h, struct dso *dso)
  231. {
  232. size_t i;
  233. Sym *syms = dso->syms;
  234. Elf_Symndx *hashtab = dso->hashtab;
  235. char *strings = dso->strings;
  236. for (i=hashtab[2+h%hashtab[0]]; i; i=hashtab[2+hashtab[0]+i]) {
  237. if ((!dso->versym || dso->versym[i] >= 0)
  238. && (!strcmp(s, strings+syms[i].st_name)))
  239. return syms+i;
  240. }
  241. return 0;
  242. }
  243. static Sym *gnu_lookup(uint32_t h1, uint32_t *hashtab, struct dso *dso, const char *s)
  244. {
  245. uint32_t nbuckets = hashtab[0];
  246. uint32_t *buckets = hashtab + 4 + hashtab[2]*(sizeof(size_t)/4);
  247. uint32_t i = buckets[h1 % nbuckets];
  248. if (!i) return 0;
  249. uint32_t *hashval = buckets + nbuckets + (i - hashtab[1]);
  250. for (h1 |= 1; ; i++) {
  251. uint32_t h2 = *hashval++;
  252. if ((h1 == (h2|1)) && (!dso->versym || dso->versym[i] >= 0)
  253. && !strcmp(s, dso->strings + dso->syms[i].st_name))
  254. return dso->syms+i;
  255. if (h2 & 1) break;
  256. }
  257. return 0;
  258. }
  259. static Sym *gnu_lookup_filtered(uint32_t h1, uint32_t *hashtab, struct dso *dso, const char *s, uint32_t fofs, size_t fmask)
  260. {
  261. const size_t *bloomwords = (const void *)(hashtab+4);
  262. size_t f = bloomwords[fofs & (hashtab[2]-1)];
  263. if (!(f & fmask)) return 0;
  264. f >>= (h1 >> hashtab[3]) % (8 * sizeof f);
  265. if (!(f & 1)) return 0;
  266. return gnu_lookup(h1, hashtab, dso, s);
  267. }
  268. #define OK_TYPES (1<<STT_NOTYPE | 1<<STT_OBJECT | 1<<STT_FUNC | 1<<STT_COMMON | 1<<STT_TLS)
  269. #define OK_BINDS (1<<STB_GLOBAL | 1<<STB_WEAK | 1<<STB_GNU_UNIQUE)
  270. #ifndef ARCH_SYM_REJECT_UND
  271. #define ARCH_SYM_REJECT_UND(s) 0
  272. #endif
  273. #if defined(__GNUC__)
  274. __attribute__((always_inline))
  275. #endif
  276. static inline struct symdef find_sym2(struct dso *dso, const char *s, int need_def, int use_deps)
  277. {
  278. uint32_t h = 0, gh = gnu_hash(s), gho = gh / (8*sizeof(size_t)), *ght;
  279. size_t ghm = 1ul << gh % (8*sizeof(size_t));
  280. struct symdef def = {0};
  281. struct dso **deps = use_deps ? dso->deps : 0;
  282. for (; dso; dso=use_deps ? *deps++ : dso->syms_next) {
  283. Sym *sym;
  284. if ((ght = dso->ghashtab)) {
  285. sym = gnu_lookup_filtered(gh, ght, dso, s, gho, ghm);
  286. } else {
  287. if (!h) h = sysv_hash(s);
  288. sym = sysv_lookup(s, h, dso);
  289. }
  290. if (!sym) continue;
  291. if (!sym->st_shndx)
  292. if (need_def || (sym->st_info&0xf) == STT_TLS
  293. || ARCH_SYM_REJECT_UND(sym))
  294. continue;
  295. if (!sym->st_value)
  296. if ((sym->st_info&0xf) != STT_TLS)
  297. continue;
  298. if (!(1<<(sym->st_info&0xf) & OK_TYPES)) continue;
  299. if (!(1<<(sym->st_info>>4) & OK_BINDS)) continue;
  300. def.sym = sym;
  301. def.dso = dso;
  302. break;
  303. }
  304. return def;
  305. }
  306. static struct symdef find_sym(struct dso *dso, const char *s, int need_def)
  307. {
  308. return find_sym2(dso, s, need_def, 0);
  309. }
  310. static struct symdef get_lfs64(const char *name)
  311. {
  312. const char *p;
  313. static const char lfs64_list[] =
  314. "aio_cancel\0aio_error\0aio_fsync\0aio_read\0aio_return\0"
  315. "aio_suspend\0aio_write\0alphasort\0creat\0fallocate\0"
  316. "fgetpos\0fopen\0freopen\0fseeko\0fsetpos\0fstat\0"
  317. "fstatat\0fstatfs\0fstatvfs\0ftello\0ftruncate\0ftw\0"
  318. "getdents\0getrlimit\0glob\0globfree\0lio_listio\0"
  319. "lockf\0lseek\0lstat\0mkostemp\0mkostemps\0mkstemp\0"
  320. "mkstemps\0mmap\0nftw\0open\0openat\0posix_fadvise\0"
  321. "posix_fallocate\0pread\0preadv\0prlimit\0pwrite\0"
  322. "pwritev\0readdir\0scandir\0sendfile\0setrlimit\0"
  323. "stat\0statfs\0statvfs\0tmpfile\0truncate\0versionsort\0"
  324. "__fxstat\0__fxstatat\0__lxstat\0__xstat\0";
  325. if (!strcmp(name, "readdir64_r"))
  326. return find_sym(&ldso, "readdir_r", 1);
  327. size_t l = strnlen(name, 18);
  328. if (l<2 || name[l-2]!='6' || name[l-1]!='4' || name[l])
  329. goto nomatch;
  330. for (p=lfs64_list; *p; p++) {
  331. if (!strncmp(name, p, l-2) && !p[l-2])
  332. return find_sym(&ldso, p, 1);
  333. while (*p) p++;
  334. }
  335. nomatch:
  336. return (struct symdef){ 0 };
  337. }
  338. static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stride)
  339. {
  340. unsigned char *base = dso->base;
  341. Sym *syms = dso->syms;
  342. char *strings = dso->strings;
  343. Sym *sym;
  344. const char *name;
  345. void *ctx;
  346. int type;
  347. int sym_index;
  348. struct symdef def;
  349. size_t *reloc_addr;
  350. size_t sym_val;
  351. size_t tls_val;
  352. size_t addend;
  353. int skip_relative = 0, reuse_addends = 0, save_slot = 0;
  354. if (dso == &ldso) {
  355. /* Only ldso's REL table needs addend saving/reuse. */
  356. if (rel == apply_addends_to)
  357. reuse_addends = 1;
  358. skip_relative = 1;
  359. }
  360. for (; rel_size; rel+=stride, rel_size-=stride*sizeof(size_t)) {
  361. if (skip_relative && IS_RELATIVE(rel[1], dso->syms)) continue;
  362. type = R_TYPE(rel[1]);
  363. if (type == REL_NONE) continue;
  364. reloc_addr = laddr(dso, rel[0]);
  365. if (stride > 2) {
  366. addend = rel[2];
  367. } else if (type==REL_GOT || type==REL_PLT|| type==REL_COPY) {
  368. addend = 0;
  369. } else if (reuse_addends) {
  370. /* Save original addend in stage 2 where the dso
  371. * chain consists of just ldso; otherwise read back
  372. * saved addend since the inline one was clobbered. */
  373. if (head==&ldso)
  374. saved_addends[save_slot] = *reloc_addr;
  375. addend = saved_addends[save_slot++];
  376. } else {
  377. addend = *reloc_addr;
  378. }
  379. sym_index = R_SYM(rel[1]);
  380. if (sym_index) {
  381. sym = syms + sym_index;
  382. name = strings + sym->st_name;
  383. ctx = type==REL_COPY ? head->syms_next : head;
  384. def = (sym->st_info>>4) == STB_LOCAL
  385. ? (struct symdef){ .dso = dso, .sym = sym }
  386. : find_sym(ctx, name, type==REL_PLT);
  387. if (!def.sym) def = get_lfs64(name);
  388. if (!def.sym && (sym->st_shndx != SHN_UNDEF
  389. || sym->st_info>>4 != STB_WEAK)) {
  390. if (dso->lazy && (type==REL_PLT || type==REL_GOT)) {
  391. dso->lazy[3*dso->lazy_cnt+0] = rel[0];
  392. dso->lazy[3*dso->lazy_cnt+1] = rel[1];
  393. dso->lazy[3*dso->lazy_cnt+2] = addend;
  394. dso->lazy_cnt++;
  395. continue;
  396. }
  397. error("Error relocating %s: %s: symbol not found",
  398. dso->name, name);
  399. if (runtime) longjmp(*rtld_fail, 1);
  400. continue;
  401. }
  402. } else {
  403. sym = 0;
  404. def.sym = 0;
  405. def.dso = dso;
  406. }
  407. sym_val = def.sym ? (size_t)laddr(def.dso, def.sym->st_value) : 0;
  408. tls_val = def.sym ? def.sym->st_value : 0;
  409. if ((type == REL_TPOFF || type == REL_TPOFF_NEG)
  410. && def.dso->tls_id > static_tls_cnt) {
  411. error("Error relocating %s: %s: initial-exec TLS "
  412. "resolves to dynamic definition in %s",
  413. dso->name, name, def.dso->name);
  414. longjmp(*rtld_fail, 1);
  415. }
  416. switch(type) {
  417. case REL_OFFSET:
  418. addend -= (size_t)reloc_addr;
  419. case REL_SYMBOLIC:
  420. case REL_GOT:
  421. case REL_PLT:
  422. *reloc_addr = sym_val + addend;
  423. break;
  424. case REL_USYMBOLIC:
  425. memcpy(reloc_addr, &(size_t){sym_val + addend}, sizeof(size_t));
  426. break;
  427. case REL_RELATIVE:
  428. *reloc_addr = (size_t)base + addend;
  429. break;
  430. case REL_SYM_OR_REL:
  431. if (sym) *reloc_addr = sym_val + addend;
  432. else *reloc_addr = (size_t)base + addend;
  433. break;
  434. case REL_COPY:
  435. memcpy(reloc_addr, (void *)sym_val, sym->st_size);
  436. break;
  437. case REL_OFFSET32:
  438. *(uint32_t *)reloc_addr = sym_val + addend
  439. - (size_t)reloc_addr;
  440. break;
  441. case REL_FUNCDESC:
  442. *reloc_addr = def.sym ? (size_t)(def.dso->funcdescs
  443. + (def.sym - def.dso->syms)) : 0;
  444. break;
  445. case REL_FUNCDESC_VAL:
  446. if ((sym->st_info&0xf) == STT_SECTION) *reloc_addr += sym_val;
  447. else *reloc_addr = sym_val;
  448. reloc_addr[1] = def.sym ? (size_t)def.dso->got : 0;
  449. break;
  450. case REL_DTPMOD:
  451. *reloc_addr = def.dso->tls_id;
  452. break;
  453. case REL_DTPOFF:
  454. *reloc_addr = tls_val + addend - DTP_OFFSET;
  455. break;
  456. #ifdef TLS_ABOVE_TP
  457. case REL_TPOFF:
  458. *reloc_addr = tls_val + def.dso->tls.offset + TPOFF_K + addend;
  459. break;
  460. #else
  461. case REL_TPOFF:
  462. *reloc_addr = tls_val - def.dso->tls.offset + addend;
  463. break;
  464. case REL_TPOFF_NEG:
  465. *reloc_addr = def.dso->tls.offset - tls_val + addend;
  466. break;
  467. #endif
  468. case REL_TLSDESC:
  469. if (stride<3) addend = reloc_addr[!TLSDESC_BACKWARDS];
  470. if (def.dso->tls_id > static_tls_cnt) {
  471. struct td_index *new = malloc(sizeof *new);
  472. if (!new) {
  473. error(
  474. "Error relocating %s: cannot allocate TLSDESC for %s",
  475. dso->name, sym ? name : "(local)" );
  476. longjmp(*rtld_fail, 1);
  477. }
  478. new->next = dso->td_index;
  479. dso->td_index = new;
  480. new->args[0] = def.dso->tls_id;
  481. new->args[1] = tls_val + addend - DTP_OFFSET;
  482. reloc_addr[0] = (size_t)__tlsdesc_dynamic;
  483. reloc_addr[1] = (size_t)new;
  484. } else {
  485. reloc_addr[0] = (size_t)__tlsdesc_static;
  486. #ifdef TLS_ABOVE_TP
  487. reloc_addr[1] = tls_val + def.dso->tls.offset
  488. + TPOFF_K + addend;
  489. #else
  490. reloc_addr[1] = tls_val - def.dso->tls.offset
  491. + addend;
  492. #endif
  493. }
  494. /* Some archs (32-bit ARM at least) invert the order of
  495. * the descriptor members. Fix them up here. */
  496. if (TLSDESC_BACKWARDS) {
  497. size_t tmp = reloc_addr[0];
  498. reloc_addr[0] = reloc_addr[1];
  499. reloc_addr[1] = tmp;
  500. }
  501. break;
  502. default:
  503. error("Error relocating %s: unsupported relocation type %d",
  504. dso->name, type);
  505. if (runtime) longjmp(*rtld_fail, 1);
  506. continue;
  507. }
  508. }
  509. }
  510. static void do_relr_relocs(struct dso *dso, size_t *relr, size_t relr_size)
  511. {
  512. if (dso == &ldso) return; /* self-relocation was done in _dlstart */
  513. unsigned char *base = dso->base;
  514. size_t *reloc_addr;
  515. for (; relr_size; relr++, relr_size-=sizeof(size_t))
  516. if ((relr[0]&1) == 0) {
  517. reloc_addr = laddr(dso, relr[0]);
  518. *reloc_addr++ += (size_t)base;
  519. } else {
  520. int i = 0;
  521. for (size_t bitmap=relr[0]; (bitmap>>=1); i++)
  522. if (bitmap&1)
  523. reloc_addr[i] += (size_t)base;
  524. reloc_addr += 8*sizeof(size_t)-1;
  525. }
  526. }
  527. static void redo_lazy_relocs()
  528. {
  529. struct dso *p = lazy_head, *next;
  530. lazy_head = 0;
  531. for (; p; p=next) {
  532. next = p->lazy_next;
  533. size_t size = p->lazy_cnt*3*sizeof(size_t);
  534. p->lazy_cnt = 0;
  535. do_relocs(p, p->lazy, size, 3);
  536. if (p->lazy_cnt) {
  537. p->lazy_next = lazy_head;
  538. lazy_head = p;
  539. } else {
  540. free(p->lazy);
  541. p->lazy = 0;
  542. p->lazy_next = 0;
  543. }
  544. }
  545. }
  546. /* A huge hack: to make up for the wastefulness of shared libraries
  547. * needing at least a page of dirty memory even if they have no global
  548. * data, we reclaim the gaps at the beginning and end of writable maps
  549. * and "donate" them to the heap. */
  550. static void reclaim(struct dso *dso, size_t start, size_t end)
  551. {
  552. if (start >= dso->relro_start && start < dso->relro_end) start = dso->relro_end;
  553. if (end >= dso->relro_start && end < dso->relro_end) end = dso->relro_start;
  554. if (start >= end) return;
  555. char *base = laddr_pg(dso, start);
  556. __malloc_donate(base, base+(end-start));
  557. }
  558. static void reclaim_gaps(struct dso *dso)
  559. {
  560. Phdr *ph = dso->phdr;
  561. size_t phcnt = dso->phnum;
  562. for (; phcnt--; ph=(void *)((char *)ph+dso->phentsize)) {
  563. if (ph->p_type!=PT_LOAD) continue;
  564. if ((ph->p_flags&(PF_R|PF_W))!=(PF_R|PF_W)) continue;
  565. reclaim(dso, ph->p_vaddr & -PAGE_SIZE, ph->p_vaddr);
  566. reclaim(dso, ph->p_vaddr+ph->p_memsz,
  567. ph->p_vaddr+ph->p_memsz+PAGE_SIZE-1 & -PAGE_SIZE);
  568. }
  569. }
  570. static ssize_t read_loop(int fd, void *p, size_t n)
  571. {
  572. for (size_t i=0; i<n; ) {
  573. ssize_t l = read(fd, (char *)p+i, n-i);
  574. if (l<0) {
  575. if (errno==EINTR) continue;
  576. else return -1;
  577. }
  578. if (l==0) return i;
  579. i += l;
  580. }
  581. return n;
  582. }
  583. static void *mmap_fixed(void *p, size_t n, int prot, int flags, int fd, off_t off)
  584. {
  585. static int no_map_fixed;
  586. char *q;
  587. if (!n) return p;
  588. if (!no_map_fixed) {
  589. q = mmap(p, n, prot, flags|MAP_FIXED, fd, off);
  590. if (!DL_NOMMU_SUPPORT || q != MAP_FAILED || errno != EINVAL)
  591. return q;
  592. no_map_fixed = 1;
  593. }
  594. /* Fallbacks for MAP_FIXED failure on NOMMU kernels. */
  595. if (flags & MAP_ANONYMOUS) {
  596. memset(p, 0, n);
  597. return p;
  598. }
  599. ssize_t r;
  600. if (lseek(fd, off, SEEK_SET) < 0) return MAP_FAILED;
  601. for (q=p; n; q+=r, off+=r, n-=r) {
  602. r = read(fd, q, n);
  603. if (r < 0 && errno != EINTR) return MAP_FAILED;
  604. if (!r) {
  605. memset(q, 0, n);
  606. break;
  607. }
  608. }
  609. return p;
  610. }
  611. static void unmap_library(struct dso *dso)
  612. {
  613. if (dso->loadmap) {
  614. size_t i;
  615. for (i=0; i<dso->loadmap->nsegs; i++) {
  616. if (!dso->loadmap->segs[i].p_memsz)
  617. continue;
  618. munmap((void *)dso->loadmap->segs[i].addr,
  619. dso->loadmap->segs[i].p_memsz);
  620. }
  621. free(dso->loadmap);
  622. } else if (dso->map && dso->map_len) {
  623. munmap(dso->map, dso->map_len);
  624. }
  625. }
  626. static void *map_library(int fd, struct dso *dso)
  627. {
  628. Ehdr buf[(896+sizeof(Ehdr))/sizeof(Ehdr)];
  629. void *allocated_buf=0;
  630. size_t phsize;
  631. size_t addr_min=SIZE_MAX, addr_max=0, map_len;
  632. size_t this_min, this_max;
  633. size_t nsegs = 0;
  634. off_t off_start;
  635. Ehdr *eh;
  636. Phdr *ph, *ph0;
  637. unsigned prot;
  638. unsigned char *map=MAP_FAILED, *base;
  639. size_t dyn=0;
  640. size_t tls_image=0;
  641. size_t i;
  642. ssize_t l = read(fd, buf, sizeof buf);
  643. eh = buf;
  644. if (l<0) return 0;
  645. if (l<sizeof *eh || (eh->e_type != ET_DYN && eh->e_type != ET_EXEC))
  646. goto noexec;
  647. phsize = eh->e_phentsize * eh->e_phnum;
  648. if (phsize > sizeof buf - sizeof *eh) {
  649. allocated_buf = malloc(phsize);
  650. if (!allocated_buf) return 0;
  651. l = pread(fd, allocated_buf, phsize, eh->e_phoff);
  652. if (l < 0) goto error;
  653. if (l != phsize) goto noexec;
  654. ph = ph0 = allocated_buf;
  655. } else if (eh->e_phoff + phsize > l) {
  656. l = pread(fd, buf+1, phsize, eh->e_phoff);
  657. if (l < 0) goto error;
  658. if (l != phsize) goto noexec;
  659. ph = ph0 = (void *)(buf + 1);
  660. } else {
  661. ph = ph0 = (void *)((char *)buf + eh->e_phoff);
  662. }
  663. for (i=eh->e_phnum; i; i--, ph=(void *)((char *)ph+eh->e_phentsize)) {
  664. if (ph->p_type == PT_DYNAMIC) {
  665. dyn = ph->p_vaddr;
  666. } else if (ph->p_type == PT_TLS) {
  667. tls_image = ph->p_vaddr;
  668. dso->tls.align = ph->p_align;
  669. dso->tls.len = ph->p_filesz;
  670. dso->tls.size = ph->p_memsz;
  671. } else if (ph->p_type == PT_GNU_RELRO) {
  672. dso->relro_start = ph->p_vaddr & -PAGE_SIZE;
  673. dso->relro_end = (ph->p_vaddr + ph->p_memsz) & -PAGE_SIZE;
  674. } else if (ph->p_type == PT_GNU_STACK) {
  675. if (!runtime && ph->p_memsz > __default_stacksize) {
  676. __default_stacksize =
  677. ph->p_memsz < DEFAULT_STACK_MAX ?
  678. ph->p_memsz : DEFAULT_STACK_MAX;
  679. }
  680. }
  681. if (ph->p_type != PT_LOAD) continue;
  682. nsegs++;
  683. if (ph->p_vaddr < addr_min) {
  684. addr_min = ph->p_vaddr;
  685. off_start = ph->p_offset;
  686. prot = (((ph->p_flags&PF_R) ? PROT_READ : 0) |
  687. ((ph->p_flags&PF_W) ? PROT_WRITE: 0) |
  688. ((ph->p_flags&PF_X) ? PROT_EXEC : 0));
  689. }
  690. if (ph->p_vaddr+ph->p_memsz > addr_max) {
  691. addr_max = ph->p_vaddr+ph->p_memsz;
  692. }
  693. }
  694. if (!dyn) goto noexec;
  695. if (DL_FDPIC && !(eh->e_flags & FDPIC_CONSTDISP_FLAG)) {
  696. dso->loadmap = calloc(1, sizeof *dso->loadmap
  697. + nsegs * sizeof *dso->loadmap->segs);
  698. if (!dso->loadmap) goto error;
  699. dso->loadmap->nsegs = nsegs;
  700. for (ph=ph0, i=0; i<nsegs; ph=(void *)((char *)ph+eh->e_phentsize)) {
  701. if (ph->p_type != PT_LOAD) continue;
  702. prot = (((ph->p_flags&PF_R) ? PROT_READ : 0) |
  703. ((ph->p_flags&PF_W) ? PROT_WRITE: 0) |
  704. ((ph->p_flags&PF_X) ? PROT_EXEC : 0));
  705. map = mmap(0, ph->p_memsz + (ph->p_vaddr & PAGE_SIZE-1),
  706. prot, MAP_PRIVATE,
  707. fd, ph->p_offset & -PAGE_SIZE);
  708. if (map == MAP_FAILED) {
  709. unmap_library(dso);
  710. goto error;
  711. }
  712. dso->loadmap->segs[i].addr = (size_t)map +
  713. (ph->p_vaddr & PAGE_SIZE-1);
  714. dso->loadmap->segs[i].p_vaddr = ph->p_vaddr;
  715. dso->loadmap->segs[i].p_memsz = ph->p_memsz;
  716. i++;
  717. if (prot & PROT_WRITE) {
  718. size_t brk = (ph->p_vaddr & PAGE_SIZE-1)
  719. + ph->p_filesz;
  720. size_t pgbrk = brk + PAGE_SIZE-1 & -PAGE_SIZE;
  721. size_t pgend = brk + ph->p_memsz - ph->p_filesz
  722. + PAGE_SIZE-1 & -PAGE_SIZE;
  723. if (pgend > pgbrk && mmap_fixed(map+pgbrk,
  724. pgend-pgbrk, prot,
  725. MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS,
  726. -1, off_start) == MAP_FAILED)
  727. goto error;
  728. memset(map + brk, 0, pgbrk-brk);
  729. }
  730. }
  731. map = (void *)dso->loadmap->segs[0].addr;
  732. map_len = 0;
  733. goto done_mapping;
  734. }
  735. addr_max += PAGE_SIZE-1;
  736. addr_max &= -PAGE_SIZE;
  737. addr_min &= -PAGE_SIZE;
  738. off_start &= -PAGE_SIZE;
  739. map_len = addr_max - addr_min + off_start;
  740. /* The first time, we map too much, possibly even more than
  741. * the length of the file. This is okay because we will not
  742. * use the invalid part; we just need to reserve the right
  743. * amount of virtual address space to map over later. */
  744. map = DL_NOMMU_SUPPORT
  745. ? mmap((void *)addr_min, map_len, PROT_READ|PROT_WRITE|PROT_EXEC,
  746. MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
  747. : mmap((void *)addr_min, map_len, prot,
  748. MAP_PRIVATE, fd, off_start);
  749. if (map==MAP_FAILED) goto error;
  750. dso->map = map;
  751. dso->map_len = map_len;
  752. /* If the loaded file is not relocatable and the requested address is
  753. * not available, then the load operation must fail. */
  754. if (eh->e_type != ET_DYN && addr_min && map!=(void *)addr_min) {
  755. errno = EBUSY;
  756. goto error;
  757. }
  758. base = map - addr_min;
  759. dso->phdr = 0;
  760. dso->phnum = 0;
  761. for (ph=ph0, i=eh->e_phnum; i; i--, ph=(void *)((char *)ph+eh->e_phentsize)) {
  762. if (ph->p_type != PT_LOAD) continue;
  763. /* Check if the programs headers are in this load segment, and
  764. * if so, record the address for use by dl_iterate_phdr. */
  765. if (!dso->phdr && eh->e_phoff >= ph->p_offset
  766. && eh->e_phoff+phsize <= ph->p_offset+ph->p_filesz) {
  767. dso->phdr = (void *)(base + ph->p_vaddr
  768. + (eh->e_phoff-ph->p_offset));
  769. dso->phnum = eh->e_phnum;
  770. dso->phentsize = eh->e_phentsize;
  771. }
  772. this_min = ph->p_vaddr & -PAGE_SIZE;
  773. this_max = ph->p_vaddr+ph->p_memsz+PAGE_SIZE-1 & -PAGE_SIZE;
  774. off_start = ph->p_offset & -PAGE_SIZE;
  775. prot = (((ph->p_flags&PF_R) ? PROT_READ : 0) |
  776. ((ph->p_flags&PF_W) ? PROT_WRITE: 0) |
  777. ((ph->p_flags&PF_X) ? PROT_EXEC : 0));
  778. /* Reuse the existing mapping for the lowest-address LOAD */
  779. if ((ph->p_vaddr & -PAGE_SIZE) != addr_min || DL_NOMMU_SUPPORT)
  780. if (mmap_fixed(base+this_min, this_max-this_min, prot, MAP_PRIVATE|MAP_FIXED, fd, off_start) == MAP_FAILED)
  781. goto error;
  782. if (ph->p_memsz > ph->p_filesz && (ph->p_flags&PF_W)) {
  783. size_t brk = (size_t)base+ph->p_vaddr+ph->p_filesz;
  784. size_t pgbrk = brk+PAGE_SIZE-1 & -PAGE_SIZE;
  785. memset((void *)brk, 0, pgbrk-brk & PAGE_SIZE-1);
  786. if (pgbrk-(size_t)base < this_max && mmap_fixed((void *)pgbrk, (size_t)base+this_max-pgbrk, prot, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) == MAP_FAILED)
  787. goto error;
  788. }
  789. }
  790. for (i=0; ((size_t *)(base+dyn))[i]; i+=2)
  791. if (((size_t *)(base+dyn))[i]==DT_TEXTREL) {
  792. if (mprotect(map, map_len, PROT_READ|PROT_WRITE|PROT_EXEC)
  793. && errno != ENOSYS)
  794. goto error;
  795. break;
  796. }
  797. done_mapping:
  798. dso->base = base;
  799. dso->dynv = laddr(dso, dyn);
  800. if (dso->tls.size) dso->tls.image = laddr(dso, tls_image);
  801. free(allocated_buf);
  802. return map;
  803. noexec:
  804. errno = ENOEXEC;
  805. error:
  806. if (map!=MAP_FAILED) unmap_library(dso);
  807. free(allocated_buf);
  808. return 0;
  809. }
  810. static int path_open(const char *name, const char *s, char *buf, size_t buf_size)
  811. {
  812. size_t l;
  813. int fd;
  814. for (;;) {
  815. s += strspn(s, ":\n");
  816. l = strcspn(s, ":\n");
  817. if (l-1 >= INT_MAX) return -1;
  818. if (snprintf(buf, buf_size, "%.*s/%s", (int)l, s, name) < buf_size) {
  819. if ((fd = open(buf, O_RDONLY|O_CLOEXEC))>=0) return fd;
  820. switch (errno) {
  821. case ENOENT:
  822. case ENOTDIR:
  823. case EACCES:
  824. case ENAMETOOLONG:
  825. break;
  826. default:
  827. /* Any negative value but -1 will inhibit
  828. * futher path search. */
  829. return -2;
  830. }
  831. }
  832. s += l;
  833. }
  834. }
  835. static int fixup_rpath(struct dso *p, char *buf, size_t buf_size)
  836. {
  837. size_t n, l;
  838. const char *s, *t, *origin;
  839. char *d;
  840. if (p->rpath || !p->rpath_orig) return 0;
  841. if (!strchr(p->rpath_orig, '$')) {
  842. p->rpath = p->rpath_orig;
  843. return 0;
  844. }
  845. n = 0;
  846. s = p->rpath_orig;
  847. while ((t=strchr(s, '$'))) {
  848. if (strncmp(t, "$ORIGIN", 7) && strncmp(t, "${ORIGIN}", 9))
  849. return 0;
  850. s = t+1;
  851. n++;
  852. }
  853. if (n > SSIZE_MAX/PATH_MAX) return 0;
  854. if (p->kernel_mapped) {
  855. /* $ORIGIN searches cannot be performed for the main program
  856. * when it is suid/sgid/AT_SECURE. This is because the
  857. * pathname is under the control of the caller of execve.
  858. * For libraries, however, $ORIGIN can be processed safely
  859. * since the library's pathname came from a trusted source
  860. * (either system paths or a call to dlopen). */
  861. if (libc.secure)
  862. return 0;
  863. l = readlink("/proc/self/exe", buf, buf_size);
  864. if (l == -1) switch (errno) {
  865. case ENOENT:
  866. case ENOTDIR:
  867. case EACCES:
  868. return 0;
  869. default:
  870. return -1;
  871. }
  872. if (l >= buf_size)
  873. return 0;
  874. buf[l] = 0;
  875. origin = buf;
  876. } else {
  877. origin = p->name;
  878. }
  879. t = strrchr(origin, '/');
  880. if (t) {
  881. l = t-origin;
  882. } else {
  883. /* Normally p->name will always be an absolute or relative
  884. * pathname containing at least one '/' character, but in the
  885. * case where ldso was invoked as a command to execute a
  886. * program in the working directory, app.name may not. Fix. */
  887. origin = ".";
  888. l = 1;
  889. }
  890. /* Disallow non-absolute origins for suid/sgid/AT_SECURE. */
  891. if (libc.secure && *origin != '/')
  892. return 0;
  893. p->rpath = malloc(strlen(p->rpath_orig) + n*l + 1);
  894. if (!p->rpath) return -1;
  895. d = p->rpath;
  896. s = p->rpath_orig;
  897. while ((t=strchr(s, '$'))) {
  898. memcpy(d, s, t-s);
  899. d += t-s;
  900. memcpy(d, origin, l);
  901. d += l;
  902. /* It was determined previously that the '$' is followed
  903. * either by "ORIGIN" or "{ORIGIN}". */
  904. s = t + 7 + 2*(t[1]=='{');
  905. }
  906. strcpy(d, s);
  907. return 0;
  908. }
  909. static void decode_dyn(struct dso *p)
  910. {
  911. size_t dyn[DYN_CNT];
  912. decode_vec(p->dynv, dyn, DYN_CNT);
  913. p->syms = laddr(p, dyn[DT_SYMTAB]);
  914. p->strings = laddr(p, dyn[DT_STRTAB]);
  915. if (dyn[0]&(1<<DT_HASH))
  916. p->hashtab = laddr(p, dyn[DT_HASH]);
  917. if (dyn[0]&(1<<DT_RPATH))
  918. p->rpath_orig = p->strings + dyn[DT_RPATH];
  919. if (dyn[0]&(1<<DT_RUNPATH))
  920. p->rpath_orig = p->strings + dyn[DT_RUNPATH];
  921. if (dyn[0]&(1<<DT_PLTGOT))
  922. p->got = laddr(p, dyn[DT_PLTGOT]);
  923. if (search_vec(p->dynv, dyn, DT_GNU_HASH))
  924. p->ghashtab = laddr(p, *dyn);
  925. if (search_vec(p->dynv, dyn, DT_VERSYM))
  926. p->versym = laddr(p, *dyn);
  927. }
  928. static size_t count_syms(struct dso *p)
  929. {
  930. if (p->hashtab) return p->hashtab[1];
  931. size_t nsym, i;
  932. uint32_t *buckets = p->ghashtab + 4 + (p->ghashtab[2]*sizeof(size_t)/4);
  933. uint32_t *hashval;
  934. for (i = nsym = 0; i < p->ghashtab[0]; i++) {
  935. if (buckets[i] > nsym)
  936. nsym = buckets[i];
  937. }
  938. if (nsym) {
  939. hashval = buckets + p->ghashtab[0] + (nsym - p->ghashtab[1]);
  940. do nsym++;
  941. while (!(*hashval++ & 1));
  942. }
  943. return nsym;
  944. }
  945. static void *dl_mmap(size_t n)
  946. {
  947. void *p;
  948. int prot = PROT_READ|PROT_WRITE, flags = MAP_ANONYMOUS|MAP_PRIVATE;
  949. #ifdef SYS_mmap2
  950. p = (void *)__syscall(SYS_mmap2, 0, n, prot, flags, -1, 0);
  951. #else
  952. p = (void *)__syscall(SYS_mmap, 0, n, prot, flags, -1, 0);
  953. #endif
  954. return (unsigned long)p > -4096UL ? 0 : p;
  955. }
  956. static void makefuncdescs(struct dso *p)
  957. {
  958. static int self_done;
  959. size_t nsym = count_syms(p);
  960. size_t i, size = nsym * sizeof(*p->funcdescs);
  961. if (!self_done) {
  962. p->funcdescs = dl_mmap(size);
  963. self_done = 1;
  964. } else {
  965. p->funcdescs = malloc(size);
  966. }
  967. if (!p->funcdescs) {
  968. if (!runtime) a_crash();
  969. error("Error allocating function descriptors for %s", p->name);
  970. longjmp(*rtld_fail, 1);
  971. }
  972. for (i=0; i<nsym; i++) {
  973. if ((p->syms[i].st_info&0xf)==STT_FUNC && p->syms[i].st_shndx) {
  974. p->funcdescs[i].addr = laddr(p, p->syms[i].st_value);
  975. p->funcdescs[i].got = p->got;
  976. } else {
  977. p->funcdescs[i].addr = 0;
  978. p->funcdescs[i].got = 0;
  979. }
  980. }
  981. }
  982. static struct dso *load_library(const char *name, struct dso *needed_by)
  983. {
  984. char buf[2*NAME_MAX+2];
  985. const char *pathname;
  986. unsigned char *map;
  987. struct dso *p, temp_dso = {0};
  988. int fd;
  989. struct stat st;
  990. size_t alloc_size;
  991. int n_th = 0;
  992. int is_self = 0;
  993. if (!*name) {
  994. errno = EINVAL;
  995. return 0;
  996. }
  997. /* Catch and block attempts to reload the implementation itself */
  998. if (name[0]=='l' && name[1]=='i' && name[2]=='b') {
  999. static const char reserved[] =
  1000. "c.pthread.rt.m.dl.util.xnet.";
  1001. const char *rp, *next;
  1002. for (rp=reserved; *rp; rp=next) {
  1003. next = strchr(rp, '.') + 1;
  1004. if (strncmp(name+3, rp, next-rp) == 0)
  1005. break;
  1006. }
  1007. if (*rp) {
  1008. if (ldd_mode) {
  1009. /* Track which names have been resolved
  1010. * and only report each one once. */
  1011. static unsigned reported;
  1012. unsigned mask = 1U<<(rp-reserved);
  1013. if (!(reported & mask)) {
  1014. reported |= mask;
  1015. dprintf(1, "\t%s => %s (%p)\n",
  1016. name, ldso.name,
  1017. ldso.base);
  1018. }
  1019. }
  1020. is_self = 1;
  1021. }
  1022. }
  1023. if (!strcmp(name, ldso.name)) is_self = 1;
  1024. if (is_self) {
  1025. if (!ldso.prev) {
  1026. tail->next = &ldso;
  1027. ldso.prev = tail;
  1028. tail = &ldso;
  1029. }
  1030. return &ldso;
  1031. }
  1032. if (strchr(name, '/')) {
  1033. pathname = name;
  1034. fd = open(name, O_RDONLY|O_CLOEXEC);
  1035. } else {
  1036. /* Search for the name to see if it's already loaded */
  1037. for (p=head->next; p; p=p->next) {
  1038. if (p->shortname && !strcmp(p->shortname, name)) {
  1039. return p;
  1040. }
  1041. }
  1042. if (strlen(name) > NAME_MAX) return 0;
  1043. fd = -1;
  1044. if (env_path) fd = path_open(name, env_path, buf, sizeof buf);
  1045. for (p=needed_by; fd == -1 && p; p=p->needed_by) {
  1046. if (fixup_rpath(p, buf, sizeof buf) < 0)
  1047. fd = -2; /* Inhibit further search. */
  1048. if (p->rpath)
  1049. fd = path_open(name, p->rpath, buf, sizeof buf);
  1050. }
  1051. if (fd == -1) {
  1052. if (!sys_path) {
  1053. char *prefix = 0;
  1054. size_t prefix_len;
  1055. if (ldso.name[0]=='/') {
  1056. char *s, *t, *z;
  1057. for (s=t=z=ldso.name; *s; s++)
  1058. if (*s=='/') z=t, t=s;
  1059. prefix_len = z-ldso.name;
  1060. if (prefix_len < PATH_MAX)
  1061. prefix = ldso.name;
  1062. }
  1063. if (!prefix) {
  1064. prefix = "";
  1065. prefix_len = 0;
  1066. }
  1067. char etc_ldso_path[prefix_len + 1
  1068. + sizeof "/etc/ld-musl-" LDSO_ARCH ".path"];
  1069. snprintf(etc_ldso_path, sizeof etc_ldso_path,
  1070. "%.*s/etc/ld-musl-" LDSO_ARCH ".path",
  1071. (int)prefix_len, prefix);
  1072. fd = open(etc_ldso_path, O_RDONLY|O_CLOEXEC);
  1073. if (fd>=0) {
  1074. size_t n = 0;
  1075. if (!fstat(fd, &st)) n = st.st_size;
  1076. if ((sys_path = malloc(n+1)))
  1077. sys_path[n] = 0;
  1078. if (!sys_path || read_loop(fd, sys_path, n)<0) {
  1079. free(sys_path);
  1080. sys_path = "";
  1081. }
  1082. close(fd);
  1083. } else if (errno != ENOENT) {
  1084. sys_path = "";
  1085. }
  1086. }
  1087. if (!sys_path) sys_path = "/lib:/usr/local/lib:/usr/lib";
  1088. fd = path_open(name, sys_path, buf, sizeof buf);
  1089. }
  1090. pathname = buf;
  1091. }
  1092. if (fd < 0) return 0;
  1093. if (fstat(fd, &st) < 0) {
  1094. close(fd);
  1095. return 0;
  1096. }
  1097. for (p=head->next; p; p=p->next) {
  1098. if (p->dev == st.st_dev && p->ino == st.st_ino) {
  1099. /* If this library was previously loaded with a
  1100. * pathname but a search found the same inode,
  1101. * setup its shortname so it can be found by name. */
  1102. if (!p->shortname && pathname != name)
  1103. p->shortname = strrchr(p->name, '/')+1;
  1104. close(fd);
  1105. return p;
  1106. }
  1107. }
  1108. map = noload ? 0 : map_library(fd, &temp_dso);
  1109. close(fd);
  1110. if (!map) return 0;
  1111. /* Avoid the danger of getting two versions of libc mapped into the
  1112. * same process when an absolute pathname was used. The symbols
  1113. * checked are chosen to catch both musl and glibc, and to avoid
  1114. * false positives from interposition-hack libraries. */
  1115. decode_dyn(&temp_dso);
  1116. if (find_sym(&temp_dso, "__libc_start_main", 1).sym &&
  1117. find_sym(&temp_dso, "stdin", 1).sym) {
  1118. unmap_library(&temp_dso);
  1119. return load_library("libc.so", needed_by);
  1120. }
  1121. /* Past this point, if we haven't reached runtime yet, ldso has
  1122. * committed either to use the mapped library or to abort execution.
  1123. * Unmapping is not possible, so we can safely reclaim gaps. */
  1124. if (!runtime) reclaim_gaps(&temp_dso);
  1125. /* Allocate storage for the new DSO. When there is TLS, this
  1126. * storage must include a reservation for all pre-existing
  1127. * threads to obtain copies of both the new TLS, and an
  1128. * extended DTV capable of storing an additional slot for
  1129. * the newly-loaded DSO. */
  1130. alloc_size = sizeof *p + strlen(pathname) + 1;
  1131. if (runtime && temp_dso.tls.image) {
  1132. size_t per_th = temp_dso.tls.size + temp_dso.tls.align
  1133. + sizeof(void *) * (tls_cnt+3);
  1134. n_th = libc.threads_minus_1 + 1;
  1135. if (n_th > SSIZE_MAX / per_th) alloc_size = SIZE_MAX;
  1136. else alloc_size += n_th * per_th;
  1137. }
  1138. p = calloc(1, alloc_size);
  1139. if (!p) {
  1140. unmap_library(&temp_dso);
  1141. return 0;
  1142. }
  1143. memcpy(p, &temp_dso, sizeof temp_dso);
  1144. p->dev = st.st_dev;
  1145. p->ino = st.st_ino;
  1146. p->needed_by = needed_by;
  1147. p->name = p->buf;
  1148. p->runtime_loaded = runtime;
  1149. strcpy(p->name, pathname);
  1150. /* Add a shortname only if name arg was not an explicit pathname. */
  1151. if (pathname != name) p->shortname = strrchr(p->name, '/')+1;
  1152. if (p->tls.image) {
  1153. p->tls_id = ++tls_cnt;
  1154. tls_align = MAXP2(tls_align, p->tls.align);
  1155. #ifdef TLS_ABOVE_TP
  1156. p->tls.offset = tls_offset + ( (p->tls.align-1) &
  1157. (-tls_offset + (uintptr_t)p->tls.image) );
  1158. tls_offset = p->tls.offset + p->tls.size;
  1159. #else
  1160. tls_offset += p->tls.size + p->tls.align - 1;
  1161. tls_offset -= (tls_offset + (uintptr_t)p->tls.image)
  1162. & (p->tls.align-1);
  1163. p->tls.offset = tls_offset;
  1164. #endif
  1165. p->new_dtv = (void *)(-sizeof(size_t) &
  1166. (uintptr_t)(p->name+strlen(p->name)+sizeof(size_t)));
  1167. p->new_tls = (void *)(p->new_dtv + n_th*(tls_cnt+1));
  1168. if (tls_tail) tls_tail->next = &p->tls;
  1169. else libc.tls_head = &p->tls;
  1170. tls_tail = &p->tls;
  1171. }
  1172. tail->next = p;
  1173. p->prev = tail;
  1174. tail = p;
  1175. if (DL_FDPIC) makefuncdescs(p);
  1176. if (ldd_mode) dprintf(1, "\t%s => %s (%p)\n", name, pathname, p->base);
  1177. return p;
  1178. }
  1179. static void load_direct_deps(struct dso *p)
  1180. {
  1181. size_t i, cnt=0;
  1182. if (p->deps) return;
  1183. /* For head, all preloads are direct pseudo-dependencies.
  1184. * Count and include them now to avoid realloc later. */
  1185. if (p==head) for (struct dso *q=p->next; q; q=q->next)
  1186. cnt++;
  1187. for (i=0; p->dynv[i]; i+=2)
  1188. if (p->dynv[i] == DT_NEEDED) cnt++;
  1189. /* Use builtin buffer for apps with no external deps, to
  1190. * preserve property of no runtime failure paths. */
  1191. p->deps = (p==head && cnt<2) ? builtin_deps :
  1192. calloc(cnt+1, sizeof *p->deps);
  1193. if (!p->deps) {
  1194. error("Error loading dependencies for %s", p->name);
  1195. if (runtime) longjmp(*rtld_fail, 1);
  1196. }
  1197. cnt=0;
  1198. if (p==head) for (struct dso *q=p->next; q; q=q->next)
  1199. p->deps[cnt++] = q;
  1200. for (i=0; p->dynv[i]; i+=2) {
  1201. if (p->dynv[i] != DT_NEEDED) continue;
  1202. struct dso *dep = load_library(p->strings + p->dynv[i+1], p);
  1203. if (!dep) {
  1204. error("Error loading shared library %s: %m (needed by %s)",
  1205. p->strings + p->dynv[i+1], p->name);
  1206. if (runtime) longjmp(*rtld_fail, 1);
  1207. continue;
  1208. }
  1209. p->deps[cnt++] = dep;
  1210. }
  1211. p->deps[cnt] = 0;
  1212. p->ndeps_direct = cnt;
  1213. }
  1214. static void load_deps(struct dso *p)
  1215. {
  1216. if (p->deps) return;
  1217. for (; p; p=p->next)
  1218. load_direct_deps(p);
  1219. }
  1220. static void extend_bfs_deps(struct dso *p)
  1221. {
  1222. size_t i, j, cnt, ndeps_all;
  1223. struct dso **tmp;
  1224. /* Can't use realloc if the original p->deps was allocated at
  1225. * program entry and malloc has been replaced, or if it's
  1226. * the builtin non-allocated trivial main program deps array. */
  1227. int no_realloc = (__malloc_replaced && !p->runtime_loaded)
  1228. || p->deps == builtin_deps;
  1229. if (p->bfs_built) return;
  1230. ndeps_all = p->ndeps_direct;
  1231. /* Mark existing (direct) deps so they won't be duplicated. */
  1232. for (i=0; p->deps[i]; i++)
  1233. p->deps[i]->mark = 1;
  1234. /* For each dependency already in the list, copy its list of direct
  1235. * dependencies to the list, excluding any items already in the
  1236. * list. Note that the list this loop iterates over will grow during
  1237. * the loop, but since duplicates are excluded, growth is bounded. */
  1238. for (i=0; p->deps[i]; i++) {
  1239. struct dso *dep = p->deps[i];
  1240. for (j=cnt=0; j<dep->ndeps_direct; j++)
  1241. if (!dep->deps[j]->mark) cnt++;
  1242. tmp = no_realloc ?
  1243. malloc(sizeof(*tmp) * (ndeps_all+cnt+1)) :
  1244. realloc(p->deps, sizeof(*tmp) * (ndeps_all+cnt+1));
  1245. if (!tmp) {
  1246. error("Error recording dependencies for %s", p->name);
  1247. if (runtime) longjmp(*rtld_fail, 1);
  1248. continue;
  1249. }
  1250. if (no_realloc) {
  1251. memcpy(tmp, p->deps, sizeof(*tmp) * (ndeps_all+1));
  1252. no_realloc = 0;
  1253. }
  1254. p->deps = tmp;
  1255. for (j=0; j<dep->ndeps_direct; j++) {
  1256. if (dep->deps[j]->mark) continue;
  1257. dep->deps[j]->mark = 1;
  1258. p->deps[ndeps_all++] = dep->deps[j];
  1259. }
  1260. p->deps[ndeps_all] = 0;
  1261. }
  1262. p->bfs_built = 1;
  1263. for (p=head; p; p=p->next)
  1264. p->mark = 0;
  1265. }
  1266. static void load_preload(char *s)
  1267. {
  1268. int tmp;
  1269. char *z;
  1270. for (z=s; *z; s=z) {
  1271. for ( ; *s && (isspace(*s) || *s==':'); s++);
  1272. for (z=s; *z && !isspace(*z) && *z!=':'; z++);
  1273. tmp = *z;
  1274. *z = 0;
  1275. load_library(s, 0);
  1276. *z = tmp;
  1277. }
  1278. }
  1279. static void add_syms(struct dso *p)
  1280. {
  1281. if (!p->syms_next && syms_tail != p) {
  1282. syms_tail->syms_next = p;
  1283. syms_tail = p;
  1284. }
  1285. }
  1286. static void revert_syms(struct dso *old_tail)
  1287. {
  1288. struct dso *p, *next;
  1289. /* Chop off the tail of the list of dsos that participate in
  1290. * the global symbol table, reverting them to RTLD_LOCAL. */
  1291. for (p=old_tail; p; p=next) {
  1292. next = p->syms_next;
  1293. p->syms_next = 0;
  1294. }
  1295. syms_tail = old_tail;
  1296. }
  1297. static void do_mips_relocs(struct dso *p, size_t *got)
  1298. {
  1299. size_t i, j, rel[2];
  1300. unsigned char *base = p->base;
  1301. i=0; search_vec(p->dynv, &i, DT_MIPS_LOCAL_GOTNO);
  1302. if (p==&ldso) {
  1303. got += i;
  1304. } else {
  1305. while (i--) *got++ += (size_t)base;
  1306. }
  1307. j=0; search_vec(p->dynv, &j, DT_MIPS_GOTSYM);
  1308. i=0; search_vec(p->dynv, &i, DT_MIPS_SYMTABNO);
  1309. Sym *sym = p->syms + j;
  1310. rel[0] = (unsigned char *)got - base;
  1311. for (i-=j; i; i--, sym++, rel[0]+=sizeof(size_t)) {
  1312. rel[1] = R_INFO(sym-p->syms, R_MIPS_JUMP_SLOT);
  1313. do_relocs(p, rel, sizeof rel, 2);
  1314. }
  1315. }
  1316. static void reloc_all(struct dso *p)
  1317. {
  1318. size_t dyn[DYN_CNT];
  1319. for (; p; p=p->next) {
  1320. if (p->relocated) continue;
  1321. decode_vec(p->dynv, dyn, DYN_CNT);
  1322. if (NEED_MIPS_GOT_RELOCS)
  1323. do_mips_relocs(p, laddr(p, dyn[DT_PLTGOT]));
  1324. do_relocs(p, laddr(p, dyn[DT_JMPREL]), dyn[DT_PLTRELSZ],
  1325. 2+(dyn[DT_PLTREL]==DT_RELA));
  1326. do_relocs(p, laddr(p, dyn[DT_REL]), dyn[DT_RELSZ], 2);
  1327. do_relocs(p, laddr(p, dyn[DT_RELA]), dyn[DT_RELASZ], 3);
  1328. if (!DL_FDPIC)
  1329. do_relr_relocs(p, laddr(p, dyn[DT_RELR]), dyn[DT_RELRSZ]);
  1330. if (head != &ldso && p->relro_start != p->relro_end) {
  1331. long ret = __syscall(SYS_mprotect, laddr(p, p->relro_start),
  1332. p->relro_end-p->relro_start, PROT_READ);
  1333. if (ret != 0 && ret != -ENOSYS) {
  1334. error("Error relocating %s: RELRO protection failed: %m",
  1335. p->name);
  1336. if (runtime) longjmp(*rtld_fail, 1);
  1337. }
  1338. }
  1339. p->relocated = 1;
  1340. }
  1341. }
  1342. static void kernel_mapped_dso(struct dso *p)
  1343. {
  1344. size_t min_addr = -1, max_addr = 0, cnt;
  1345. Phdr *ph = p->phdr;
  1346. for (cnt = p->phnum; cnt--; ph = (void *)((char *)ph + p->phentsize)) {
  1347. if (ph->p_type == PT_DYNAMIC) {
  1348. p->dynv = laddr(p, ph->p_vaddr);
  1349. } else if (ph->p_type == PT_GNU_RELRO) {
  1350. p->relro_start = ph->p_vaddr & -PAGE_SIZE;
  1351. p->relro_end = (ph->p_vaddr + ph->p_memsz) & -PAGE_SIZE;
  1352. } else if (ph->p_type == PT_GNU_STACK) {
  1353. if (!runtime && ph->p_memsz > __default_stacksize) {
  1354. __default_stacksize =
  1355. ph->p_memsz < DEFAULT_STACK_MAX ?
  1356. ph->p_memsz : DEFAULT_STACK_MAX;
  1357. }
  1358. }
  1359. if (ph->p_type != PT_LOAD) continue;
  1360. if (ph->p_vaddr < min_addr)
  1361. min_addr = ph->p_vaddr;
  1362. if (ph->p_vaddr+ph->p_memsz > max_addr)
  1363. max_addr = ph->p_vaddr+ph->p_memsz;
  1364. }
  1365. min_addr &= -PAGE_SIZE;
  1366. max_addr = (max_addr + PAGE_SIZE-1) & -PAGE_SIZE;
  1367. p->map = p->base + min_addr;
  1368. p->map_len = max_addr - min_addr;
  1369. p->kernel_mapped = 1;
  1370. }
  1371. void __libc_exit_fini()
  1372. {
  1373. struct dso *p;
  1374. size_t dyn[DYN_CNT];
  1375. pthread_t self = __pthread_self();
  1376. /* Take both locks before setting shutting_down, so that
  1377. * either lock is sufficient to read its value. The lock
  1378. * order matches that in dlopen to avoid deadlock. */
  1379. pthread_rwlock_wrlock(&lock);
  1380. pthread_mutex_lock(&init_fini_lock);
  1381. shutting_down = 1;
  1382. pthread_rwlock_unlock(&lock);
  1383. for (p=fini_head; p; p=p->fini_next) {
  1384. while (p->ctor_visitor && p->ctor_visitor!=self)
  1385. pthread_cond_wait(&ctor_cond, &init_fini_lock);
  1386. if (!p->constructed) continue;
  1387. decode_vec(p->dynv, dyn, DYN_CNT);
  1388. if (dyn[0] & (1<<DT_FINI_ARRAY)) {
  1389. size_t n = dyn[DT_FINI_ARRAYSZ]/sizeof(size_t);
  1390. size_t *fn = (size_t *)laddr(p, dyn[DT_FINI_ARRAY])+n;
  1391. while (n--) ((void (*)(void))*--fn)();
  1392. }
  1393. #ifndef NO_LEGACY_INITFINI
  1394. if ((dyn[0] & (1<<DT_FINI)) && dyn[DT_FINI])
  1395. fpaddr(p, dyn[DT_FINI])();
  1396. #endif
  1397. }
  1398. }
  1399. void __ldso_atfork(int who)
  1400. {
  1401. if (who<0) {
  1402. pthread_rwlock_wrlock(&lock);
  1403. pthread_mutex_lock(&init_fini_lock);
  1404. } else {
  1405. pthread_mutex_unlock(&init_fini_lock);
  1406. pthread_rwlock_unlock(&lock);
  1407. }
  1408. }
  1409. static struct dso **queue_ctors(struct dso *dso)
  1410. {
  1411. size_t cnt, qpos, spos, i;
  1412. struct dso *p, **queue, **stack;
  1413. if (ldd_mode) return 0;
  1414. /* Bound on queue size is the total number of indirect deps.
  1415. * If a bfs deps list was built, we can use it. Otherwise,
  1416. * bound by the total number of DSOs, which is always safe and
  1417. * is reasonable we use it (for main app at startup). */
  1418. if (dso->bfs_built) {
  1419. for (cnt=0; dso->deps[cnt]; cnt++)
  1420. dso->deps[cnt]->mark = 0;
  1421. cnt++; /* self, not included in deps */
  1422. } else {
  1423. for (cnt=0, p=head; p; cnt++, p=p->next)
  1424. p->mark = 0;
  1425. }
  1426. cnt++; /* termination slot */
  1427. if (dso==head && cnt <= countof(builtin_ctor_queue))
  1428. queue = builtin_ctor_queue;
  1429. else
  1430. queue = calloc(cnt, sizeof *queue);
  1431. if (!queue) {
  1432. error("Error allocating constructor queue: %m\n");
  1433. if (runtime) longjmp(*rtld_fail, 1);
  1434. return 0;
  1435. }
  1436. /* Opposite ends of the allocated buffer serve as an output queue
  1437. * and a working stack. Setup initial stack with just the argument
  1438. * dso and initial queue empty... */
  1439. stack = queue;
  1440. qpos = 0;
  1441. spos = cnt;
  1442. stack[--spos] = dso;
  1443. dso->next_dep = 0;
  1444. dso->mark = 1;
  1445. /* Then perform pseudo-DFS sort, but ignoring circular deps. */
  1446. while (spos<cnt) {
  1447. p = stack[spos++];
  1448. while (p->next_dep < p->ndeps_direct) {
  1449. if (p->deps[p->next_dep]->mark) {
  1450. p->next_dep++;
  1451. } else {
  1452. stack[--spos] = p;
  1453. p = p->deps[p->next_dep];
  1454. p->next_dep = 0;
  1455. p->mark = 1;
  1456. }
  1457. }
  1458. queue[qpos++] = p;
  1459. }
  1460. queue[qpos] = 0;
  1461. for (i=0; i<qpos; i++) queue[i]->mark = 0;
  1462. for (i=0; i<qpos; i++)
  1463. if (queue[i]->ctor_visitor && queue[i]->ctor_visitor->tid < 0) {
  1464. error("State of %s is inconsistent due to multithreaded fork\n",
  1465. queue[i]->name);
  1466. free(queue);
  1467. if (runtime) longjmp(*rtld_fail, 1);
  1468. }
  1469. return queue;
  1470. }
  1471. static void do_init_fini(struct dso **queue)
  1472. {
  1473. struct dso *p;
  1474. size_t dyn[DYN_CNT], i;
  1475. pthread_t self = __pthread_self();
  1476. pthread_mutex_lock(&init_fini_lock);
  1477. for (i=0; (p=queue[i]); i++) {
  1478. while ((p->ctor_visitor && p->ctor_visitor!=self) || shutting_down)
  1479. pthread_cond_wait(&ctor_cond, &init_fini_lock);
  1480. if (p->ctor_visitor || p->constructed)
  1481. continue;
  1482. p->ctor_visitor = self;
  1483. decode_vec(p->dynv, dyn, DYN_CNT);
  1484. if (dyn[0] & ((1<<DT_FINI) | (1<<DT_FINI_ARRAY))) {
  1485. p->fini_next = fini_head;
  1486. fini_head = p;
  1487. }
  1488. pthread_mutex_unlock(&init_fini_lock);
  1489. #ifndef NO_LEGACY_INITFINI
  1490. if ((dyn[0] & (1<<DT_INIT)) && dyn[DT_INIT])
  1491. fpaddr(p, dyn[DT_INIT])();
  1492. #endif
  1493. if (dyn[0] & (1<<DT_INIT_ARRAY)) {
  1494. size_t n = dyn[DT_INIT_ARRAYSZ]/sizeof(size_t);
  1495. size_t *fn = laddr(p, dyn[DT_INIT_ARRAY]);
  1496. while (n--) ((void (*)(void))*fn++)();
  1497. }
  1498. pthread_mutex_lock(&init_fini_lock);
  1499. p->ctor_visitor = 0;
  1500. p->constructed = 1;
  1501. pthread_cond_broadcast(&ctor_cond);
  1502. }
  1503. pthread_mutex_unlock(&init_fini_lock);
  1504. }
  1505. void __libc_start_init(void)
  1506. {
  1507. do_init_fini(main_ctor_queue);
  1508. if (!__malloc_replaced && main_ctor_queue != builtin_ctor_queue)
  1509. free(main_ctor_queue);
  1510. main_ctor_queue = 0;
  1511. }
  1512. static void dl_debug_state(void)
  1513. {
  1514. }
  1515. weak_alias(dl_debug_state, _dl_debug_state);
  1516. void __init_tls(size_t *auxv)
  1517. {
  1518. }
  1519. static void update_tls_size()
  1520. {
  1521. libc.tls_cnt = tls_cnt;
  1522. libc.tls_align = tls_align;
  1523. libc.tls_size = ALIGN(
  1524. (1+tls_cnt) * sizeof(void *) +
  1525. tls_offset +
  1526. sizeof(struct pthread) +
  1527. tls_align * 2,
  1528. tls_align);
  1529. }
  1530. static void install_new_tls(void)
  1531. {
  1532. sigset_t set;
  1533. pthread_t self = __pthread_self(), td;
  1534. struct dso *dtv_provider = container_of(tls_tail, struct dso, tls);
  1535. uintptr_t (*newdtv)[tls_cnt+1] = (void *)dtv_provider->new_dtv;
  1536. struct dso *p;
  1537. size_t i, j;
  1538. size_t old_cnt = self->dtv[0];
  1539. __block_app_sigs(&set);
  1540. __tl_lock();
  1541. /* Copy existing dtv contents from all existing threads. */
  1542. for (i=0, td=self; !i || td!=self; i++, td=td->next) {
  1543. memcpy(newdtv+i, td->dtv,
  1544. (old_cnt+1)*sizeof(uintptr_t));
  1545. newdtv[i][0] = tls_cnt;
  1546. }
  1547. /* Install new dtls into the enlarged, uninstalled dtv copies. */
  1548. for (p=head; ; p=p->next) {
  1549. if (p->tls_id <= old_cnt) continue;
  1550. unsigned char *mem = p->new_tls;
  1551. for (j=0; j<i; j++) {
  1552. unsigned char *new = mem;
  1553. new += ((uintptr_t)p->tls.image - (uintptr_t)mem)
  1554. & (p->tls.align-1);
  1555. memcpy(new, p->tls.image, p->tls.len);
  1556. newdtv[j][p->tls_id] =
  1557. (uintptr_t)new + DTP_OFFSET;
  1558. mem += p->tls.size + p->tls.align;
  1559. }
  1560. if (p->tls_id == tls_cnt) break;
  1561. }
  1562. /* Broadcast barrier to ensure contents of new dtv is visible
  1563. * if the new dtv pointer is. The __membarrier function has a
  1564. * fallback emulation using signals for kernels that lack the
  1565. * feature at the syscall level. */
  1566. __membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0);
  1567. /* Install new dtv for each thread. */
  1568. for (j=0, td=self; !j || td!=self; j++, td=td->next) {
  1569. td->dtv = newdtv[j];
  1570. }
  1571. __tl_unlock();
  1572. __restore_sigs(&set);
  1573. }
  1574. /* Stage 1 of the dynamic linker is defined in dlstart.c. It calls the
  1575. * following stage 2 and stage 3 functions via primitive symbolic lookup
  1576. * since it does not have access to their addresses to begin with. */
  1577. /* Stage 2 of the dynamic linker is called after relative relocations
  1578. * have been processed. It can make function calls to static functions
  1579. * and access string literals and static data, but cannot use extern
  1580. * symbols. Its job is to perform symbolic relocations on the dynamic
  1581. * linker itself, but some of the relocations performed may need to be
  1582. * replaced later due to copy relocations in the main program. */
  1583. hidden void __dls2(unsigned char *base, size_t *sp)
  1584. {
  1585. size_t *auxv;
  1586. for (auxv=sp+1+*sp+1; *auxv; auxv++);
  1587. auxv++;
  1588. if (DL_FDPIC) {
  1589. void *p1 = (void *)sp[-2];
  1590. void *p2 = (void *)sp[-1];
  1591. if (!p1) {
  1592. size_t aux[AUX_CNT];
  1593. decode_vec(auxv, aux, AUX_CNT);
  1594. if (aux[AT_BASE]) ldso.base = (void *)aux[AT_BASE];
  1595. else ldso.base = (void *)(aux[AT_PHDR] & -4096);
  1596. }
  1597. app_loadmap = p2 ? p1 : 0;
  1598. ldso.loadmap = p2 ? p2 : p1;
  1599. ldso.base = laddr(&ldso, 0);
  1600. } else {
  1601. ldso.base = base;
  1602. }
  1603. Ehdr *ehdr = __ehdr_start ? (void *)__ehdr_start : (void *)ldso.base;
  1604. ldso.name = ldso.shortname = "libc.so";
  1605. ldso.phnum = ehdr->e_phnum;
  1606. ldso.phdr = laddr(&ldso, ehdr->e_phoff);
  1607. ldso.phentsize = ehdr->e_phentsize;
  1608. search_vec(auxv, &ldso_page_size, AT_PAGESZ);
  1609. kernel_mapped_dso(&ldso);
  1610. decode_dyn(&ldso);
  1611. if (DL_FDPIC) makefuncdescs(&ldso);
  1612. /* Prepare storage for to save clobbered REL addends so they
  1613. * can be reused in stage 3. There should be very few. If
  1614. * something goes wrong and there are a huge number, abort
  1615. * instead of risking stack overflow. */
  1616. size_t dyn[DYN_CNT];
  1617. decode_vec(ldso.dynv, dyn, DYN_CNT);
  1618. size_t *rel = laddr(&ldso, dyn[DT_REL]);
  1619. size_t rel_size = dyn[DT_RELSZ];
  1620. size_t symbolic_rel_cnt = 0;
  1621. apply_addends_to = rel;
  1622. for (; rel_size; rel+=2, rel_size-=2*sizeof(size_t))
  1623. if (!IS_RELATIVE(rel[1], ldso.syms)) symbolic_rel_cnt++;
  1624. if (symbolic_rel_cnt >= ADDEND_LIMIT) a_crash();
  1625. size_t addends[symbolic_rel_cnt+1];
  1626. saved_addends = addends;
  1627. head = &ldso;
  1628. reloc_all(&ldso);
  1629. ldso.relocated = 0;
  1630. /* Call dynamic linker stage-2b, __dls2b, looking it up
  1631. * symbolically as a barrier against moving the address
  1632. * load across the above relocation processing. */
  1633. struct symdef dls2b_def = find_sym(&ldso, "__dls2b", 0);
  1634. if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp, auxv);
  1635. else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp, auxv);
  1636. }
  1637. /* Stage 2b sets up a valid thread pointer, which requires relocations
  1638. * completed in stage 2, and on which stage 3 is permitted to depend.
  1639. * This is done as a separate stage, with symbolic lookup as a barrier,
  1640. * so that loads of the thread pointer and &errno can be pure/const and
  1641. * thereby hoistable. */
  1642. void __dls2b(size_t *sp, size_t *auxv)
  1643. {
  1644. /* Setup early thread pointer in builtin_tls for ldso/libc itself to
  1645. * use during dynamic linking. If possible it will also serve as the
  1646. * thread pointer at runtime. */
  1647. search_vec(auxv, &__hwcap, AT_HWCAP);
  1648. libc.auxv = auxv;
  1649. libc.tls_size = sizeof builtin_tls;
  1650. libc.tls_align = tls_align;
  1651. if (__init_tp(__copy_tls((void *)builtin_tls)) < 0) {
  1652. a_crash();
  1653. }
  1654. struct symdef dls3_def = find_sym(&ldso, "__dls3", 0);
  1655. if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp, auxv);
  1656. else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp, auxv);
  1657. }
  1658. /* Stage 3 of the dynamic linker is called with the dynamic linker/libc
  1659. * fully functional. Its job is to load (if not already loaded) and
  1660. * process dependencies and relocations for the main application and
  1661. * transfer control to its entry point. */
  1662. void __dls3(size_t *sp, size_t *auxv)
  1663. {
  1664. static struct dso app, vdso;
  1665. size_t aux[AUX_CNT];
  1666. size_t i;
  1667. char *env_preload=0;
  1668. char *replace_argv0=0;
  1669. size_t vdso_base;
  1670. int argc = *sp;
  1671. char **argv = (void *)(sp+1);
  1672. char **argv_orig = argv;
  1673. char **envp = argv+argc+1;
  1674. /* Find aux vector just past environ[] and use it to initialize
  1675. * global data that may be needed before we can make syscalls. */
  1676. __environ = envp;
  1677. decode_vec(auxv, aux, AUX_CNT);
  1678. search_vec(auxv, &__sysinfo, AT_SYSINFO);
  1679. __pthread_self()->sysinfo = __sysinfo;
  1680. libc.page_size = aux[AT_PAGESZ];
  1681. libc.secure = ((aux[0]&0x7800)!=0x7800 || aux[AT_UID]!=aux[AT_EUID]
  1682. || aux[AT_GID]!=aux[AT_EGID] || aux[AT_SECURE]);
  1683. /* Only trust user/env if kernel says we're not suid/sgid */
  1684. if (!libc.secure) {
  1685. env_path = getenv("LD_LIBRARY_PATH");
  1686. env_preload = getenv("LD_PRELOAD");
  1687. }
  1688. /* Activate error handler function */
  1689. error = error_impl;
  1690. /* If the main program was already loaded by the kernel,
  1691. * AT_PHDR will point to some location other than the dynamic
  1692. * linker's program headers. */
  1693. if (aux[AT_PHDR] != (size_t)ldso.phdr) {
  1694. size_t interp_off = 0;
  1695. size_t tls_image = 0;
  1696. /* Find load address of the main program, via AT_PHDR vs PT_PHDR. */
  1697. Phdr *phdr = app.phdr = (void *)aux[AT_PHDR];
  1698. app.phnum = aux[AT_PHNUM];
  1699. app.phentsize = aux[AT_PHENT];
  1700. for (i=aux[AT_PHNUM]; i; i--, phdr=(void *)((char *)phdr + aux[AT_PHENT])) {
  1701. if (phdr->p_type == PT_PHDR)
  1702. app.base = (void *)(aux[AT_PHDR] - phdr->p_vaddr);
  1703. else if (phdr->p_type == PT_INTERP)
  1704. interp_off = (size_t)phdr->p_vaddr;
  1705. else if (phdr->p_type == PT_TLS) {
  1706. tls_image = phdr->p_vaddr;
  1707. app.tls.len = phdr->p_filesz;
  1708. app.tls.size = phdr->p_memsz;
  1709. app.tls.align = phdr->p_align;
  1710. }
  1711. }
  1712. if (DL_FDPIC) app.loadmap = app_loadmap;
  1713. if (app.tls.size) app.tls.image = laddr(&app, tls_image);
  1714. if (interp_off) ldso.name = laddr(&app, interp_off);
  1715. if ((aux[0] & (1UL<<AT_EXECFN))
  1716. && strncmp((char *)aux[AT_EXECFN], "/proc/", 6))
  1717. app.name = (char *)aux[AT_EXECFN];
  1718. else
  1719. app.name = argv[0];
  1720. kernel_mapped_dso(&app);
  1721. } else {
  1722. int fd;
  1723. char *ldname = argv[0];
  1724. size_t l = strlen(ldname);
  1725. if (l >= 3 && !strcmp(ldname+l-3, "ldd")) ldd_mode = 1;
  1726. argv++;
  1727. while (argv[0] && argv[0][0]=='-' && argv[0][1]=='-') {
  1728. char *opt = argv[0]+2;
  1729. *argv++ = (void *)-1;
  1730. if (!*opt) {
  1731. break;
  1732. } else if (!memcmp(opt, "list", 5)) {
  1733. ldd_mode = 1;
  1734. } else if (!memcmp(opt, "library-path", 12)) {
  1735. if (opt[12]=='=') env_path = opt+13;
  1736. else if (opt[12]) *argv = 0;
  1737. else if (*argv) env_path = *argv++;
  1738. } else if (!memcmp(opt, "preload", 7)) {
  1739. if (opt[7]=='=') env_preload = opt+8;
  1740. else if (opt[7]) *argv = 0;
  1741. else if (*argv) env_preload = *argv++;
  1742. } else if (!memcmp(opt, "argv0", 5)) {
  1743. if (opt[5]=='=') replace_argv0 = opt+6;
  1744. else if (opt[5]) *argv = 0;
  1745. else if (*argv) replace_argv0 = *argv++;
  1746. } else {
  1747. argv[0] = 0;
  1748. }
  1749. }
  1750. argv[-1] = (void *)(argc - (argv-argv_orig));
  1751. if (!argv[0]) {
  1752. dprintf(2, "musl libc (" LDSO_ARCH ")\n"
  1753. "Version %s\n"
  1754. "Dynamic Program Loader\n"
  1755. "Usage: %s [options] [--] pathname%s\n",
  1756. __libc_version, ldname,
  1757. ldd_mode ? "" : " [args]");
  1758. _exit(1);
  1759. }
  1760. fd = open(argv[0], O_RDONLY);
  1761. if (fd < 0) {
  1762. dprintf(2, "%s: cannot load %s: %s\n", ldname, argv[0], strerror(errno));
  1763. _exit(1);
  1764. }
  1765. Ehdr *ehdr = map_library(fd, &app);
  1766. if (!ehdr) {
  1767. dprintf(2, "%s: %s: Not a valid dynamic program\n", ldname, argv[0]);
  1768. _exit(1);
  1769. }
  1770. close(fd);
  1771. ldso.name = ldname;
  1772. app.name = argv[0];
  1773. aux[AT_ENTRY] = (size_t)laddr(&app, ehdr->e_entry);
  1774. /* Find the name that would have been used for the dynamic
  1775. * linker had ldd not taken its place. */
  1776. if (ldd_mode) {
  1777. for (i=0; i<app.phnum; i++) {
  1778. if (app.phdr[i].p_type == PT_INTERP)
  1779. ldso.name = laddr(&app, app.phdr[i].p_vaddr);
  1780. }
  1781. dprintf(1, "\t%s (%p)\n", ldso.name, ldso.base);
  1782. }
  1783. }
  1784. if (app.tls.size) {
  1785. libc.tls_head = tls_tail = &app.tls;
  1786. app.tls_id = tls_cnt = 1;
  1787. #ifdef TLS_ABOVE_TP
  1788. app.tls.offset = GAP_ABOVE_TP;
  1789. app.tls.offset += (-GAP_ABOVE_TP + (uintptr_t)app.tls.image)
  1790. & (app.tls.align-1);
  1791. tls_offset = app.tls.offset + app.tls.size;
  1792. #else
  1793. tls_offset = app.tls.offset = app.tls.size
  1794. + ( -((uintptr_t)app.tls.image + app.tls.size)
  1795. & (app.tls.align-1) );
  1796. #endif
  1797. tls_align = MAXP2(tls_align, app.tls.align);
  1798. }
  1799. decode_dyn(&app);
  1800. if (DL_FDPIC) {
  1801. makefuncdescs(&app);
  1802. if (!app.loadmap) {
  1803. app.loadmap = (void *)&app_dummy_loadmap;
  1804. app.loadmap->nsegs = 1;
  1805. app.loadmap->segs[0].addr = (size_t)app.map;
  1806. app.loadmap->segs[0].p_vaddr = (size_t)app.map
  1807. - (size_t)app.base;
  1808. app.loadmap->segs[0].p_memsz = app.map_len;
  1809. }
  1810. argv[-3] = (void *)app.loadmap;
  1811. }
  1812. /* Initial dso chain consists only of the app. */
  1813. head = tail = syms_tail = &app;
  1814. /* Donate unused parts of app and library mapping to malloc */
  1815. reclaim_gaps(&app);
  1816. reclaim_gaps(&ldso);
  1817. /* Load preload/needed libraries, add symbols to global namespace. */
  1818. ldso.deps = (struct dso **)no_deps;
  1819. if (env_preload) load_preload(env_preload);
  1820. load_deps(&app);
  1821. for (struct dso *p=head; p; p=p->next)
  1822. add_syms(p);
  1823. /* Attach to vdso, if provided by the kernel, last so that it does
  1824. * not become part of the global namespace. */
  1825. if (search_vec(auxv, &vdso_base, AT_SYSINFO_EHDR) && vdso_base) {
  1826. Ehdr *ehdr = (void *)vdso_base;
  1827. Phdr *phdr = vdso.phdr = (void *)(vdso_base + ehdr->e_phoff);
  1828. vdso.phnum = ehdr->e_phnum;
  1829. vdso.phentsize = ehdr->e_phentsize;
  1830. for (i=ehdr->e_phnum; i; i--, phdr=(void *)((char *)phdr + ehdr->e_phentsize)) {
  1831. if (phdr->p_type == PT_DYNAMIC)
  1832. vdso.dynv = (void *)(vdso_base + phdr->p_offset);
  1833. if (phdr->p_type == PT_LOAD)
  1834. vdso.base = (void *)(vdso_base - phdr->p_vaddr + phdr->p_offset);
  1835. }
  1836. vdso.name = "";
  1837. vdso.shortname = "linux-gate.so.1";
  1838. vdso.relocated = 1;
  1839. vdso.deps = (struct dso **)no_deps;
  1840. decode_dyn(&vdso);
  1841. vdso.prev = tail;
  1842. tail->next = &vdso;
  1843. tail = &vdso;
  1844. }
  1845. for (i=0; app.dynv[i]; i+=2) {
  1846. if (!DT_DEBUG_INDIRECT && app.dynv[i]==DT_DEBUG)
  1847. app.dynv[i+1] = (size_t)&debug;
  1848. if (DT_DEBUG_INDIRECT && app.dynv[i]==DT_DEBUG_INDIRECT) {
  1849. size_t *ptr = (size_t *) app.dynv[i+1];
  1850. *ptr = (size_t)&debug;
  1851. }
  1852. if (app.dynv[i]==DT_DEBUG_INDIRECT_REL) {
  1853. size_t *ptr = (size_t *)((size_t)&app.dynv[i] + app.dynv[i+1]);
  1854. *ptr = (size_t)&debug;
  1855. }
  1856. }
  1857. /* This must be done before final relocations, since it calls
  1858. * malloc, which may be provided by the application. Calling any
  1859. * application code prior to the jump to its entry point is not
  1860. * valid in our model and does not work with FDPIC, where there
  1861. * are additional relocation-like fixups that only the entry point
  1862. * code can see to perform. */
  1863. main_ctor_queue = queue_ctors(&app);
  1864. /* Initial TLS must also be allocated before final relocations
  1865. * might result in calloc being a call to application code. */
  1866. update_tls_size();
  1867. void *initial_tls = builtin_tls;
  1868. if (libc.tls_size > sizeof builtin_tls || tls_align > MIN_TLS_ALIGN) {
  1869. initial_tls = calloc(libc.tls_size, 1);
  1870. if (!initial_tls) {
  1871. dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
  1872. argv[0], libc.tls_size);
  1873. _exit(127);
  1874. }
  1875. }
  1876. static_tls_cnt = tls_cnt;
  1877. /* The main program must be relocated LAST since it may contain
  1878. * copy relocations which depend on libraries' relocations. */
  1879. reloc_all(app.next);
  1880. reloc_all(&app);
  1881. /* Actual copying to new TLS needs to happen after relocations,
  1882. * since the TLS images might have contained relocated addresses. */
  1883. if (initial_tls != builtin_tls) {
  1884. if (__init_tp(__copy_tls(initial_tls)) < 0) {
  1885. a_crash();
  1886. }
  1887. } else {
  1888. size_t tmp_tls_size = libc.tls_size;
  1889. pthread_t self = __pthread_self();
  1890. /* Temporarily set the tls size to the full size of
  1891. * builtin_tls so that __copy_tls will use the same layout
  1892. * as it did for before. Then check, just to be safe. */
  1893. libc.tls_size = sizeof builtin_tls;
  1894. if (__copy_tls((void*)builtin_tls) != self) a_crash();
  1895. libc.tls_size = tmp_tls_size;
  1896. }
  1897. if (ldso_fail) _exit(127);
  1898. if (ldd_mode) _exit(0);
  1899. /* Determine if malloc was interposed by a replacement implementation
  1900. * so that calloc and the memalign family can harden against the
  1901. * possibility of incomplete replacement. */
  1902. if (find_sym(head, "malloc", 1).dso != &ldso)
  1903. __malloc_replaced = 1;
  1904. if (find_sym(head, "aligned_alloc", 1).dso != &ldso)
  1905. __aligned_alloc_replaced = 1;
  1906. /* Switch to runtime mode: any further failures in the dynamic
  1907. * linker are a reportable failure rather than a fatal startup
  1908. * error. */
  1909. runtime = 1;
  1910. debug.ver = 1;
  1911. debug.bp = dl_debug_state;
  1912. debug.head = head;
  1913. debug.base = ldso.base;
  1914. debug.state = RT_CONSISTENT;
  1915. _dl_debug_state();
  1916. if (replace_argv0) argv[0] = replace_argv0;
  1917. errno = 0;
  1918. CRTJMP((void *)aux[AT_ENTRY], argv-1);
  1919. for(;;);
  1920. }
  1921. static void prepare_lazy(struct dso *p)
  1922. {
  1923. size_t dyn[DYN_CNT], n, flags1=0;
  1924. decode_vec(p->dynv, dyn, DYN_CNT);
  1925. search_vec(p->dynv, &flags1, DT_FLAGS_1);
  1926. if (dyn[DT_BIND_NOW] || (dyn[DT_FLAGS] & DF_BIND_NOW) || (flags1 & DF_1_NOW))
  1927. return;
  1928. n = dyn[DT_RELSZ]/2 + dyn[DT_RELASZ]/3 + dyn[DT_PLTRELSZ]/2 + 1;
  1929. if (NEED_MIPS_GOT_RELOCS) {
  1930. size_t j=0; search_vec(p->dynv, &j, DT_MIPS_GOTSYM);
  1931. size_t i=0; search_vec(p->dynv, &i, DT_MIPS_SYMTABNO);
  1932. n += i-j;
  1933. }
  1934. p->lazy = calloc(n, 3*sizeof(size_t));
  1935. if (!p->lazy) {
  1936. error("Error preparing lazy relocation for %s: %m", p->name);
  1937. longjmp(*rtld_fail, 1);
  1938. }
  1939. p->lazy_next = lazy_head;
  1940. lazy_head = p;
  1941. }
  1942. void *dlopen(const char *file, int mode)
  1943. {
  1944. struct dso *volatile p, *orig_tail, *orig_syms_tail, *orig_lazy_head, *next;
  1945. struct tls_module *orig_tls_tail;
  1946. size_t orig_tls_cnt, orig_tls_offset, orig_tls_align;
  1947. size_t i;
  1948. int cs;
  1949. jmp_buf jb;
  1950. struct dso **volatile ctor_queue = 0;
  1951. if (!file) return head;
  1952. pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs);
  1953. pthread_rwlock_wrlock(&lock);
  1954. __inhibit_ptc();
  1955. debug.state = RT_ADD;
  1956. _dl_debug_state();
  1957. p = 0;
  1958. if (shutting_down) {
  1959. error("Cannot dlopen while program is exiting.");
  1960. goto end;
  1961. }
  1962. orig_tls_tail = tls_tail;
  1963. orig_tls_cnt = tls_cnt;
  1964. orig_tls_offset = tls_offset;
  1965. orig_tls_align = tls_align;
  1966. orig_lazy_head = lazy_head;
  1967. orig_syms_tail = syms_tail;
  1968. orig_tail = tail;
  1969. noload = mode & RTLD_NOLOAD;
  1970. rtld_fail = &jb;
  1971. if (setjmp(*rtld_fail)) {
  1972. /* Clean up anything new that was (partially) loaded */
  1973. revert_syms(orig_syms_tail);
  1974. for (p=orig_tail->next; p; p=next) {
  1975. next = p->next;
  1976. while (p->td_index) {
  1977. void *tmp = p->td_index->next;
  1978. free(p->td_index);
  1979. p->td_index = tmp;
  1980. }
  1981. free(p->funcdescs);
  1982. if (p->rpath != p->rpath_orig)
  1983. free(p->rpath);
  1984. free(p->deps);
  1985. unmap_library(p);
  1986. free(p);
  1987. }
  1988. free(ctor_queue);
  1989. ctor_queue = 0;
  1990. if (!orig_tls_tail) libc.tls_head = 0;
  1991. tls_tail = orig_tls_tail;
  1992. if (tls_tail) tls_tail->next = 0;
  1993. tls_cnt = orig_tls_cnt;
  1994. tls_offset = orig_tls_offset;
  1995. tls_align = orig_tls_align;
  1996. lazy_head = orig_lazy_head;
  1997. tail = orig_tail;
  1998. tail->next = 0;
  1999. p = 0;
  2000. goto end;
  2001. } else p = load_library(file, head);
  2002. if (!p) {
  2003. error(noload ?
  2004. "Library %s is not already loaded" :
  2005. "Error loading shared library %s: %m",
  2006. file);
  2007. goto end;
  2008. }
  2009. /* First load handling */
  2010. load_deps(p);
  2011. extend_bfs_deps(p);
  2012. pthread_mutex_lock(&init_fini_lock);
  2013. int constructed = p->constructed;
  2014. pthread_mutex_unlock(&init_fini_lock);
  2015. if (!constructed) ctor_queue = queue_ctors(p);
  2016. if (!p->relocated && (mode & RTLD_LAZY)) {
  2017. prepare_lazy(p);
  2018. for (i=0; p->deps[i]; i++)
  2019. if (!p->deps[i]->relocated)
  2020. prepare_lazy(p->deps[i]);
  2021. }
  2022. if (!p->relocated || (mode & RTLD_GLOBAL)) {
  2023. /* Make new symbols global, at least temporarily, so we can do
  2024. * relocations. If not RTLD_GLOBAL, this is reverted below. */
  2025. add_syms(p);
  2026. for (i=0; p->deps[i]; i++)
  2027. add_syms(p->deps[i]);
  2028. }
  2029. if (!p->relocated) {
  2030. reloc_all(p);
  2031. }
  2032. /* If RTLD_GLOBAL was not specified, undo any new additions
  2033. * to the global symbol table. This is a nop if the library was
  2034. * previously loaded and already global. */
  2035. if (!(mode & RTLD_GLOBAL))
  2036. revert_syms(orig_syms_tail);
  2037. /* Processing of deferred lazy relocations must not happen until
  2038. * the new libraries are committed; otherwise we could end up with
  2039. * relocations resolved to symbol definitions that get removed. */
  2040. redo_lazy_relocs();
  2041. update_tls_size();
  2042. if (tls_cnt != orig_tls_cnt)
  2043. install_new_tls();
  2044. orig_tail = tail;
  2045. end:
  2046. debug.state = RT_CONSISTENT;
  2047. _dl_debug_state();
  2048. __release_ptc();
  2049. if (p) gencnt++;
  2050. pthread_rwlock_unlock(&lock);
  2051. if (ctor_queue) {
  2052. do_init_fini(ctor_queue);
  2053. free(ctor_queue);
  2054. }
  2055. pthread_setcancelstate(cs, 0);
  2056. return p;
  2057. }
  2058. hidden int __dl_invalid_handle(void *h)
  2059. {
  2060. struct dso *p;
  2061. for (p=head; p; p=p->next) if (h==p) return 0;
  2062. error("Invalid library handle %p", (void *)h);
  2063. return 1;
  2064. }
  2065. static void *addr2dso(size_t a)
  2066. {
  2067. struct dso *p;
  2068. size_t i;
  2069. if (DL_FDPIC) for (p=head; p; p=p->next) {
  2070. i = count_syms(p);
  2071. if (a-(size_t)p->funcdescs < i*sizeof(*p->funcdescs))
  2072. return p;
  2073. }
  2074. for (p=head; p; p=p->next) {
  2075. if (DL_FDPIC && p->loadmap) {
  2076. for (i=0; i<p->loadmap->nsegs; i++) {
  2077. if (a-p->loadmap->segs[i].p_vaddr
  2078. < p->loadmap->segs[i].p_memsz)
  2079. return p;
  2080. }
  2081. } else {
  2082. Phdr *ph = p->phdr;
  2083. size_t phcnt = p->phnum;
  2084. size_t entsz = p->phentsize;
  2085. size_t base = (size_t)p->base;
  2086. for (; phcnt--; ph=(void *)((char *)ph+entsz)) {
  2087. if (ph->p_type != PT_LOAD) continue;
  2088. if (a-base-ph->p_vaddr < ph->p_memsz)
  2089. return p;
  2090. }
  2091. if (a-(size_t)p->map < p->map_len)
  2092. return 0;
  2093. }
  2094. }
  2095. return 0;
  2096. }
  2097. static void *do_dlsym(struct dso *p, const char *s, void *ra)
  2098. {
  2099. int use_deps = 0;
  2100. if (p == head || p == RTLD_DEFAULT) {
  2101. p = head;
  2102. } else if (p == RTLD_NEXT) {
  2103. p = addr2dso((size_t)ra);
  2104. if (!p) p=head;
  2105. p = p->next;
  2106. } else if (__dl_invalid_handle(p)) {
  2107. return 0;
  2108. } else
  2109. use_deps = 1;
  2110. struct symdef def = find_sym2(p, s, 0, use_deps);
  2111. if (!def.sym) {
  2112. error("Symbol not found: %s", s);
  2113. return 0;
  2114. }
  2115. if ((def.sym->st_info&0xf) == STT_TLS)
  2116. return __tls_get_addr((tls_mod_off_t []){def.dso->tls_id, def.sym->st_value-DTP_OFFSET});
  2117. if (DL_FDPIC && (def.sym->st_info&0xf) == STT_FUNC)
  2118. return def.dso->funcdescs + (def.sym - def.dso->syms);
  2119. return laddr(def.dso, def.sym->st_value);
  2120. }
  2121. int dladdr(const void *addr_arg, Dl_info *info)
  2122. {
  2123. size_t addr = (size_t)addr_arg;
  2124. struct dso *p;
  2125. Sym *sym, *bestsym;
  2126. uint32_t nsym;
  2127. char *strings;
  2128. size_t best = 0;
  2129. size_t besterr = -1;
  2130. pthread_rwlock_rdlock(&lock);
  2131. p = addr2dso(addr);
  2132. pthread_rwlock_unlock(&lock);
  2133. if (!p) return 0;
  2134. sym = p->syms;
  2135. strings = p->strings;
  2136. nsym = count_syms(p);
  2137. if (DL_FDPIC) {
  2138. size_t idx = (addr-(size_t)p->funcdescs)
  2139. / sizeof(*p->funcdescs);
  2140. if (idx < nsym && (sym[idx].st_info&0xf) == STT_FUNC) {
  2141. best = (size_t)(p->funcdescs + idx);
  2142. bestsym = sym + idx;
  2143. besterr = 0;
  2144. }
  2145. }
  2146. if (!best) for (; nsym; nsym--, sym++) {
  2147. if (sym->st_value
  2148. && (1<<(sym->st_info&0xf) & OK_TYPES)
  2149. && (1<<(sym->st_info>>4) & OK_BINDS)) {
  2150. size_t symaddr = (size_t)laddr(p, sym->st_value);
  2151. if (symaddr > addr || symaddr <= best)
  2152. continue;
  2153. best = symaddr;
  2154. bestsym = sym;
  2155. besterr = addr - symaddr;
  2156. if (addr == symaddr)
  2157. break;
  2158. }
  2159. }
  2160. if (best && besterr > bestsym->st_size-1) {
  2161. best = 0;
  2162. bestsym = 0;
  2163. }
  2164. info->dli_fname = p->name;
  2165. info->dli_fbase = p->map;
  2166. if (!best) {
  2167. info->dli_sname = 0;
  2168. info->dli_saddr = 0;
  2169. return 1;
  2170. }
  2171. if (DL_FDPIC && (bestsym->st_info&0xf) == STT_FUNC)
  2172. best = (size_t)(p->funcdescs + (bestsym - p->syms));
  2173. info->dli_sname = strings + bestsym->st_name;
  2174. info->dli_saddr = (void *)best;
  2175. return 1;
  2176. }
  2177. hidden void *__dlsym(void *restrict p, const char *restrict s, void *restrict ra)
  2178. {
  2179. void *res;
  2180. pthread_rwlock_rdlock(&lock);
  2181. res = do_dlsym(p, s, ra);
  2182. pthread_rwlock_unlock(&lock);
  2183. return res;
  2184. }
  2185. hidden void *__dlsym_redir_time64(void *restrict p, const char *restrict s, void *restrict ra)
  2186. {
  2187. #if _REDIR_TIME64
  2188. const char *suffix, *suffix2 = "";
  2189. char redir[36];
  2190. /* Map the symbol name to a time64 version of itself according to the
  2191. * pattern used for naming the redirected time64 symbols. */
  2192. size_t l = strnlen(s, sizeof redir);
  2193. if (l<4 || l==sizeof redir) goto no_redir;
  2194. if (s[l-2]=='_' && s[l-1]=='r') {
  2195. l -= 2;
  2196. suffix2 = s+l;
  2197. }
  2198. if (l<4) goto no_redir;
  2199. if (!strcmp(s+l-4, "time")) suffix = "64";
  2200. else suffix = "_time64";
  2201. /* Use the presence of the remapped symbol name in libc to determine
  2202. * whether it's one that requires time64 redirection; replace if so. */
  2203. snprintf(redir, sizeof redir, "__%.*s%s%s", (int)l, s, suffix, suffix2);
  2204. if (find_sym(&ldso, redir, 1).sym) s = redir;
  2205. no_redir:
  2206. #endif
  2207. return __dlsym(p, s, ra);
  2208. }
  2209. int dl_iterate_phdr(int(*callback)(struct dl_phdr_info *info, size_t size, void *data), void *data)
  2210. {
  2211. struct dso *current;
  2212. struct dl_phdr_info info;
  2213. int ret = 0;
  2214. for(current = head; current;) {
  2215. info.dlpi_addr = (uintptr_t)current->base;
  2216. info.dlpi_name = current->name;
  2217. info.dlpi_phdr = current->phdr;
  2218. info.dlpi_phnum = current->phnum;
  2219. info.dlpi_adds = gencnt;
  2220. info.dlpi_subs = 0;
  2221. info.dlpi_tls_modid = current->tls_id;
  2222. info.dlpi_tls_data = !current->tls_id ? 0 :
  2223. __tls_get_addr((tls_mod_off_t[]){current->tls_id,0});
  2224. ret = (callback)(&info, sizeof (info), data);
  2225. if (ret != 0) break;
  2226. pthread_rwlock_rdlock(&lock);
  2227. current = current->next;
  2228. pthread_rwlock_unlock(&lock);
  2229. }
  2230. return ret;
  2231. }
  2232. static void error_impl(const char *fmt, ...)
  2233. {
  2234. va_list ap;
  2235. va_start(ap, fmt);
  2236. if (!runtime) {
  2237. vdprintf(2, fmt, ap);
  2238. dprintf(2, "\n");
  2239. ldso_fail = 1;
  2240. va_end(ap);
  2241. return;
  2242. }
  2243. __dl_vseterr(fmt, ap);
  2244. va_end(ap);
  2245. }
  2246. static void error_noop(const char *fmt, ...)
  2247. {
  2248. }