SidebarPlugin.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760
  1. import re
  2. import os
  3. import cgi
  4. import sys
  5. import math
  6. import time
  7. import json
  8. try:
  9. import cStringIO as StringIO
  10. except:
  11. import StringIO
  12. import gevent
  13. from Config import config
  14. from Plugin import PluginManager
  15. from Debug import Debug
  16. from Translate import Translate
  17. from util import helper
  18. from ZipStream import ZipStream
  19. plugin_dir = "plugins/Sidebar"
  20. media_dir = plugin_dir + "/media"
  21. sys.path.append(plugin_dir) # To able to load geoip lib
  22. loc_cache = {}
  23. if "_" not in locals():
  24. _ = Translate(plugin_dir + "/languages/")
  25. @PluginManager.registerTo("UiRequest")
  26. class UiRequestPlugin(object):
  27. # Inject our resources to end of original file streams
  28. def actionUiMedia(self, path):
  29. if path == "/uimedia/all.js" or path == "/uimedia/all.css":
  30. # First yield the original file and header
  31. body_generator = super(UiRequestPlugin, self).actionUiMedia(path)
  32. for part in body_generator:
  33. yield part
  34. # Append our media file to the end
  35. ext = re.match(".*(js|css)$", path).group(1)
  36. plugin_media_file = "%s/all.%s" % (media_dir, ext)
  37. if config.debug:
  38. # If debugging merge *.css to all.css and *.js to all.js
  39. from Debug import DebugMedia
  40. DebugMedia.merge(plugin_media_file)
  41. if ext == "js":
  42. yield _.translateData(open(plugin_media_file).read())
  43. else:
  44. for part in self.actionFile(plugin_media_file, send_header=False):
  45. yield part
  46. elif path.startswith("/uimedia/globe/"): # Serve WebGL globe files
  47. file_name = re.match(".*/(.*)", path).group(1)
  48. plugin_media_file = "%s-globe/%s" % (media_dir, file_name)
  49. if config.debug and path.endswith("all.js"):
  50. # If debugging merge *.css to all.css and *.js to all.js
  51. from Debug import DebugMedia
  52. DebugMedia.merge(plugin_media_file)
  53. for part in self.actionFile(plugin_media_file):
  54. yield part
  55. else:
  56. for part in super(UiRequestPlugin, self).actionUiMedia(path):
  57. yield part
  58. def actionZip(self):
  59. address = self.get["address"]
  60. site = self.server.site_manager.get(address)
  61. if not site:
  62. return self.error404("Site not found")
  63. title = site.content_manager.contents.get("content.json", {}).get("title", "").encode('ascii', 'ignore')
  64. filename = "%s-backup-%s.zip" % (title, time.strftime("%Y-%m-%d_%H_%M"))
  65. self.sendHeader(content_type="application/zip", extra_headers={'Content-Disposition': 'attachment; filename="%s"' % filename})
  66. return self.streamZip(site.storage.getPath("."))
  67. def streamZip(self, file_path):
  68. zs = ZipStream(file_path)
  69. while 1:
  70. data = zs.read()
  71. if not data:
  72. break
  73. yield data
  74. @PluginManager.registerTo("UiWebsocket")
  75. class UiWebsocketPlugin(object):
  76. def sidebarRenderPeerStats(self, body, site):
  77. connected = len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected])
  78. connectable = len([peer_id for peer_id in site.peers.keys() if not peer_id.endswith(":0")])
  79. onion = len([peer_id for peer_id in site.peers.keys() if ".onion" in peer_id])
  80. local = len([peer for peer in site.peers.values() if helper.isPrivateIp(peer.ip)])
  81. peers_total = len(site.peers)
  82. # Add myself
  83. if site.settings["serving"]:
  84. peers_total += 1
  85. if any(site.connection_server.port_opened.values()):
  86. connectable += 1
  87. if site.connection_server.tor_manager.start_onions:
  88. onion += 1
  89. if peers_total:
  90. percent_connected = float(connected) / peers_total
  91. percent_connectable = float(connectable) / peers_total
  92. percent_onion = float(onion) / peers_total
  93. else:
  94. percent_connectable = percent_connected = percent_onion = 0
  95. if local:
  96. local_html = _(u"<li class='color-yellow'><span>{_[Local]}:</span><b>{local}</b></li>")
  97. else:
  98. local_html = ""
  99. peer_ips = [peer.key for peer in site.getConnectablePeers(20, allow_private=False)]
  100. peer_ips.sort(key=lambda peer_ip: ".onion:" in peer_ip)
  101. copy_link = "http://127.0.0.1:43110/%s/?zeronet_peers=%s" % (
  102. site.content_manager.contents["content.json"].get("domain", site.address),
  103. ",".join(peer_ips)
  104. )
  105. body.append(_(u"""
  106. <li>
  107. <label>
  108. {_[Peers]}
  109. <small class="label-right"><a href='{copy_link}' id='link-copypeers' class='link-right'>{_[Copy to clipboard]}</a></small>
  110. </label>
  111. <ul class='graph'>
  112. <li style='width: 100%' class='total back-black' title="{_[Total peers]}"></li>
  113. <li style='width: {percent_connectable:.0%}' class='connectable back-blue' title='{_[Connectable peers]}'></li>
  114. <li style='width: {percent_onion:.0%}' class='connected back-purple' title='{_[Onion]}'></li>
  115. <li style='width: {percent_connected:.0%}' class='connected back-green' title='{_[Connected peers]}'></li>
  116. </ul>
  117. <ul class='graph-legend'>
  118. <li class='color-green'><span>{_[Connected]}:</span><b>{connected}</b></li>
  119. <li class='color-blue'><span>{_[Connectable]}:</span><b>{connectable}</b></li>
  120. <li class='color-purple'><span>{_[Onion]}:</span><b>{onion}</b></li>
  121. {local_html}
  122. <li class='color-black'><span>{_[Total]}:</span><b>{peers_total}</b></li>
  123. </ul>
  124. </li>
  125. """.replace("{local_html}", local_html)))
  126. def sidebarRenderTransferStats(self, body, site):
  127. recv = float(site.settings.get("bytes_recv", 0)) / 1024 / 1024
  128. sent = float(site.settings.get("bytes_sent", 0)) / 1024 / 1024
  129. transfer_total = recv + sent
  130. if transfer_total:
  131. percent_recv = recv / transfer_total
  132. percent_sent = sent / transfer_total
  133. else:
  134. percent_recv = 0.5
  135. percent_sent = 0.5
  136. body.append(_(u"""
  137. <li>
  138. <label>{_[Data transfer]}</label>
  139. <ul class='graph graph-stacked'>
  140. <li style='width: {percent_recv:.0%}' class='received back-yellow' title="{_[Received bytes]}"></li>
  141. <li style='width: {percent_sent:.0%}' class='sent back-green' title="{_[Sent bytes]}"></li>
  142. </ul>
  143. <ul class='graph-legend'>
  144. <li class='color-yellow'><span>{_[Received]}:</span><b>{recv:.2f}MB</b></li>
  145. <li class='color-green'<span>{_[Sent]}:</span><b>{sent:.2f}MB</b></li>
  146. </ul>
  147. </li>
  148. """))
  149. def sidebarRenderFileStats(self, body, site):
  150. body.append(_(u"""
  151. <li>
  152. <label>
  153. {_[Files]}
  154. <small class="label-right"><a href='#Site+directory' id='link-directory' class='link-right'>{_[Open site directory]}</a>
  155. <a href='/ZeroNet-Internal/Zip?address={site.address}' id='link-zip' class='link-right' download='site.zip'>{_[Save as .zip]}</a></small>
  156. </label>
  157. <ul class='graph graph-stacked'>
  158. """))
  159. extensions = (
  160. ("html", "yellow"),
  161. ("css", "orange"),
  162. ("js", "purple"),
  163. ("Image", "green"),
  164. ("json", "darkblue"),
  165. ("User data", "blue"),
  166. ("Other", "white"),
  167. ("Total", "black")
  168. )
  169. # Collect stats
  170. size_filetypes = {}
  171. size_total = 0
  172. contents = site.content_manager.listContents() # Without user files
  173. for inner_path in contents:
  174. content = site.content_manager.contents[inner_path]
  175. if "files" not in content or content["files"] is None:
  176. continue
  177. for file_name, file_details in content["files"].items():
  178. size_total += file_details["size"]
  179. ext = file_name.split(".")[-1]
  180. size_filetypes[ext] = size_filetypes.get(ext, 0) + file_details["size"]
  181. # Get user file sizes
  182. size_user_content = site.content_manager.contents.execute(
  183. "SELECT SUM(size) + SUM(size_files) AS size FROM content WHERE ?",
  184. {"not__inner_path": contents}
  185. ).fetchone()["size"]
  186. if not size_user_content:
  187. size_user_content = 0
  188. size_filetypes["User data"] = size_user_content
  189. size_total += size_user_content
  190. # The missing difference is content.json sizes
  191. if "json" in size_filetypes:
  192. size_filetypes["json"] += max(0, site.settings["size"] - size_total)
  193. size_total = size_other = site.settings["size"]
  194. # Bar
  195. for extension, color in extensions:
  196. if extension == "Total":
  197. continue
  198. if extension == "Other":
  199. size = max(0, size_other)
  200. elif extension == "Image":
  201. size = size_filetypes.get("jpg", 0) + size_filetypes.get("png", 0) + size_filetypes.get("gif", 0)
  202. size_other -= size
  203. else:
  204. size = size_filetypes.get(extension, 0)
  205. size_other -= size
  206. if size_total == 0:
  207. percent = 0
  208. else:
  209. percent = 100 * (float(size) / size_total)
  210. percent = math.floor(percent * 100) / 100 # Floor to 2 digits
  211. body.append(
  212. u"""<li style='width: %.2f%%' class='%s back-%s' title="%s"></li>""" %
  213. (percent, _[extension], color, _[extension])
  214. )
  215. # Legend
  216. body.append("</ul><ul class='graph-legend'>")
  217. for extension, color in extensions:
  218. if extension == "Other":
  219. size = max(0, size_other)
  220. elif extension == "Image":
  221. size = size_filetypes.get("jpg", 0) + size_filetypes.get("png", 0) + size_filetypes.get("gif", 0)
  222. elif extension == "Total":
  223. size = size_total
  224. else:
  225. size = size_filetypes.get(extension, 0)
  226. if extension == "js":
  227. title = "javascript"
  228. else:
  229. title = extension
  230. if size > 1024 * 1024 * 10: # Format as mB is more than 10mB
  231. size_formatted = "%.0fMB" % (size / 1024 / 1024)
  232. else:
  233. size_formatted = "%.0fkB" % (size / 1024)
  234. body.append(u"<li class='color-%s'><span>%s:</span><b>%s</b></li>" % (color, _[title], size_formatted))
  235. body.append("</ul></li>")
  236. def sidebarRenderSizeLimit(self, body, site):
  237. free_space = helper.getFreeSpace() / 1024 / 1024
  238. size = float(site.settings["size"]) / 1024 / 1024
  239. size_limit = site.getSizeLimit()
  240. percent_used = size / size_limit
  241. body.append(_(u"""
  242. <li>
  243. <label>{_[Size limit]} <small>({_[limit used]}: {percent_used:.0%}, {_[free space]}: {free_space:,d}MB)</small></label>
  244. <input type='text' class='text text-num' value="{size_limit}" id='input-sitelimit'/><span class='text-post'>MB</span>
  245. <a href='#Set' class='button' id='button-sitelimit'>{_[Set]}</a>
  246. </li>
  247. """))
  248. def sidebarRenderOptionalFileStats(self, body, site):
  249. size_total = float(site.settings["size_optional"])
  250. size_downloaded = float(site.settings["optional_downloaded"])
  251. if not size_total:
  252. return False
  253. percent_downloaded = size_downloaded / size_total
  254. size_formatted_total = size_total / 1024 / 1024
  255. size_formatted_downloaded = size_downloaded / 1024 / 1024
  256. body.append(_(u"""
  257. <li>
  258. <label>{_[Optional files]}</label>
  259. <ul class='graph'>
  260. <li style='width: 100%' class='total back-black' title="{_[Total size]}"></li>
  261. <li style='width: {percent_downloaded:.0%}' class='connected back-green' title='{_[Downloaded files]}'></li>
  262. </ul>
  263. <ul class='graph-legend'>
  264. <li class='color-green'><span>{_[Downloaded]}:</span><b>{size_formatted_downloaded:.2f}MB</b></li>
  265. <li class='color-black'><span>{_[Total]}:</span><b>{size_formatted_total:.2f}MB</b></li>
  266. </ul>
  267. </li>
  268. """))
  269. return True
  270. def sidebarRenderOptionalFileSettings(self, body, site):
  271. if self.site.settings.get("autodownloadoptional"):
  272. checked = "checked='checked'"
  273. else:
  274. checked = ""
  275. body.append(_(u"""
  276. <li>
  277. <label>{_[Download and help distribute all files]}</label>
  278. <input type="checkbox" class="checkbox" id="checkbox-autodownloadoptional" {checked}/><div class="checkbox-skin"></div>
  279. """))
  280. autodownload_bigfile_size_limit = int(site.settings.get("autodownload_bigfile_size_limit", config.autodownload_bigfile_size_limit))
  281. body.append(_(u"""
  282. <div class='settings-autodownloadoptional'>
  283. <label>{_[Auto download big file size limit]}</label>
  284. <input type='text' class='text text-num' value="{autodownload_bigfile_size_limit}" id='input-autodownload_bigfile_size_limit'/><span class='text-post'>MB</span>
  285. <a href='#Set' class='button' id='button-autodownload_bigfile_size_limit'>{_[Set]}</a>
  286. </div>
  287. """))
  288. body.append("</li>")
  289. def sidebarRenderBadFiles(self, body, site):
  290. body.append(_(u"""
  291. <li>
  292. <label>{_[Needs to be updated]}:</label>
  293. <ul class='filelist'>
  294. """))
  295. i = 0
  296. for bad_file, tries in site.bad_files.iteritems():
  297. i += 1
  298. body.append(_(u"""<li class='color-red' title="{bad_file_path} ({tries})">{bad_filename}</li>""", {
  299. "bad_file_path": bad_file,
  300. "bad_filename": helper.getFilename(bad_file),
  301. "tries": _.pluralize(tries, "{} try", "{} tries")
  302. }))
  303. if i > 30:
  304. break
  305. if len(site.bad_files) > 30:
  306. num_bad_files = len(site.bad_files) - 30
  307. body.append(_(u"""<li class='color-red'>{_[+ {num_bad_files} more]}</li>""", nested=True))
  308. body.append("""
  309. </ul>
  310. </li>
  311. """)
  312. def sidebarRenderDbOptions(self, body, site):
  313. if site.storage.db:
  314. inner_path = site.storage.getInnerPath(site.storage.db.db_path)
  315. size = float(site.storage.getSize(inner_path)) / 1024
  316. feeds = len(site.storage.db.schema.get("feeds", {}))
  317. else:
  318. inner_path = _[u"No database found"]
  319. size = 0.0
  320. feeds = 0
  321. body.append(_(u"""
  322. <li>
  323. <label>{_[Database]} <small>({size:.2f}kB, {_[search feeds]}: {_[{feeds} query]})</small></label>
  324. <div class='flex'>
  325. <input type='text' class='text disabled' value="{inner_path}" disabled='disabled'/>
  326. <a href='#Reload' id="button-dbreload" class='button'>{_[Reload]}</a>
  327. <a href='#Rebuild' id="button-dbrebuild" class='button'>{_[Rebuild]}</a>
  328. </div>
  329. </li>
  330. """, nested=True))
  331. def sidebarRenderIdentity(self, body, site):
  332. auth_address = self.user.getAuthAddress(self.site.address, create=False)
  333. rules = self.site.content_manager.getRules("data/users/%s/content.json" % auth_address)
  334. if rules and rules.get("max_size"):
  335. quota = rules["max_size"] / 1024
  336. try:
  337. content = site.content_manager.contents["data/users/%s/content.json" % auth_address]
  338. used = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()])
  339. except:
  340. used = 0
  341. used = used / 1024
  342. else:
  343. quota = used = 0
  344. body.append(_(u"""
  345. <li>
  346. <label>{_[Identity address]} <small>({_[limit used]}: {used:.2f}kB / {quota:.2f}kB)</small></label>
  347. <div class='flex'>
  348. <span class='input text disabled'>{auth_address}</span>
  349. <a href='#Change' class='button' id='button-identity'>{_[Change]}</a>
  350. </div>
  351. </li>
  352. """))
  353. def sidebarRenderControls(self, body, site):
  354. auth_address = self.user.getAuthAddress(self.site.address, create=False)
  355. if self.site.settings["serving"]:
  356. class_pause = ""
  357. class_resume = "hidden"
  358. else:
  359. class_pause = "hidden"
  360. class_resume = ""
  361. body.append(_(u"""
  362. <li>
  363. <label>{_[Site control]}</label>
  364. <a href='#Update' class='button noupdate' id='button-update'>{_[Update]}</a>
  365. <a href='#Pause' class='button {class_pause}' id='button-pause'>{_[Pause]}</a>
  366. <a href='#Resume' class='button {class_resume}' id='button-resume'>{_[Resume]}</a>
  367. <a href='#Delete' class='button noupdate' id='button-delete'>{_[Delete]}</a>
  368. </li>
  369. """))
  370. donate_key = site.content_manager.contents.get("content.json", {}).get("donate", True)
  371. site_address = self.site.address
  372. body.append(_(u"""
  373. <li>
  374. <label>{_[Site address]}</label><br>
  375. <div class='flex'>
  376. <span class='input text disabled'>{site_address}</span>
  377. """))
  378. if donate_key == False or donate_key == "":
  379. pass
  380. elif (type(donate_key) == str or type(donate_key) == unicode) and len(donate_key) > 0:
  381. body.append(_(u"""
  382. </div>
  383. </li>
  384. <li>
  385. <label>{_[Donate]}</label><br>
  386. <div class='flex'>
  387. {donate_key}
  388. """))
  389. else:
  390. body.append(_(u"""
  391. <a href='bitcoin:{site_address}' class='button' id='button-donate'>{_[Donate]}</a>
  392. """))
  393. body.append(_(u"""
  394. </div>
  395. </li>
  396. """))
  397. def sidebarRenderOwnedCheckbox(self, body, site):
  398. if self.site.settings["own"]:
  399. checked = "checked='checked'"
  400. else:
  401. checked = ""
  402. body.append(_(u"""
  403. <h2 class='owned-title'>{_[This is my site]}</h2>
  404. <input type="checkbox" class="checkbox" id="checkbox-owned" {checked}/><div class="checkbox-skin"></div>
  405. """))
  406. def sidebarRenderOwnSettings(self, body, site):
  407. title = site.content_manager.contents.get("content.json", {}).get("title", "")
  408. description = site.content_manager.contents.get("content.json", {}).get("description", "")
  409. body.append(_(u"""
  410. <li>
  411. <label for='settings-title'>{_[Site title]}</label>
  412. <input type='text' class='text' value="{title}" id='settings-title'/>
  413. </li>
  414. <li>
  415. <label for='settings-description'>{_[Site description]}</label>
  416. <input type='text' class='text' value="{description}" id='settings-description'/>
  417. </li>
  418. <li>
  419. <a href='#Save' class='button' id='button-settings'>{_[Save site settings]}</a>
  420. </li>
  421. """))
  422. def sidebarRenderContents(self, body, site):
  423. has_privatekey = bool(self.user.getSiteData(site.address, create=False).get("privatekey"))
  424. if has_privatekey:
  425. tag_privatekey = _(u"{_[Private key saved.]} <a href='#Forgot+private+key' id='privatekey-forgot' class='link-right'>{_[Forgot]}</a>")
  426. else:
  427. tag_privatekey = _(u"<a href='#Add+private+key' id='privatekey-add' class='link-right'>{_[Add saved private key]}</a>")
  428. body.append(_(u"""
  429. <li>
  430. <label>{_[Content publishing]} <small class='label-right'>{tag_privatekey}</small></label>
  431. """.replace("{tag_privatekey}", tag_privatekey)))
  432. # Choose content you want to sign
  433. body.append(_(u"""
  434. <div class='flex'>
  435. <input type='text' class='text' value="content.json" id='input-contents'/>
  436. <a href='#Sign-and-Publish' id='button-sign-publish' class='button'>{_[Sign and publish]}</a>
  437. <a href='#Sign-or-Publish' id='menu-sign-publish'>\u22EE</a>
  438. </div>
  439. """))
  440. contents = ["content.json"]
  441. contents += site.content_manager.contents.get("content.json", {}).get("includes", {}).keys()
  442. body.append(_(u"<div class='contents'>{_[Choose]}: "))
  443. for content in contents:
  444. body.append(_("<a href='{content}' class='contents-content'>{content}</a> "))
  445. body.append("</div>")
  446. body.append("</li>")
  447. def actionSidebarGetHtmlTag(self, to):
  448. permissions = self.getPermissions(to)
  449. if "ADMIN" not in permissions:
  450. return self.response(to, "You don't have permission to run this command")
  451. site = self.site
  452. body = []
  453. body.append("<div>")
  454. body.append("<a href='#Close' class='close'>&times;</a>")
  455. body.append("<h1>%s</h1>" % cgi.escape(site.content_manager.contents.get("content.json", {}).get("title", ""), True))
  456. body.append("<div class='globe loading'></div>")
  457. body.append("<ul class='fields'>")
  458. self.sidebarRenderPeerStats(body, site)
  459. self.sidebarRenderTransferStats(body, site)
  460. self.sidebarRenderFileStats(body, site)
  461. self.sidebarRenderSizeLimit(body, site)
  462. has_optional = self.sidebarRenderOptionalFileStats(body, site)
  463. if has_optional:
  464. self.sidebarRenderOptionalFileSettings(body, site)
  465. self.sidebarRenderDbOptions(body, site)
  466. self.sidebarRenderIdentity(body, site)
  467. self.sidebarRenderControls(body, site)
  468. if site.bad_files:
  469. self.sidebarRenderBadFiles(body, site)
  470. self.sidebarRenderOwnedCheckbox(body, site)
  471. body.append("<div class='settings-owned'>")
  472. self.sidebarRenderOwnSettings(body, site)
  473. self.sidebarRenderContents(body, site)
  474. body.append("</div>")
  475. body.append("</ul>")
  476. body.append("</div>")
  477. body.append("<div class='menu template'>")
  478. body.append("<a href='#'' class='menu-item template'>Template</a>")
  479. body.append("</div>")
  480. self.response(to, "".join(body))
  481. def downloadGeoLiteDb(self, db_path):
  482. import urllib
  483. import gzip
  484. import shutil
  485. from util import helper
  486. self.log.info("Downloading GeoLite2 City database...")
  487. self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], 0])
  488. db_urls = [
  489. "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
  490. "https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz"
  491. ]
  492. for db_url in db_urls:
  493. try:
  494. # Download
  495. response = helper.httpRequest(db_url)
  496. data_size = response.getheader('content-length')
  497. data_recv = 0
  498. data = StringIO.StringIO()
  499. while True:
  500. buff = response.read(1024 * 512)
  501. if not buff:
  502. break
  503. data.write(buff)
  504. data_recv += 1024 * 512
  505. if data_size:
  506. progress = int(float(data_recv) / int(data_size) * 100)
  507. self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress])
  508. self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell())
  509. data.seek(0)
  510. # Unpack
  511. with gzip.GzipFile(fileobj=data) as gzip_file:
  512. shutil.copyfileobj(gzip_file, open(db_path, "wb"))
  513. self.cmd("progress", ["geolite-info", _["GeoLite2 City database downloaded!"], 100])
  514. time.sleep(2) # Wait for notify animation
  515. return True
  516. except Exception as err:
  517. self.log.error("Error downloading %s: %s" % (db_url, err))
  518. pass
  519. self.cmd("progress", [
  520. "geolite-info",
  521. _["GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}"].format(err, db_urls[0]),
  522. -100
  523. ])
  524. def getLoc(self, geodb, ip):
  525. global loc_cache
  526. if ip in loc_cache:
  527. return loc_cache[ip]
  528. else:
  529. try:
  530. loc_data = geodb.get(ip)
  531. except:
  532. loc_data = None
  533. if not loc_data or "location" not in loc_data:
  534. loc_cache[ip] = None
  535. return None
  536. loc = {
  537. "lat": loc_data["location"]["latitude"],
  538. "lon": loc_data["location"]["longitude"],
  539. }
  540. if "city" in loc_data:
  541. loc["city"] = loc_data["city"]["names"]["en"]
  542. if "country" in loc_data:
  543. loc["country"] = loc_data["country"]["names"]["en"]
  544. loc_cache[ip] = loc
  545. return loc
  546. def getPeerLocations(self, peers):
  547. import maxminddb
  548. db_path = config.data_dir + '/GeoLite2-City.mmdb'
  549. if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
  550. if not self.downloadGeoLiteDb(db_path):
  551. return False
  552. geodb = maxminddb.open_database(db_path)
  553. peers = peers.values()
  554. # Place bars
  555. peer_locations = []
  556. placed = {} # Already placed bars here
  557. for peer in peers:
  558. # Height of bar
  559. if peer.connection and peer.connection.last_ping_delay:
  560. ping = round(peer.connection.last_ping_delay * 1000)
  561. else:
  562. ping = None
  563. loc = self.getLoc(geodb, peer.ip)
  564. if not loc:
  565. continue
  566. # Create position array
  567. lat, lon = loc["lat"], loc["lon"]
  568. latlon = "%s,%s" % (lat, lon)
  569. if latlon in placed and helper.getIpType(peer.ip) == "ipv4": # Dont place more than 1 bar to same place, fake repos using ip address last two part
  570. lat += float(128 - int(peer.ip.split(".")[-2])) / 50
  571. lon += float(128 - int(peer.ip.split(".")[-1])) / 50
  572. latlon = "%s,%s" % (lat, lon)
  573. placed[latlon] = True
  574. peer_location = {}
  575. peer_location.update(loc)
  576. peer_location["lat"] = lat
  577. peer_location["lon"] = lon
  578. peer_location["ping"] = ping
  579. peer_locations.append(peer_location)
  580. # Append myself
  581. for ip in self.site.connection_server.ip_external_list:
  582. my_loc = self.getLoc(geodb, ip)
  583. if my_loc:
  584. my_loc["ping"] = 0
  585. peer_locations.append(my_loc)
  586. return peer_locations
  587. def actionSidebarGetPeers(self, to):
  588. permissions = self.getPermissions(to)
  589. if "ADMIN" not in permissions:
  590. return self.response(to, "You don't have permission to run this command")
  591. try:
  592. peer_locations = self.getPeerLocations(self.site.peers)
  593. globe_data = []
  594. ping_times = [
  595. peer_location["ping"]
  596. for peer_location in peer_locations
  597. if peer_location["ping"]
  598. ]
  599. if ping_times:
  600. ping_avg = sum(ping_times) / float(len(ping_times))
  601. else:
  602. ping_avg = 0
  603. for peer_location in peer_locations:
  604. if peer_location["ping"] == 0: # Me
  605. height = -0.135
  606. elif peer_location["ping"]:
  607. height = min(0.20, math.log(1 + peer_location["ping"] / ping_avg, 300))
  608. else:
  609. height = -0.03
  610. globe_data += [peer_location["lat"], peer_location["lon"], height]
  611. self.response(to, globe_data)
  612. except Exception, err:
  613. self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err))
  614. self.response(to, {"error": err})
  615. def actionSiteSetOwned(self, to, owned):
  616. permissions = self.getPermissions(to)
  617. if "ADMIN" not in permissions:
  618. return self.response(to, "You don't have permission to run this command")
  619. if self.site.address == config.updatesite:
  620. return self.response(to, "You can't change the ownership of the updater site")
  621. self.site.settings["own"] = bool(owned)
  622. self.site.updateWebsocket(owned=owned)
  623. def actionUserSetSitePrivatekey(self, to, privatekey):
  624. permissions = self.getPermissions(to)
  625. if "ADMIN" not in permissions:
  626. return self.response(to, "You don't have permission to run this command")
  627. site_data = self.user.sites[self.site.address]
  628. site_data["privatekey"] = privatekey
  629. self.site.updateWebsocket(set_privatekey=bool(privatekey))
  630. return "ok"
  631. def actionSiteSetAutodownloadoptional(self, to, owned):
  632. permissions = self.getPermissions(to)
  633. if "ADMIN" not in permissions:
  634. return self.response(to, "You don't have permission to run this command")
  635. self.site.settings["autodownloadoptional"] = bool(owned)
  636. self.site.bad_files = {}
  637. gevent.spawn(self.site.update, check_files=True)
  638. self.site.worker_manager.removeSolvedFileTasks()
  639. def actionDbReload(self, to):
  640. permissions = self.getPermissions(to)
  641. if "ADMIN" not in permissions:
  642. return self.response(to, "You don't have permission to run this command")
  643. self.site.storage.closeDb()
  644. self.site.storage.getDb()
  645. return self.response(to, "ok")
  646. def actionDbRebuild(self, to):
  647. permissions = self.getPermissions(to)
  648. if "ADMIN" not in permissions:
  649. return self.response(to, "You don't have permission to run this command")
  650. self.site.storage.rebuildDb()
  651. return self.response(to, "ok")