sectors2sqlite.py 2.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132
  1. #!/usr/bin/python3
  2. # Loads block files from sectors folders into map.sqlite database.
  3. # The sectors folder should be safe to remove after this prints "Finished."
  4. import time, os, sys
  5. try:
  6. import sqlite3
  7. except:
  8. exit('You need to have the Python sqlite3 module.')
  9. path = "../world/"
  10. paths = []
  11. # sectors2 gets to try first
  12. if os.path.isdir(path + 'sectors2/'):
  13. paths.append('sectors2')
  14. if os.path.isdir(path + 'sectors/'):
  15. paths.append('sectors')
  16. if not paths:
  17. exit('Could not find sectors folder at ' + path + 'sectors2/ or ' + path + 'sectors/')
  18. def parseSigned12bit(u):
  19. u = int('0x'+u, 16)
  20. return (u if u < 2**11 else u - 2**12)
  21. def parseSigned16bit(u):
  22. u = int('0x'+u, 16)
  23. return (u if u < 2**15 else u - 2**16)
  24. def int64(u):
  25. while u >= 2**63:
  26. u -= 2**64
  27. while u <= -2**63:
  28. u += 2**64
  29. return u
  30. # Convert sector folder(s) to integer
  31. def getSectorPos(dirname):
  32. if len(dirname) == 8:
  33. # Old layout
  34. x = parseSigned16bit(dirname[:4])
  35. z = parseSigned16bit(dirname[4:])
  36. elif len(dirname) == 7:
  37. # New layout
  38. x = parseSigned12bit(dirname[:3])
  39. z = parseSigned12bit(dirname[4:])
  40. else:
  41. print('Terrible sector at ' + dirname)
  42. return
  43. return x, z
  44. # Convert block file to integer position
  45. def getBlockPos(sectordir, blockfile):
  46. p2d = getSectorPos(sectordir)
  47. if not p2d:
  48. return
  49. if len(blockfile) != 4:
  50. print("Invalid block filename: " + blockfile)
  51. y = parseSigned16bit(blockfile)
  52. return p2d[0], y, p2d[1]
  53. # Convert location to integer
  54. def getBlockAsInteger(p):
  55. return int64(p[2]*16777216 + p[1]*4096 + p[0])
  56. # Init
  57. create = False
  58. if not os.path.isfile(path + 'map.sqlite'):
  59. create = True
  60. conn = sqlite3.connect(path + 'map.sqlite')
  61. if not conn:
  62. exit('Could not open database.')
  63. cur = conn.cursor()
  64. if create:
  65. cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (`pos` INT NOT NULL PRIMARY KEY, `data` BLOB);")
  66. conn.commit()
  67. print('Created database at ' + path + 'map.sqlite')
  68. # Crawl the folders
  69. count = 0
  70. t = time.time()
  71. for base in paths:
  72. v = 0
  73. if base == 'sectors':
  74. v = 1
  75. elif base == 'sectors2':
  76. v= 2
  77. else:
  78. print('Ignoring base ' + base)
  79. continue
  80. for root, dirs, files in os.walk(path + base):
  81. if files:
  82. for block in files:
  83. pos = getBlockAsInteger(getBlockPos(root[(-8 if v == 1 else -7 if v == 2 else 0):], block))
  84. if pos is None:
  85. print('Ignoring broken path ' + root + '/' + block)
  86. continue
  87. f = open(root+'/'+block, 'rb')
  88. blob = f.read()
  89. f.close()
  90. if sys.version_info.major == 2:
  91. blob = buffer(blob)
  92. else:
  93. blob = memoryview(blob)
  94. cur.execute('INSERT OR IGNORE INTO `blocks` VALUES(?, ?)', (pos, blob))
  95. count += 1
  96. if(time.time() - t > 3):
  97. t = time.time()
  98. print(str(count)+' blocks processed...')
  99. conn.commit()
  100. print('Finished. (' + str(count) + ' blocks)')