ids_parser.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. #!/usr/bin/env python3
  2. import re
  3. import sys
  4. from pyparsing import (Word, White, Literal, Regex,
  5. LineEnd, SkipTo,
  6. ZeroOrMore, OneOrMore, Combine, Optional, Suppress,
  7. Group,
  8. stringEnd, pythonStyleComment)
  9. EOL = LineEnd().suppress()
  10. NUM1 = Word('0123456789abcdefABCDEF', exact=1)
  11. NUM2 = Word('0123456789abcdefABCDEF', exact=2)
  12. NUM3 = Word('0123456789abcdefABCDEF', exact=3)
  13. NUM4 = Word('0123456789abcdefABCDEF', exact=4)
  14. NUM6 = Word('0123456789abcdefABCDEF', exact=6)
  15. TAB = White('\t', exact=1).suppress()
  16. COMMENTLINE = pythonStyleComment + EOL
  17. EMPTYLINE = LineEnd()
  18. text_eol = lambda name: Regex(r'[^\n]+')(name) + EOL
  19. def klass_grammar():
  20. klass_line = Literal('C ').suppress() + NUM2('klass') + text_eol('text')
  21. subclass_line = TAB + NUM2('subclass') + text_eol('text')
  22. protocol_line = TAB + TAB + NUM2('protocol') + text_eol('name')
  23. subclass = (subclass_line('SUBCLASS') -
  24. ZeroOrMore(Group(protocol_line)('PROTOCOLS*')
  25. ^ COMMENTLINE.suppress()))
  26. klass = (klass_line('KLASS') -
  27. ZeroOrMore(Group(subclass)('SUBCLASSES*')
  28. ^ COMMENTLINE.suppress()))
  29. return klass
  30. def usb_ids_grammar():
  31. vendor_line = NUM4('vendor') + text_eol('text')
  32. device_line = TAB + NUM4('device') + text_eol('text')
  33. vendor = (vendor_line('VENDOR') +
  34. ZeroOrMore(Group(device_line)('VENDOR_DEV*') ^ COMMENTLINE.suppress()))
  35. klass = klass_grammar()
  36. other_line = (Literal('AT ') ^ Literal('HID ') ^ Literal('R ')
  37. ^ Literal('PHY ') ^ Literal('BIAS ') ^ Literal('HUT ')
  38. ^ Literal('L ') ^ Literal('VT ') ^ Literal('HCC ')) + text_eol('text')
  39. other_group = (other_line - ZeroOrMore(TAB + text_eol('text')))
  40. commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  41. grammar = OneOrMore(Group(vendor)('VENDORS*')
  42. ^ Group(klass)('CLASSES*')
  43. ^ other_group.suppress() ^ commentgroup) + stringEnd()
  44. grammar.parseWithTabs()
  45. return grammar
  46. def pci_ids_grammar():
  47. vendor_line = NUM4('vendor') + text_eol('text')
  48. device_line = TAB + NUM4('device') + text_eol('text')
  49. subvendor_line = TAB + TAB + NUM4('a') + White(' ') + NUM4('b') + text_eol('name')
  50. device = (device_line('DEVICE') +
  51. ZeroOrMore(Group(subvendor_line)('SUBVENDORS*') ^ COMMENTLINE.suppress()))
  52. vendor = (vendor_line('VENDOR') +
  53. ZeroOrMore(Group(device)('DEVICES*') ^ COMMENTLINE.suppress()))
  54. klass = klass_grammar()
  55. commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  56. grammar = OneOrMore(Group(vendor)('VENDORS*')
  57. ^ Group(klass)('CLASSES*')
  58. ^ commentgroup) + stringEnd()
  59. grammar.parseWithTabs()
  60. return grammar
  61. def sdio_ids_grammar():
  62. vendor_line = NUM4('vendor') + text_eol('text')
  63. device_line = TAB + NUM4('device') + text_eol('text')
  64. vendor = (vendor_line('VENDOR') +
  65. ZeroOrMore(Group(device_line)('DEVICES*') ^ COMMENTLINE.suppress()))
  66. klass = klass_grammar()
  67. commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  68. grammar = OneOrMore(Group(vendor)('VENDORS*')
  69. ^ Group(klass)('CLASSES*')
  70. ^ commentgroup) + stringEnd()
  71. grammar.parseWithTabs()
  72. return grammar
  73. def oui_grammar(type):
  74. prefix_line = (Combine(NUM2 - Suppress('-') - NUM2 - Suppress('-') - NUM2)('prefix')
  75. - Literal('(hex)') - text_eol('text'))
  76. if type == 'small':
  77. vendor_line = (NUM3('start') - '000-' - NUM3('end') - 'FFF'
  78. - Literal('(base 16)') - text_eol('text2'))
  79. elif type == 'medium':
  80. vendor_line = (NUM1('start') - '00000-' - NUM1('end') - 'FFFFF'
  81. - Literal('(base 16)') - text_eol('text2'))
  82. else:
  83. assert type == 'large'
  84. vendor_line = (NUM6('start')
  85. - Literal('(base 16)') - text_eol('text2'))
  86. extra_line = TAB - TAB - TAB - TAB - SkipTo(EOL)
  87. vendor = prefix_line + vendor_line + ZeroOrMore(extra_line) + Optional(EMPTYLINE)
  88. grammar = (Literal('OUI') + text_eol('header')
  89. + text_eol('header') + text_eol('header') + EMPTYLINE
  90. + OneOrMore(Group(vendor)('VENDORS*')) + stringEnd())
  91. grammar.parseWithTabs()
  92. return grammar
  93. def header(file, *sources):
  94. print('''\
  95. # This file is part of systemd.
  96. #
  97. # Data imported from:{}{}'''.format(' ' if len(sources) == 1 else '\n# ',
  98. '\n# '.join(sources)),
  99. file=file)
  100. def add_item(items, key, value):
  101. if key in items:
  102. print(f'Ignoring duplicate entry: {key} = "{items[key]}", "{value}"')
  103. else:
  104. items[key] = value
  105. def usb_vendor_model(p):
  106. items = {}
  107. for vendor_group in p.VENDORS:
  108. vendor = vendor_group.vendor.upper()
  109. text = vendor_group.text.strip()
  110. add_item(items, (vendor,), text)
  111. for vendor_dev in vendor_group.VENDOR_DEV:
  112. device = vendor_dev.device.upper()
  113. text = vendor_dev.text.strip()
  114. add_item(items, (vendor, device), text)
  115. with open('20-usb-vendor-model.hwdb', 'wt') as out:
  116. header(out, 'http://www.linux-usb.org/usb.ids')
  117. for key in sorted(items):
  118. if len(key) == 1:
  119. p, n = 'usb:v{}*', 'VENDOR'
  120. else:
  121. p, n = 'usb:v{}p{}*', 'MODEL',
  122. print('', p.format(*key),
  123. f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
  124. print(f'Wrote {out.name}')
  125. def usb_classes(p):
  126. items = {}
  127. for klass_group in p.CLASSES:
  128. klass = klass_group.klass.upper()
  129. text = klass_group.text.strip()
  130. if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
  131. add_item(items, (klass,), text)
  132. for subclass_group in klass_group.SUBCLASSES:
  133. subclass = subclass_group.subclass.upper()
  134. text = subclass_group.text.strip()
  135. if subclass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
  136. add_item(items, (klass, subclass), text)
  137. for protocol_group in subclass_group.PROTOCOLS:
  138. protocol = protocol_group.protocol.upper()
  139. text = protocol_group.name.strip()
  140. if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
  141. add_item(items, (klass, subclass, protocol), text)
  142. with open('20-usb-classes.hwdb', 'wt') as out:
  143. header(out, 'http://www.linux-usb.org/usb.ids')
  144. for key in sorted(items):
  145. if len(key) == 1:
  146. p, n = 'usb:v*p*d*dc{}*', 'CLASS'
  147. elif len(key) == 2:
  148. p, n = 'usb:v*p*d*dc{}dsc{}*', 'SUBCLASS'
  149. else:
  150. p, n = 'usb:v*p*d*dc{}dsc{}dp{}*', 'PROTOCOL'
  151. print('', p.format(*key),
  152. f' ID_USB_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
  153. print(f'Wrote {out.name}')
  154. def pci_vendor_model(p):
  155. items = {}
  156. for vendor_group in p.VENDORS:
  157. vendor = vendor_group.vendor.upper()
  158. text = vendor_group.text.strip()
  159. add_item(items, (vendor,), text)
  160. for device_group in vendor_group.DEVICES:
  161. device = device_group.device.upper()
  162. text = device_group.text.strip()
  163. add_item(items, (vendor, device), text)
  164. for subvendor_group in device_group.SUBVENDORS:
  165. sub_vendor = subvendor_group.a.upper()
  166. sub_model = subvendor_group.b.upper()
  167. sub_text = subvendor_group.name.strip()
  168. if sub_text.startswith(text):
  169. sub_text = sub_text[len(text):].lstrip()
  170. if sub_text:
  171. sub_text = f' ({sub_text})'
  172. add_item(items, (vendor, device, sub_vendor, sub_model), text + sub_text)
  173. with open('20-pci-vendor-model.hwdb', 'wt') as out:
  174. header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
  175. for key in sorted(items):
  176. if len(key) == 1:
  177. p, n = 'pci:v0000{}*', 'VENDOR'
  178. elif len(key) == 2:
  179. p, n = 'pci:v0000{}d0000{}*', 'MODEL'
  180. else:
  181. p, n = 'pci:v0000{}d0000{}sv0000{}sd0000{}*', 'MODEL'
  182. print('', p.format(*key),
  183. f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
  184. print(f'Wrote {out.name}')
  185. def pci_classes(p):
  186. items = {}
  187. for klass_group in p.CLASSES:
  188. klass = klass_group.klass.upper()
  189. text = klass_group.text.strip()
  190. add_item(items, (klass,), text)
  191. for subclass_group in klass_group.SUBCLASSES:
  192. subclass = subclass_group.subclass.upper()
  193. text = subclass_group.text.strip()
  194. add_item(items, (klass, subclass), text)
  195. for protocol_group in subclass_group.PROTOCOLS:
  196. protocol = protocol_group.protocol.upper()
  197. text = protocol_group.name.strip()
  198. add_item(items, (klass, subclass, protocol), text)
  199. with open('20-pci-classes.hwdb', 'wt') as out:
  200. header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
  201. for key in sorted(items):
  202. if len(key) == 1:
  203. p, n = 'pci:v*d*sv*sd*bc{}*', 'CLASS'
  204. elif len(key) == 2:
  205. p, n = 'pci:v*d*sv*sd*bc{}sc{}*', 'SUBCLASS'
  206. else:
  207. p, n = 'pci:v*d*sv*sd*bc{}sc{}i{}*', 'INTERFACE'
  208. print('', p.format(*key),
  209. f' ID_PCI_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
  210. print(f'Wrote {out.name}')
  211. def sdio_vendor_model(p):
  212. items = {}
  213. for vendor_group in p.VENDORS:
  214. vendor = vendor_group.vendor.upper()
  215. text = vendor_group.text.strip()
  216. add_item(items, (vendor,), text)
  217. for device_group in vendor_group.DEVICES:
  218. device = device_group.device.upper()
  219. text = device_group.text.strip()
  220. add_item(items, (vendor, device), text)
  221. with open('20-sdio-vendor-model.hwdb', 'wt') as out:
  222. header(out, 'hwdb.d/sdio.ids')
  223. for key in sorted(items):
  224. if len(key) == 1:
  225. p, n = 'sdio:c*v{}*', 'VENDOR'
  226. else:
  227. p, n = 'sdio:c*v{}d{}*', 'MODEL'
  228. print('', p.format(*key),
  229. f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
  230. print(f'Wrote {out.name}')
  231. def sdio_classes(p):
  232. items = {}
  233. for klass_group in p.CLASSES:
  234. klass = klass_group.klass.upper()
  235. text = klass_group.text.strip()
  236. add_item(items, klass, text)
  237. with open('20-sdio-classes.hwdb', 'wt') as out:
  238. header(out, 'hwdb.d/sdio.ids')
  239. for klass in sorted(items):
  240. print(f'',
  241. f'sdio:c{klass}v*d*',
  242. f' ID_SDIO_CLASS_FROM_DATABASE={items[klass]}', sep='\n', file=out)
  243. print(f'Wrote {out.name}')
  244. # MAC Address Block Large/Medium/Small
  245. # Large MA-L 24/24 bit (OUI)
  246. # Medium MA-M 28/20 bit (OUI prefix owned by IEEE)
  247. # Small MA-S 36/12 bit (OUI prefix owned by IEEE)
  248. def oui(p1, p2, p3):
  249. prefixes = set()
  250. items = {}
  251. for p, check in ((p1, False), (p2, False), (p3, True)):
  252. for vendor_group in p.VENDORS:
  253. prefix = vendor_group.prefix.upper()
  254. if check:
  255. if prefix in prefixes:
  256. continue
  257. else:
  258. prefixes.add(prefix)
  259. start = vendor_group.start.upper()
  260. end = vendor_group.end.upper()
  261. if end and start != end:
  262. print(f'{prefix:} {start} != {end}', file=sys.stderr)
  263. text = vendor_group.text.strip()
  264. key = prefix + start if end else prefix
  265. add_item(items, key, text)
  266. with open('20-OUI.hwdb', 'wt') as out:
  267. header(out,
  268. 'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-L&format=txt',
  269. 'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-M&format=txt',
  270. 'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-S&format=txt')
  271. for pattern in sorted(items):
  272. print(f'',
  273. f'OUI:{pattern}*',
  274. f' ID_OUI_FROM_DATABASE={items[pattern]}', sep='\n', file=out)
  275. print(f'Wrote {out.name}')
  276. if __name__ == '__main__':
  277. args = sys.argv[1:]
  278. if not args or 'usb' in args:
  279. p = usb_ids_grammar().parseFile(open('usb.ids', errors='replace'))
  280. usb_vendor_model(p)
  281. usb_classes(p)
  282. if not args or 'pci' in args:
  283. p = pci_ids_grammar().parseFile(open('pci.ids', errors='replace'))
  284. pci_vendor_model(p)
  285. pci_classes(p)
  286. if not args or 'sdio' in args:
  287. p = pci_ids_grammar().parseFile(open('sdio.ids', errors='replace'))
  288. sdio_vendor_model(p)
  289. sdio_classes(p)
  290. if not args or 'oui' in args:
  291. p = oui_grammar('small').parseFile(open('ma-small.txt'))
  292. p2 = oui_grammar('medium').parseFile(open('ma-medium.txt'))
  293. p3 = oui_grammar('large').parseFile(open('ma-large.txt'))
  294. oui(p, p2, p3)