A corporation manager and dashboard for EVE Online
您最多选择25个主题 主题必须以字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符

import_sde.py 11 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Import an EVE Online Static Data Export dump to Calefaction.
  5. """
  6. import gzip
  7. from pathlib import Path
  8. import shutil
  9. import sys
  10. import yaml
  11. _SHIP_CAT = 6
  12. _FIGHTER_CAT = 87
  13. _STRUCT_CATS = [22, 23, 40, 46, 65]
  14. _REGION = 3
  15. _CONSTELLATION = 4
  16. _SOLAR_SYSTEM = 5
  17. def _load_yaml(filename):
  18. with filename.open("rb") as fp:
  19. return yaml.load(fp, Loader=yaml.CLoader)
  20. def _save_yaml(filename, data):
  21. with filename.open("w") as fp:
  22. fp.write(yaml.dump(data, Dumper=yaml.CDumper))
  23. def _verify_categoryids(sde_dir):
  24. print("Verifying categoryIDs... ", end="", flush=True)
  25. data = _load_yaml(sde_dir / "fsd" / "categoryIDs.yaml")
  26. assert data[_SHIP_CAT]["name"]["en"] == "Ship"
  27. print("done.")
  28. def _load_groupids(sde_dir):
  29. print("Loading groupIDs... ", end="", flush=True)
  30. data = _load_yaml(sde_dir / "fsd" / "groupIDs.yaml")
  31. groups = {cid: {} for cid in [_SHIP_CAT, _FIGHTER_CAT] + _STRUCT_CATS}
  32. for gid, group in data.items():
  33. cat = group["categoryID"]
  34. if cat in groups:
  35. name = group["name"]["en"]
  36. assert isinstance(gid, int)
  37. assert isinstance(name, str)
  38. groups[cat][gid] = name
  39. print("done.")
  40. return groups
  41. def _load_typeids(sde_dir, groups):
  42. print("Loading typeIDs... ", end="", flush=True)
  43. data = _load_yaml(sde_dir / "fsd" / "typeIDs.yaml")
  44. assert data[_REGION]["groupID"] == _REGION
  45. assert data[_REGION]["name"]["en"] == "Region"
  46. assert data[_CONSTELLATION]["groupID"] == _CONSTELLATION
  47. assert data[_CONSTELLATION]["name"]["en"] == "Constellation"
  48. assert data[_SOLAR_SYSTEM]["groupID"] == _SOLAR_SYSTEM
  49. assert data[_SOLAR_SYSTEM]["name"]["en"] == "Solar System"
  50. types = {}
  51. killables = {"ships": {}, "structures": {}, "fighters": {}}
  52. cat_conv = {_SHIP_CAT: "ships", _FIGHTER_CAT: "fighters"}
  53. cat_conv.update({cid: "structures" for cid in _STRUCT_CATS})
  54. group_conv = {gid: cid for cid, gids in groups.items() for gid in gids}
  55. for tid, type_ in data.items():
  56. name = type_["name"].get("en", "Unknown")
  57. gid = type_["groupID"]
  58. assert isinstance(tid, int)
  59. assert isinstance(gid, int)
  60. assert tid >= 0
  61. assert gid >= 0
  62. types[tid] = {"name": name, "group_id": gid}
  63. if "marketGroupID" in type_:
  64. mgid = type_["marketGroupID"]
  65. assert isinstance(mgid, int)
  66. assert mgid >= 0
  67. types[tid]["market_group_id"] = mgid
  68. if gid in group_conv:
  69. cid = group_conv[gid]
  70. cname = cat_conv[cid]
  71. group = groups[cid][gid]
  72. killables[cname][tid] = {"name": name, "group": group}
  73. print("done.")
  74. return types, killables
  75. def _load_ids(sde_dir):
  76. print("Loading itemIDs... ", end="", flush=True)
  77. data = _load_yaml(sde_dir / "bsd" / "invItems.yaml")
  78. ids = {_REGION: [], _CONSTELLATION: [], _SOLAR_SYSTEM: []}
  79. for entry in data:
  80. if entry["typeID"] in ids:
  81. ids[entry["typeID"]].append(entry["itemID"])
  82. print("done.")
  83. return ids
  84. def _load_names(sde_dir):
  85. print("Loading itemNames... ", end="", flush=True)
  86. data = _load_yaml(sde_dir / "bsd" / "invNames.yaml")
  87. names = {}
  88. for entry in data:
  89. name = entry["itemName"]
  90. assert isinstance(name, str)
  91. names[entry["itemID"]] = name
  92. print("done.")
  93. return names
  94. def _build_galaxy_skeleton(ids, names):
  95. print("Building galaxy skeleton... ", end="", flush=True)
  96. galaxy = {"regions": {}, "constellations": {}, "systems": {}}
  97. d = galaxy["regions"]
  98. for rid in ids[_REGION]:
  99. assert isinstance(rid, int)
  100. d[rid] = {
  101. "name": names[rid]
  102. }
  103. d = galaxy["constellations"]
  104. for cid in ids[_CONSTELLATION]:
  105. assert isinstance(cid, int)
  106. d[cid] = {
  107. "name": names[cid],
  108. "region": -1
  109. }
  110. d = galaxy["systems"]
  111. for sid in ids[_SOLAR_SYSTEM]:
  112. assert isinstance(sid, int)
  113. d[sid] = {
  114. "name": names[sid],
  115. "constellation": -1,
  116. "region": -1,
  117. "security": 0.0,
  118. "coords": [0.0, 0.0, 0.0],
  119. "gates": []
  120. }
  121. print("done.")
  122. return galaxy
  123. def _load_assoc_for_system(galaxy, system, rid, cid):
  124. data = _load_yaml(system / "solarsystem.staticdata")
  125. sid = data["solarSystemID"]
  126. sec = data["security"]
  127. coords = data["center"]
  128. assert isinstance(sid, int)
  129. assert isinstance(sec, float)
  130. assert sid >= 0
  131. assert sec >= -1.0 and sec <= 1.0
  132. assert len(coords) == 3 and all(isinstance(val, float) for val in coords)
  133. galaxy["systems"][sid]["constellation"] = cid
  134. galaxy["systems"][sid]["region"] = rid
  135. galaxy["systems"][sid]["security"] = sec
  136. galaxy["systems"][sid]["coords"] = coords
  137. if "factionID" in data:
  138. facid = data["factionID"]
  139. assert isinstance(facid, int)
  140. assert facid >= 0
  141. galaxy["systems"][sid]["faction"] = facid
  142. galaxy["systems->stargates"][sid] = []
  143. for sgid, gate in data["stargates"].items():
  144. dest = gate["destination"]
  145. assert isinstance(sgid, int)
  146. assert isinstance(dest, int)
  147. assert sgid >= 0
  148. assert dest >= 0
  149. galaxy["systems->stargates"][sid].append(dest)
  150. galaxy["stargates->systems"][sgid] = sid
  151. def _load_assoc_for_constellation(galaxy, constellation, rid):
  152. data = _load_yaml(constellation / "constellation.staticdata")
  153. cid = data["constellationID"]
  154. assert isinstance(cid, int)
  155. assert cid >= 0
  156. galaxy["constellations"][cid]["region"] = rid
  157. if "factionID" in data:
  158. facid = data["factionID"]
  159. assert isinstance(facid, int)
  160. assert facid >= 0
  161. galaxy["constellations"][cid]["faction"] = facid
  162. for system in constellation.iterdir():
  163. if not system.is_dir():
  164. continue
  165. _load_assoc_for_system(galaxy, system, rid, cid)
  166. def _load_assoc_for_region(galaxy, region):
  167. data = _load_yaml(region / "region.staticdata")
  168. rid = data["regionID"]
  169. assert isinstance(rid, int)
  170. assert rid >= 0
  171. if "factionID" in data:
  172. facid = data["factionID"]
  173. assert isinstance(facid, int)
  174. assert facid >= 0
  175. galaxy["regions"][rid]["faction"] = facid
  176. for constellation in region.iterdir():
  177. if not constellation.is_dir():
  178. continue
  179. _load_assoc_for_constellation(galaxy, constellation, rid)
  180. def _load_gate_info(galaxy):
  181. sys2gate = galaxy["systems->stargates"]
  182. gate2sys = galaxy["stargates->systems"]
  183. for sid, gates in sys2gate.items():
  184. galaxy["systems"][sid]["gates"] = [gate2sys[gate] for gate in gates]
  185. def _load_galaxy_associations(sde_dir, galaxy):
  186. print("Loading galaxy staticdata... ", end="", flush=True)
  187. galaxy["systems->stargates"] = {}
  188. galaxy["stargates->systems"] = {}
  189. univdir = sde_dir / "fsd" / "universe"
  190. for base in univdir.iterdir():
  191. if not base.is_dir():
  192. continue
  193. for region in base.iterdir():
  194. if not region.is_dir():
  195. continue
  196. _load_assoc_for_region(galaxy, region)
  197. _load_gate_info(galaxy)
  198. del galaxy["systems->stargates"]
  199. del galaxy["stargates->systems"]
  200. for cid, constellation in galaxy["constellations"].items():
  201. if constellation["region"] < 0:
  202. print("[WARNING] Orphaned constellation: %d=%s" % (
  203. cid, constellation["name"]))
  204. for sid, system in galaxy["systems"].items():
  205. if system["region"] < 0 or system["constellation"] < 0:
  206. print("[WARNING] Orphaned system: %d=%s" % (sid, system["name"]))
  207. print("done.")
  208. def _load_factions(sde_dir):
  209. print("Loading factions... ", end="", flush=True)
  210. data = _load_yaml(sde_dir / "bsd" / "chrFactions.yaml")
  211. factions = {}
  212. for entry in data:
  213. fid = entry["factionID"]
  214. name = entry["factionName"]
  215. assert isinstance(fid, int)
  216. assert isinstance(name, str)
  217. assert fid >= 0
  218. factions[fid] = {"name": name}
  219. print("done.")
  220. return factions
  221. def _dump_types(out_dir, types):
  222. print("Dumping types... ", end="", flush=True)
  223. _save_yaml(out_dir / "types.yml", types)
  224. print("done.")
  225. def _dump_killables(out_dir, killables):
  226. print("Dumping killables... ", end="", flush=True)
  227. _save_yaml(out_dir / "killables.yml", killables)
  228. print("done.")
  229. def _dump_galaxy(out_dir, galaxy):
  230. print("Dumping galaxy... ", end="", flush=True)
  231. _save_yaml(out_dir / "galaxy.yml", galaxy)
  232. print("done.")
  233. def _dump_entities(out_dir, factions):
  234. print("Dumping entities... ", end="", flush=True)
  235. entities = {"factions": factions}
  236. _save_yaml(out_dir / "entities.yml", entities)
  237. print("done.")
  238. def _compress(out_dir):
  239. targets = ["types", "killables", "galaxy", "entities"]
  240. for basename in targets:
  241. print("Compressing %s... " % basename, end="", flush=True)
  242. fn_src = out_dir / (basename + ".yml")
  243. fn_dst = out_dir / (basename + ".yml.gz")
  244. with fn_src.open("rb") as f_in:
  245. with gzip.open(str(fn_dst), "wb") as f_out:
  246. shutil.copyfileobj(f_in, f_out)
  247. print("done.")
  248. def _cleanup(out_dir):
  249. print("Cleaning up... ", end="", flush=True)
  250. targets = ["types", "killables", "galaxy", "entities"]
  251. for basename in targets:
  252. (out_dir / (basename + ".yml")).unlink()
  253. print("done.")
  254. def import_sde(sde_dir, out_dir):
  255. """Import the SDE unzipped at sde_dir to out_dir."""
  256. print("EVE Online static data import")
  257. print("- from: %s" % sde_dir)
  258. print("- to: %s" % out_dir)
  259. _verify_categoryids(sde_dir)
  260. groups = _load_groupids(sde_dir)
  261. types, killables = _load_typeids(sde_dir, groups)
  262. _dump_types(out_dir, types)
  263. _dump_killables(out_dir, killables)
  264. del groups, types, killables
  265. ids = _load_ids(sde_dir)
  266. print("Counts: regions=%d, constellations=%d, systems=%d" % (
  267. len(ids[_REGION]), len(ids[_CONSTELLATION]), len(ids[_SOLAR_SYSTEM])))
  268. names = _load_names(sde_dir)
  269. galaxy = _build_galaxy_skeleton(ids, names)
  270. del ids, names
  271. _load_galaxy_associations(sde_dir, galaxy)
  272. _dump_galaxy(out_dir, galaxy)
  273. del galaxy
  274. factions = _load_factions(sde_dir)
  275. _dump_entities(out_dir, factions)
  276. del factions
  277. _compress(out_dir)
  278. _cleanup(out_dir)
  279. def main():
  280. if len(sys.argv) < 2:
  281. print("usage: %s <sde_directory>" % sys.argv[0])
  282. exit(1)
  283. sde_dir = Path(sys.argv[1]).resolve()
  284. out_dir = Path(__file__).resolve().parent.parent / "data" / "universe"
  285. import_sde(sde_dir, out_dir)
  286. if __name__ == "__main__":
  287. main()