A corporation manager and dashboard for EVE Online
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

import_sde.py 11 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Import an EVE Online Static Data Export dump to Calefaction.
  5. """
  6. import gzip
  7. from pathlib import Path
  8. import shutil
  9. import sys
  10. import yaml
  11. _SHIP_CAT = 6
  12. _FIGHTER_CAT = 87
  13. _STRUCT_CATS = [22, 23, 40, 46, 65]
  14. _REGION = 3
  15. _CONSTELLATION = 4
  16. _SOLAR_SYSTEM = 5
  17. def _load_yaml(filename):
  18. with filename.open("rb") as fp:
  19. return yaml.load(fp, Loader=yaml.CLoader)
  20. def _save_yaml(filename, data):
  21. with filename.open("w") as fp:
  22. fp.write(yaml.dump(data, Dumper=yaml.CDumper))
  23. def _verify_categoryids(sde_dir):
  24. print("Verifying categoryIDs... ", end="", flush=True)
  25. data = _load_yaml(sde_dir / "fsd" / "categoryIDs.yaml")
  26. assert data[_SHIP_CAT]["name"]["en"] == "Ship"
  27. print("done.")
  28. def _load_groupids(sde_dir):
  29. print("Loading groupIDs... ", end="", flush=True)
  30. data = _load_yaml(sde_dir / "fsd" / "groupIDs.yaml")
  31. groups = {cid: {} for cid in [_SHIP_CAT, _FIGHTER_CAT] + _STRUCT_CATS}
  32. for gid, group in data.items():
  33. cat = group["categoryID"]
  34. if cat in groups:
  35. name = group["name"]["en"]
  36. assert isinstance(gid, int)
  37. assert isinstance(name, str)
  38. groups[cat][gid] = name
  39. print("done.")
  40. return groups
  41. def _load_typeids(sde_dir, groups):
  42. print("Loading typeIDs... ", end="", flush=True)
  43. data = _load_yaml(sde_dir / "fsd" / "typeIDs.yaml")
  44. assert data[_REGION]["groupID"] == _REGION
  45. assert data[_REGION]["name"]["en"] == "Region"
  46. assert data[_CONSTELLATION]["groupID"] == _CONSTELLATION
  47. assert data[_CONSTELLATION]["name"]["en"] == "Constellation"
  48. assert data[_SOLAR_SYSTEM]["groupID"] == _SOLAR_SYSTEM
  49. assert data[_SOLAR_SYSTEM]["name"]["en"] == "Solar System"
  50. types = {}
  51. killables = {"ships": {}, "structures": {}, "fighters": {}}
  52. cat_conv = {_SHIP_CAT: "ships", _FIGHTER_CAT: "fighters"}
  53. cat_conv.update({cid: "structures" for cid in _STRUCT_CATS})
  54. group_conv = {gid: cid for cid, gids in groups.items() for gid in gids}
  55. for tid, type_ in data.items():
  56. name = type_["name"].get("en", "Unknown")
  57. gid = type_["groupID"]
  58. assert isinstance(tid, int)
  59. assert isinstance(gid, int)
  60. assert tid >= 0
  61. assert gid >= 0
  62. types[tid] = {"name": name, "group_id": gid}
  63. if "marketGroupID" in type_:
  64. mgid = type_["marketGroupID"]
  65. assert isinstance(mgid, int)
  66. assert mgid >= 0
  67. types[tid]["market_group_id"] = mgid
  68. if gid in group_conv:
  69. cid = group_conv[gid]
  70. cname = cat_conv[cid]
  71. group = groups[cid][gid]
  72. killables[cname][tid] = {"name": name, "group": group}
  73. print("done.")
  74. return types, killables
  75. def _load_ids(sde_dir):
  76. print("Loading itemIDs... ", end="", flush=True)
  77. data = _load_yaml(sde_dir / "bsd" / "invItems.yaml")
  78. ids = {_REGION: [], _CONSTELLATION: [], _SOLAR_SYSTEM: []}
  79. for entry in data:
  80. if entry["typeID"] in ids:
  81. ids[entry["typeID"]].append(entry["itemID"])
  82. print("done.")
  83. return ids
  84. def _load_names(sde_dir):
  85. print("Loading itemNames... ", end="", flush=True)
  86. data = _load_yaml(sde_dir / "bsd" / "invNames.yaml")
  87. names = {}
  88. for entry in data:
  89. name = entry["itemName"]
  90. assert isinstance(name, str)
  91. names[entry["itemID"]] = name
  92. print("done.")
  93. return names
  94. def _build_galaxy_skeleton(ids, names):
  95. print("Building galaxy skeleton... ", end="", flush=True)
  96. galaxy = {"regions": {}, "constellations": {}, "systems": {}}
  97. d = galaxy["regions"]
  98. for rid in ids[_REGION]:
  99. assert isinstance(rid, int)
  100. d[rid] = {
  101. "name": names[rid]
  102. }
  103. d = galaxy["constellations"]
  104. for cid in ids[_CONSTELLATION]:
  105. assert isinstance(cid, int)
  106. d[cid] = {
  107. "name": names[cid],
  108. "region": -1
  109. }
  110. d = galaxy["systems"]
  111. for sid in ids[_SOLAR_SYSTEM]:
  112. assert isinstance(sid, int)
  113. d[sid] = {
  114. "name": names[sid],
  115. "constellation": -1,
  116. "region": -1,
  117. "security": 0.0,
  118. "coords": [0.0, 0.0, 0.0],
  119. "gates": []
  120. }
  121. print("done.")
  122. return galaxy
  123. def _load_assoc_for_system(galaxy, system, rid, cid):
  124. data = _load_yaml(system / "solarsystem.staticdata")
  125. sid = data["solarSystemID"]
  126. sec = data["security"]
  127. coords = data["center"]
  128. assert isinstance(sid, int)
  129. assert isinstance(sec, float)
  130. assert sid >= 0
  131. assert sec >= -1.0 and sec <= 1.0
  132. assert len(coords) == 3 and all(isinstance(val, float) for val in coords)
  133. galaxy["systems"][sid]["constellation"] = cid
  134. galaxy["systems"][sid]["region"] = rid
  135. galaxy["systems"][sid]["security"] = sec
  136. galaxy["systems"][sid]["coords"] = coords
  137. if "factionID" in data:
  138. facid = data["factionID"]
  139. assert isinstance(facid, int)
  140. assert facid >= 0
  141. galaxy["systems"][sid]["faction"] = facid
  142. galaxy["systems->stargates"][sid] = []
  143. for sgid, gate in data["stargates"].items():
  144. dest = gate["destination"]
  145. assert isinstance(sgid, int)
  146. assert isinstance(dest, int)
  147. assert sgid >= 0
  148. assert dest >= 0
  149. galaxy["systems->stargates"][sid].append(dest)
  150. galaxy["stargates->systems"][sgid] = sid
  151. def _load_assoc_for_constellation(galaxy, constellation, rid):
  152. data = _load_yaml(constellation / "constellation.staticdata")
  153. cid = data["constellationID"]
  154. assert isinstance(cid, int)
  155. assert cid >= 0
  156. galaxy["constellations"][cid]["region"] = rid
  157. if "factionID" in data:
  158. facid = data["factionID"]
  159. assert isinstance(facid, int)
  160. assert facid >= 0
  161. galaxy["constellations"][cid]["faction"] = facid
  162. for system in constellation.iterdir():
  163. if not system.is_dir():
  164. continue
  165. _load_assoc_for_system(galaxy, system, rid, cid)
  166. def _load_assoc_for_region(galaxy, region):
  167. data = _load_yaml(region / "region.staticdata")
  168. rid = data["regionID"]
  169. assert isinstance(rid, int)
  170. assert rid >= 0
  171. if "factionID" in data:
  172. facid = data["factionID"]
  173. assert isinstance(facid, int)
  174. assert facid >= 0
  175. galaxy["regions"][rid]["faction"] = facid
  176. for constellation in region.iterdir():
  177. if not constellation.is_dir():
  178. continue
  179. _load_assoc_for_constellation(galaxy, constellation, rid)
  180. def _load_gate_info(galaxy):
  181. sys2gate = galaxy["systems->stargates"]
  182. gate2sys = galaxy["stargates->systems"]
  183. for sid, gates in sys2gate.items():
  184. galaxy["systems"][sid]["gates"] = [gate2sys[gate] for gate in gates]
  185. def _load_galaxy_associations(sde_dir, galaxy):
  186. print("Loading galaxy staticdata... ", end="", flush=True)
  187. galaxy["systems->stargates"] = {}
  188. galaxy["stargates->systems"] = {}
  189. univdir = sde_dir / "fsd" / "universe"
  190. for base in univdir.iterdir():
  191. if not base.is_dir():
  192. continue
  193. for region in base.iterdir():
  194. if not region.is_dir():
  195. continue
  196. _load_assoc_for_region(galaxy, region)
  197. _load_gate_info(galaxy)
  198. del galaxy["systems->stargates"]
  199. del galaxy["stargates->systems"]
  200. for cid, constellation in galaxy["constellations"].items():
  201. if constellation["region"] < 0:
  202. print("[WARNING] Orphaned constellation: %d=%s" % (
  203. cid, constellation["name"]))
  204. for sid, system in galaxy["systems"].items():
  205. if system["region"] < 0 or system["constellation"] < 0:
  206. print("[WARNING] Orphaned system: %d=%s" % (sid, system["name"]))
  207. print("done.")
  208. def _load_factions(sde_dir):
  209. print("Loading factions... ", end="", flush=True)
  210. data = _load_yaml(sde_dir / "bsd" / "chrFactions.yaml")
  211. factions = {}
  212. for entry in data:
  213. fid = entry["factionID"]
  214. name = entry["factionName"]
  215. assert isinstance(fid, int)
  216. assert isinstance(name, str)
  217. assert fid >= 0
  218. factions[fid] = {"name": name}
  219. print("done.")
  220. return factions
  221. def _dump_types(out_dir, types):
  222. print("Dumping types... ", end="", flush=True)
  223. _save_yaml(out_dir / "types.yml", types)
  224. print("done.")
  225. def _dump_killables(out_dir, killables):
  226. print("Dumping killables... ", end="", flush=True)
  227. _save_yaml(out_dir / "killables.yml", killables)
  228. print("done.")
  229. def _dump_galaxy(out_dir, galaxy):
  230. print("Dumping galaxy... ", end="", flush=True)
  231. _save_yaml(out_dir / "galaxy.yml", galaxy)
  232. print("done.")
  233. def _dump_entities(out_dir, factions):
  234. print("Dumping entities... ", end="", flush=True)
  235. entities = {"factions": factions}
  236. _save_yaml(out_dir / "entities.yml", entities)
  237. print("done.")
  238. def _compress(out_dir):
  239. targets = ["types", "killables", "galaxy", "entities"]
  240. for basename in targets:
  241. print("Compressing %s... " % basename, end="", flush=True)
  242. fn_src = out_dir / (basename + ".yml")
  243. fn_dst = out_dir / (basename + ".yml.gz")
  244. with fn_src.open("rb") as f_in:
  245. with gzip.open(str(fn_dst), "wb") as f_out:
  246. shutil.copyfileobj(f_in, f_out)
  247. print("done.")
  248. def _cleanup(out_dir):
  249. print("Cleaning up... ", end="", flush=True)
  250. targets = ["types", "killables", "galaxy", "entities"]
  251. for basename in targets:
  252. (out_dir / (basename + ".yml")).unlink()
  253. print("done.")
  254. def import_sde(sde_dir, out_dir):
  255. """Import the SDE unzipped at sde_dir to out_dir."""
  256. print("EVE Online static data import")
  257. print("- from: %s" % sde_dir)
  258. print("- to: %s" % out_dir)
  259. _verify_categoryids(sde_dir)
  260. groups = _load_groupids(sde_dir)
  261. types, killables = _load_typeids(sde_dir, groups)
  262. _dump_types(out_dir, types)
  263. _dump_killables(out_dir, killables)
  264. del groups, types, killables
  265. ids = _load_ids(sde_dir)
  266. print("Counts: regions=%d, constellations=%d, systems=%d" % (
  267. len(ids[_REGION]), len(ids[_CONSTELLATION]), len(ids[_SOLAR_SYSTEM])))
  268. names = _load_names(sde_dir)
  269. galaxy = _build_galaxy_skeleton(ids, names)
  270. del ids, names
  271. _load_galaxy_associations(sde_dir, galaxy)
  272. _dump_galaxy(out_dir, galaxy)
  273. del galaxy
  274. factions = _load_factions(sde_dir)
  275. _dump_entities(out_dir, factions)
  276. del factions
  277. _compress(out_dir)
  278. _cleanup(out_dir)
  279. def main():
  280. if len(sys.argv) < 2:
  281. print("usage: %s <sde_directory>" % sys.argv[0])
  282. exit(1)
  283. sde_dir = Path(sys.argv[1]).resolve()
  284. out_dir = Path(__file__).resolve().parent.parent / "data" / "universe"
  285. import_sde(sde_dir, out_dir)
  286. if __name__ == "__main__":
  287. main()