A corporation manager and dashboard for EVE Online
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

import_sde.py 9.6 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Import an EVE Online Static Data Export dump to Calefaction.
  5. """
  6. import gzip
  7. from pathlib import Path
  8. import shutil
  9. import sys
  10. import yaml
  11. _SHIP_CAT = 6
  12. _FIGHTER_CAT = 87
  13. _STRUCT_CATS = [22, 23, 40, 46, 65]
  14. _REGION = 3
  15. _CONSTELLATION = 4
  16. _SOLAR_SYSTEM = 5
  17. def _load_yaml(filename):
  18. with filename.open("rb") as fp:
  19. return yaml.load(fp, Loader=yaml.CLoader)
  20. def _save_yaml(filename, data):
  21. with filename.open("w") as fp:
  22. fp.write(yaml.dump(data, Dumper=yaml.CDumper))
  23. def _verify_categoryids(sde_dir):
  24. print("Verifying categoryIDs... ", end="", flush=True)
  25. data = _load_yaml(sde_dir / "fsd" / "categoryIDs.yaml")
  26. assert data[_SHIP_CAT]["name"]["en"] == "Ship"
  27. print("done.")
  28. def _load_groupids(sde_dir):
  29. print("Loading groupIDs... ", end="", flush=True)
  30. data = _load_yaml(sde_dir / "fsd" / "groupIDs.yaml")
  31. groups = {cid: {} for cid in [_SHIP_CAT, _FIGHTER_CAT] + _STRUCT_CATS}
  32. for gid, group in data.items():
  33. cat = group["categoryID"]
  34. if cat in groups:
  35. name = group["name"]["en"]
  36. assert isinstance(gid, int)
  37. assert isinstance(name, str)
  38. groups[cat][gid] = name
  39. print("done.")
  40. return groups
  41. def _load_typeids(sde_dir, groups):
  42. print("Loading typeIDs... ", end="", flush=True)
  43. data = _load_yaml(sde_dir / "fsd" / "typeIDs.yaml")
  44. assert data[_REGION]["groupID"] == _REGION
  45. assert data[_REGION]["name"]["en"] == "Region"
  46. assert data[_CONSTELLATION]["groupID"] == _CONSTELLATION
  47. assert data[_CONSTELLATION]["name"]["en"] == "Constellation"
  48. assert data[_SOLAR_SYSTEM]["groupID"] == _SOLAR_SYSTEM
  49. assert data[_SOLAR_SYSTEM]["name"]["en"] == "Solar System"
  50. types = {}
  51. killables = {"ships": {}, "structures": {}, "fighters": {}}
  52. cat_conv = {_SHIP_CAT: "ships", _FIGHTER_CAT: "fighters"}
  53. cat_conv.update({cid: "structures" for cid in _STRUCT_CATS})
  54. group_conv = {gid: cid for cid, gids in groups.items() for gid in gids}
  55. for tid, type_ in data.items():
  56. name = type_["name"].get("en", "Unknown")
  57. gid = type_["groupID"]
  58. assert isinstance(tid, int)
  59. assert isinstance(gid, int)
  60. assert tid >= 0
  61. assert gid >= 0
  62. types[tid] = {"name": name, "group_id": gid}
  63. if "marketGroupID" in type_:
  64. mgid = type_["marketGroupID"]
  65. assert isinstance(mgid, int)
  66. assert mgid >= 0
  67. types[tid]["market_group_id"] = mgid
  68. if gid in group_conv:
  69. cid = group_conv[gid]
  70. cname = cat_conv[cid]
  71. group = groups[cid][gid]
  72. killables[cname][tid] = {"name": name, "group": group}
  73. print("done.")
  74. return types, killables
  75. def _load_ids(sde_dir):
  76. print("Loading itemIDs... ", end="", flush=True)
  77. data = _load_yaml(sde_dir / "bsd" / "invItems.yaml")
  78. ids = {_REGION: [], _CONSTELLATION: [], _SOLAR_SYSTEM: []}
  79. for entry in data:
  80. if entry["typeID"] in ids:
  81. ids[entry["typeID"]].append(entry["itemID"])
  82. print("done.")
  83. return ids
  84. def _load_names(sde_dir):
  85. print("Loading itemNames... ", end="", flush=True)
  86. data = _load_yaml(sde_dir / "bsd" / "invNames.yaml")
  87. names = {}
  88. for entry in data:
  89. name = entry["itemName"]
  90. assert isinstance(name, str)
  91. names[entry["itemID"]] = name
  92. print("done.")
  93. return names
  94. def _build_galaxy_skeleton(ids, names):
  95. print("Building galaxy skeleton... ", end="", flush=True)
  96. galaxy = {"regions": {}, "constellations": {}, "systems": {}}
  97. d = galaxy["regions"]
  98. for rid in ids[_REGION]:
  99. assert isinstance(rid, int)
  100. d[rid] = {
  101. "name": names[rid]
  102. }
  103. d = galaxy["constellations"]
  104. for cid in ids[_CONSTELLATION]:
  105. assert isinstance(cid, int)
  106. d[cid] = {
  107. "name": names[cid],
  108. "region": -1
  109. }
  110. d = galaxy["systems"]
  111. for sid in ids[_SOLAR_SYSTEM]:
  112. assert isinstance(sid, int)
  113. d[sid] = {
  114. "name": names[sid],
  115. "constellation": -1, "region": -1, "security": 0.0
  116. }
  117. print("done.")
  118. return galaxy
  119. def _load_assoc_for_system(galaxy, system, rid, cid):
  120. data = _load_yaml(system / "solarsystem.staticdata")
  121. sid = data["solarSystemID"]
  122. sec = data["security"]
  123. assert isinstance(sid, int)
  124. assert isinstance(sec, float)
  125. assert sid >= 0
  126. assert sec >= -1.0 and sec <= 1.0
  127. galaxy["systems"][sid]["constellation"] = cid
  128. galaxy["systems"][sid]["region"] = rid
  129. galaxy["systems"][sid]["security"] = sec
  130. if "factionID" in data:
  131. facid = data["factionID"]
  132. assert isinstance(facid, int)
  133. assert facid >= 0
  134. galaxy["systems"][sid]["faction"] = facid
  135. def _load_assoc_for_constellation(galaxy, constellation, rid):
  136. data = _load_yaml(constellation / "constellation.staticdata")
  137. cid = data["constellationID"]
  138. assert isinstance(cid, int)
  139. assert cid >= 0
  140. galaxy["constellations"][cid]["region"] = rid
  141. if "factionID" in data:
  142. facid = data["factionID"]
  143. assert isinstance(facid, int)
  144. assert facid >= 0
  145. galaxy["constellations"][cid]["faction"] = facid
  146. for system in constellation.iterdir():
  147. if not system.is_dir():
  148. continue
  149. _load_assoc_for_system(galaxy, system, rid, cid)
  150. def _load_assoc_for_region(galaxy, region):
  151. data = _load_yaml(region / "region.staticdata")
  152. rid = data["regionID"]
  153. assert isinstance(rid, int)
  154. assert rid >= 0
  155. if "factionID" in data:
  156. facid = data["factionID"]
  157. assert isinstance(facid, int)
  158. assert facid >= 0
  159. galaxy["regions"][rid]["faction"] = facid
  160. for constellation in region.iterdir():
  161. if not constellation.is_dir():
  162. continue
  163. _load_assoc_for_constellation(galaxy, constellation, rid)
  164. def _load_galaxy_associations(sde_dir, galaxy):
  165. print("Loading galaxy staticdata... ", end="", flush=True)
  166. univdir = sde_dir / "fsd" / "universe"
  167. for base in univdir.iterdir():
  168. if not base.is_dir():
  169. continue
  170. for region in base.iterdir():
  171. if not region.is_dir():
  172. continue
  173. _load_assoc_for_region(galaxy, region)
  174. print("done.")
  175. for cid, constellation in galaxy["constellations"].items():
  176. if constellation["region"] < 0:
  177. print("[WARNING] Orphaned constellation: %d=%s" % (
  178. cid, constellation["name"]))
  179. for sid, system in galaxy["systems"].items():
  180. if system["region"] < 0 or system["constellation"] < 0:
  181. print("[WARNING] Orphaned system: %d=%s" % (sid, system["name"]))
  182. def _load_factions(sde_dir):
  183. print("Loading factions... ", end="", flush=True)
  184. data = _load_yaml(sde_dir / "bsd" / "chrFactions.yaml")
  185. factions = {}
  186. for entry in data:
  187. fid = entry["factionID"]
  188. name = entry["factionName"]
  189. assert isinstance(fid, int)
  190. assert isinstance(name, str)
  191. assert fid >= 0
  192. factions[fid] = {"name": name}
  193. print("done.")
  194. return factions
  195. def _dump_types(out_dir, types):
  196. print("Dumping types... ", end="", flush=True)
  197. _save_yaml(out_dir / "types.yml", types)
  198. print("done.")
  199. def _dump_killables(out_dir, killables):
  200. print("Dumping killables... ", end="", flush=True)
  201. _save_yaml(out_dir / "killables.yml", killables)
  202. print("done.")
  203. def _dump_galaxy(out_dir, galaxy):
  204. print("Dumping galaxy... ", end="", flush=True)
  205. _save_yaml(out_dir / "galaxy.yml", galaxy)
  206. print("done.")
  207. def _dump_entities(out_dir, factions):
  208. print("Dumping entities... ", end="", flush=True)
  209. entities = {"factions": factions}
  210. _save_yaml(out_dir / "entities.yml", entities)
  211. print("done.")
  212. def _compress(out_dir):
  213. targets = ["types", "killables", "galaxy", "entities"]
  214. for basename in targets:
  215. print("Compressing %s... " % basename, end="", flush=True)
  216. fn_src = out_dir / (basename + ".yml")
  217. fn_dst = out_dir / (basename + ".yml.gz")
  218. with fn_src.open("rb") as f_in:
  219. with gzip.open(str(fn_dst), "wb") as f_out:
  220. shutil.copyfileobj(f_in, f_out)
  221. print("done.")
  222. def _cleanup(out_dir):
  223. print("Cleaning up... ", end="", flush=True)
  224. targets = ["types", "killables", "galaxy", "entities"]
  225. for basename in targets:
  226. (out_dir / (basename + ".yml")).unlink()
  227. print("done.")
  228. def import_sde(sde_dir, out_dir):
  229. """Import the SDE unzipped at sde_dir to out_dir."""
  230. print("EVE Online static data import")
  231. print("- from: %s" % sde_dir)
  232. print("- to: %s" % out_dir)
  233. _verify_categoryids(sde_dir)
  234. groups = _load_groupids(sde_dir)
  235. types, killables = _load_typeids(sde_dir, groups)
  236. _dump_types(out_dir, types)
  237. _dump_killables(out_dir, killables)
  238. del groups, types, killables
  239. ids = _load_ids(sde_dir)
  240. print("Counts: regions=%d, constellations=%d, systems=%d" % (
  241. len(ids[_REGION]), len(ids[_CONSTELLATION]), len(ids[_SOLAR_SYSTEM])))
  242. names = _load_names(sde_dir)
  243. galaxy = _build_galaxy_skeleton(ids, names)
  244. del ids, names
  245. _load_galaxy_associations(sde_dir, galaxy)
  246. _dump_galaxy(out_dir, galaxy)
  247. del galaxy
  248. factions = _load_factions(sde_dir)
  249. _dump_entities(out_dir, factions)
  250. del factions
  251. _compress(out_dir)
  252. _cleanup(out_dir)
  253. def main():
  254. if len(sys.argv) < 2:
  255. print("usage: %s <sde_directory>" % sys.argv[0])
  256. exit(1)
  257. sde_dir = Path(sys.argv[1]).resolve()
  258. out_dir = Path(__file__).resolve().parent.parent / "data" / "universe"
  259. import_sde(sde_dir, out_dir)
  260. if __name__ == "__main__":
  261. main()