A corporation manager and dashboard for EVE Online
No puede seleccionar más de 25 temas Los temas deben comenzar con una letra o número, pueden incluir guiones ('-') y pueden tener hasta 35 caracteres de largo.

import_sde.py 9.6 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Import an EVE Online Static Data Export dump to Calefaction.
  5. """
  6. import gzip
  7. from pathlib import Path
  8. import shutil
  9. import sys
  10. import yaml
  11. _SHIP_CAT = 6
  12. _FIGHTER_CAT = 87
  13. _STRUCT_CATS = [22, 23, 40, 46, 65]
  14. _REGION = 3
  15. _CONSTELLATION = 4
  16. _SOLAR_SYSTEM = 5
  17. def _load_yaml(filename):
  18. with filename.open("rb") as fp:
  19. return yaml.load(fp, Loader=yaml.CLoader)
  20. def _save_yaml(filename, data):
  21. with filename.open("w") as fp:
  22. fp.write(yaml.dump(data, Dumper=yaml.CDumper))
  23. def _verify_categoryids(sde_dir):
  24. print("Verifying categoryIDs... ", end="", flush=True)
  25. data = _load_yaml(sde_dir / "fsd" / "categoryIDs.yaml")
  26. assert data[_SHIP_CAT]["name"]["en"] == "Ship"
  27. print("done.")
  28. def _load_groupids(sde_dir):
  29. print("Loading groupIDs... ", end="", flush=True)
  30. data = _load_yaml(sde_dir / "fsd" / "groupIDs.yaml")
  31. groups = {cid: {} for cid in [_SHIP_CAT, _FIGHTER_CAT] + _STRUCT_CATS}
  32. for gid, group in data.items():
  33. cat = group["categoryID"]
  34. if cat in groups:
  35. name = group["name"]["en"]
  36. assert isinstance(gid, int)
  37. assert isinstance(name, str)
  38. groups[cat][gid] = name
  39. print("done.")
  40. return groups
  41. def _load_typeids(sde_dir, groups):
  42. print("Loading typeIDs... ", end="", flush=True)
  43. data = _load_yaml(sde_dir / "fsd" / "typeIDs.yaml")
  44. assert data[_REGION]["groupID"] == _REGION
  45. assert data[_REGION]["name"]["en"] == "Region"
  46. assert data[_CONSTELLATION]["groupID"] == _CONSTELLATION
  47. assert data[_CONSTELLATION]["name"]["en"] == "Constellation"
  48. assert data[_SOLAR_SYSTEM]["groupID"] == _SOLAR_SYSTEM
  49. assert data[_SOLAR_SYSTEM]["name"]["en"] == "Solar System"
  50. types = {}
  51. killables = {"ships": {}, "structures": {}, "fighters": {}}
  52. cat_conv = {_SHIP_CAT: "ships", _FIGHTER_CAT: "fighters"}
  53. cat_conv.update({cid: "structures" for cid in _STRUCT_CATS})
  54. group_conv = {gid: cid for cid, gids in groups.items() for gid in gids}
  55. for tid, type_ in data.items():
  56. name = type_["name"].get("en", "Unknown")
  57. gid = type_["groupID"]
  58. assert isinstance(tid, int)
  59. assert isinstance(gid, int)
  60. assert tid >= 0
  61. assert gid >= 0
  62. types[tid] = {"name": name, "group_id": gid}
  63. if "marketGroupID" in type_:
  64. mgid = type_["marketGroupID"]
  65. assert isinstance(mgid, int)
  66. assert mgid >= 0
  67. types[tid]["market_group_id"] = mgid
  68. if gid in group_conv:
  69. cid = group_conv[gid]
  70. cname = cat_conv[cid]
  71. group = groups[cid][gid]
  72. killables[cname][tid] = {"name": name, "group": group}
  73. print("done.")
  74. return types, killables
  75. def _load_ids(sde_dir):
  76. print("Loading itemIDs... ", end="", flush=True)
  77. data = _load_yaml(sde_dir / "bsd" / "invItems.yaml")
  78. ids = {_REGION: [], _CONSTELLATION: [], _SOLAR_SYSTEM: []}
  79. for entry in data:
  80. if entry["typeID"] in ids:
  81. ids[entry["typeID"]].append(entry["itemID"])
  82. print("done.")
  83. return ids
  84. def _load_names(sde_dir):
  85. print("Loading itemNames... ", end="", flush=True)
  86. data = _load_yaml(sde_dir / "bsd" / "invNames.yaml")
  87. names = {}
  88. for entry in data:
  89. name = entry["itemName"]
  90. assert isinstance(name, str)
  91. names[entry["itemID"]] = name
  92. print("done.")
  93. return names
  94. def _build_galaxy_skeleton(ids, names):
  95. print("Building galaxy skeleton... ", end="", flush=True)
  96. galaxy = {"regions": {}, "constellations": {}, "systems": {}}
  97. d = galaxy["regions"]
  98. for rid in ids[_REGION]:
  99. assert isinstance(rid, int)
  100. d[rid] = {
  101. "name": names[rid]
  102. }
  103. d = galaxy["constellations"]
  104. for cid in ids[_CONSTELLATION]:
  105. assert isinstance(cid, int)
  106. d[cid] = {
  107. "name": names[cid],
  108. "region": -1
  109. }
  110. d = galaxy["systems"]
  111. for sid in ids[_SOLAR_SYSTEM]:
  112. assert isinstance(sid, int)
  113. d[sid] = {
  114. "name": names[sid],
  115. "constellation": -1, "region": -1, "security": 0.0
  116. }
  117. print("done.")
  118. return galaxy
  119. def _load_assoc_for_system(galaxy, system, rid, cid):
  120. data = _load_yaml(system / "solarsystem.staticdata")
  121. sid = data["solarSystemID"]
  122. sec = data["security"]
  123. assert isinstance(sid, int)
  124. assert isinstance(sec, float)
  125. assert sid >= 0
  126. assert sec >= -1.0 and sec <= 1.0
  127. galaxy["systems"][sid]["constellation"] = cid
  128. galaxy["systems"][sid]["region"] = rid
  129. galaxy["systems"][sid]["security"] = sec
  130. if "factionID" in data:
  131. facid = data["factionID"]
  132. assert isinstance(facid, int)
  133. assert facid >= 0
  134. galaxy["systems"][sid]["faction"] = facid
  135. def _load_assoc_for_constellation(galaxy, constellation, rid):
  136. data = _load_yaml(constellation / "constellation.staticdata")
  137. cid = data["constellationID"]
  138. assert isinstance(cid, int)
  139. assert cid >= 0
  140. galaxy["constellations"][cid]["region"] = rid
  141. if "factionID" in data:
  142. facid = data["factionID"]
  143. assert isinstance(facid, int)
  144. assert facid >= 0
  145. galaxy["constellations"][cid]["faction"] = facid
  146. for system in constellation.iterdir():
  147. if not system.is_dir():
  148. continue
  149. _load_assoc_for_system(galaxy, system, rid, cid)
  150. def _load_assoc_for_region(galaxy, region):
  151. data = _load_yaml(region / "region.staticdata")
  152. rid = data["regionID"]
  153. assert isinstance(rid, int)
  154. assert rid >= 0
  155. if "factionID" in data:
  156. facid = data["factionID"]
  157. assert isinstance(facid, int)
  158. assert facid >= 0
  159. galaxy["regions"][rid]["faction"] = facid
  160. for constellation in region.iterdir():
  161. if not constellation.is_dir():
  162. continue
  163. _load_assoc_for_constellation(galaxy, constellation, rid)
  164. def _load_galaxy_associations(sde_dir, galaxy):
  165. print("Loading galaxy staticdata... ", end="", flush=True)
  166. univdir = sde_dir / "fsd" / "universe"
  167. for base in univdir.iterdir():
  168. if not base.is_dir():
  169. continue
  170. for region in base.iterdir():
  171. if not region.is_dir():
  172. continue
  173. _load_assoc_for_region(galaxy, region)
  174. print("done.")
  175. for cid, constellation in galaxy["constellations"].items():
  176. if constellation["region"] < 0:
  177. print("[WARNING] Orphaned constellation: %d=%s" % (
  178. cid, constellation["name"]))
  179. for sid, system in galaxy["systems"].items():
  180. if system["region"] < 0 or system["constellation"] < 0:
  181. print("[WARNING] Orphaned system: %d=%s" % (sid, system["name"]))
  182. def _load_factions(sde_dir):
  183. print("Loading factions... ", end="", flush=True)
  184. data = _load_yaml(sde_dir / "bsd" / "chrFactions.yaml")
  185. factions = {}
  186. for entry in data:
  187. fid = entry["factionID"]
  188. name = entry["factionName"]
  189. assert isinstance(fid, int)
  190. assert isinstance(name, str)
  191. assert fid >= 0
  192. factions[fid] = {"name": name}
  193. print("done.")
  194. return factions
  195. def _dump_types(out_dir, types):
  196. print("Dumping types... ", end="", flush=True)
  197. _save_yaml(out_dir / "types.yml", types)
  198. print("done.")
  199. def _dump_killables(out_dir, killables):
  200. print("Dumping killables... ", end="", flush=True)
  201. _save_yaml(out_dir / "killables.yml", killables)
  202. print("done.")
  203. def _dump_galaxy(out_dir, galaxy):
  204. print("Dumping galaxy... ", end="", flush=True)
  205. _save_yaml(out_dir / "galaxy.yml", galaxy)
  206. print("done.")
  207. def _dump_entities(out_dir, factions):
  208. print("Dumping entities... ", end="", flush=True)
  209. entities = {"factions": factions}
  210. _save_yaml(out_dir / "entities.yml", entities)
  211. print("done.")
  212. def _compress(out_dir):
  213. targets = ["types", "killables", "galaxy", "entities"]
  214. for basename in targets:
  215. print("Compressing %s... " % basename, end="", flush=True)
  216. fn_src = out_dir / (basename + ".yml")
  217. fn_dst = out_dir / (basename + ".yml.gz")
  218. with fn_src.open("rb") as f_in:
  219. with gzip.open(str(fn_dst), "wb") as f_out:
  220. shutil.copyfileobj(f_in, f_out)
  221. print("done.")
  222. def _cleanup(out_dir):
  223. print("Cleaning up... ", end="", flush=True)
  224. targets = ["types", "killables", "galaxy", "entities"]
  225. for basename in targets:
  226. (out_dir / (basename + ".yml")).unlink()
  227. print("done.")
  228. def import_sde(sde_dir, out_dir):
  229. """Import the SDE unzipped at sde_dir to out_dir."""
  230. print("EVE Online static data import")
  231. print("- from: %s" % sde_dir)
  232. print("- to: %s" % out_dir)
  233. _verify_categoryids(sde_dir)
  234. groups = _load_groupids(sde_dir)
  235. types, killables = _load_typeids(sde_dir, groups)
  236. _dump_types(out_dir, types)
  237. _dump_killables(out_dir, killables)
  238. del groups, types, killables
  239. ids = _load_ids(sde_dir)
  240. print("Counts: regions=%d, constellations=%d, systems=%d" % (
  241. len(ids[_REGION]), len(ids[_CONSTELLATION]), len(ids[_SOLAR_SYSTEM])))
  242. names = _load_names(sde_dir)
  243. galaxy = _build_galaxy_skeleton(ids, names)
  244. del ids, names
  245. _load_galaxy_associations(sde_dir, galaxy)
  246. _dump_galaxy(out_dir, galaxy)
  247. del galaxy
  248. factions = _load_factions(sde_dir)
  249. _dump_entities(out_dir, factions)
  250. del factions
  251. _compress(out_dir)
  252. _cleanup(out_dir)
  253. def main():
  254. if len(sys.argv) < 2:
  255. print("usage: %s <sde_directory>" % sys.argv[0])
  256. exit(1)
  257. sde_dir = Path(sys.argv[1]).resolve()
  258. out_dir = Path(__file__).resolve().parent.parent / "data" / "universe"
  259. import_sde(sde_dir, out_dir)
  260. if __name__ == "__main__":
  261. main()