A copyright violation detector running on Wikimedia Cloud Services https://tools.wmflabs.org/copyvios/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

179 lines
5.7 KiB

  1. #! /usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. import logging
  4. import os
  5. import shutil
  6. import subprocess
  7. page_src = """#! /usr/bin/env python
  8. # -*- coding: utf-8 -*-
  9. import os
  10. import sys
  11. os.chdir("..")
  12. sys.path.insert(0, ".")
  13. try:
  14. from toolserver import preconfig
  15. except ImportError:
  16. pass
  17. from mako.template import Template
  18. from mako.lookup import TemplateLookup
  19. from toolserver.cookies import parse_cookies
  20. def main(environ, start_response):
  21. lookup = TemplateLookup(directories=["{{pages_dir}}"],
  22. input_encoding="utf8")
  23. template = Template(filename="{{src}}", module_directory="{{temp_dir}}",
  24. lookup=lookup, format_exceptions=True)
  25. headers = [("Content-Type", "text/html")]
  26. cookies = parse_cookies(environ)
  27. page = template.render(environ=environ, headers=headers,
  28. cookies=cookies).encode("utf8")
  29. start_response("200 OK", headers)
  30. return [page]
  31. if __name__ == "__main__":
  32. from flup.server.fcgi import WSGIServer
  33. WSGIServer(main).run()
  34. """
  35. rewrite_script_src = """match URL into $ with ^/~earwig/{0}(\?.*?)?$
  36. if matched then
  37. set URL = /~earwig/rewrite/{0}.fcgi$1
  38. goto END
  39. endif
  40. """
  41. class Builder(object):
  42. def __init__(self):
  43. self.build_dir = "www"
  44. self.static_dir = "static"
  45. self.pages_dir = "pages"
  46. self.support_dir = "pages/support"
  47. self.temp_dir = "temp"
  48. self.rs_file = "rewrite.script"
  49. self.root = logging.getLogger("builder")
  50. self.root.addHandler(logging.NullHandler())
  51. self._pages = []
  52. def _enable_logging(self):
  53. handler = logging.StreamHandler()
  54. handler.setFormatter(_LogFormatter())
  55. self.root.addHandler(handler)
  56. self.root.setLevel(logging.DEBUG)
  57. def _replace_file(self, name, program, logger):
  58. logger.debug("{0} {1}".format(program, name))
  59. replacement = subprocess.check_output([program, name])
  60. os.remove(name)
  61. with open(name, "w") as fp:
  62. fp.write(replacement)
  63. def _gen_page(self, page, base):
  64. if not page.endswith(".mako"):
  65. base.warn("Skipping {0} (not endswith('.mako'))".format(page))
  66. return
  67. logger = base.getChild(page.rsplit(".", 1)[0])
  68. self._pages.append(page.rsplit(".", 1)[0])
  69. src = os.path.join(self.pages_dir, page)
  70. dest = os.path.join(self.build_dir, page.replace(".mako", ".fcgi"))
  71. logger.debug("build {0} -> {1}".format(src, dest))
  72. content = page_src.replace("{{src}}", src)
  73. content = content.replace("{{pages_dir}}", self.pages_dir)
  74. content = content.replace("{{temp_dir}}", self.temp_dir)
  75. with open(dest, "w") as fp:
  76. fp.write(content)
  77. logger.debug("chmod 0755 {0}".format(dest))
  78. os.chmod(dest, 0755)
  79. def clean(self):
  80. logger = self.root.getChild("clean")
  81. targets = (self.build_dir, self.temp_dir)
  82. for target in targets:
  83. if os.path.exists(target):
  84. logger.debug("rm -r {0}".format(target))
  85. shutil.rmtree(target)
  86. logger.debug("mkdir {0}".format(target))
  87. os.mkdir(target)
  88. def gen_static(self):
  89. logger = self.root.getChild("static")
  90. dest = os.path.join(self.build_dir, "static")
  91. logger.debug("copytree {0} -> {1}".format(self.static_dir, dest))
  92. shutil.copytree(self.static_dir, dest)
  93. for dirpath, dirnames, filenames in os.walk(dest):
  94. for filename in filenames:
  95. name = os.path.join(dirpath, filename)
  96. if name.endswith(".js"):
  97. self._replace_file(name, "uglifyjs", logger)
  98. elif name.endswith(".css"):
  99. self._replace_file(name, "uglifycss", logger)
  100. def gen_pages(self):
  101. logger = self.root.getChild("pages")
  102. pages = os.listdir(self.pages_dir)
  103. for page in pages:
  104. if not os.path.isfile(os.path.join(self.pages_dir, page)):
  105. continue
  106. self._gen_page(page, logger)
  107. def gen_zws(self):
  108. logger = self.root.getChild("zws")
  109. target = self.rs_file
  110. if os.path.exists(target):
  111. logger.debug("rm {0}".format(target))
  112. os.remove(target)
  113. logger.debug("build rewrite.script")
  114. with open(target, "w") as fp:
  115. for page in self._pages:
  116. fp.write(rewrite_script_src.format(page))
  117. def build(self):
  118. self._enable_logging()
  119. self.root.info("Building project...")
  120. self.clean()
  121. self.gen_static()
  122. self.gen_pages()
  123. self.gen_zws()
  124. self.root.info("Done!")
  125. class _LogFormatter(logging.Formatter):
  126. def __init__(self):
  127. fmt = "[%(asctime)s %(lvl)s] %(name)s: %(message)s"
  128. datefmt = "%Y-%m-%d %H:%M:%S"
  129. _format = super(_LogFormatter, self).format
  130. self.format = lambda record: _format(self.colorize(record))
  131. super(_LogFormatter, self).__init__(fmt=fmt, datefmt=datefmt)
  132. def colorize(self, record):
  133. l = record.levelname.ljust(8)
  134. if record.levelno == logging.DEBUG:
  135. record.lvl = l.join(("\x1b[34m", "\x1b[0m")) # Blue
  136. if record.levelno == logging.INFO:
  137. record.lvl = l.join(("\x1b[32m", "\x1b[0m")) # Green
  138. if record.levelno == logging.WARNING:
  139. record.lvl = l.join(("\x1b[33m", "\x1b[0m")) # Yellow
  140. if record.levelno == logging.ERROR:
  141. record.lvl = l.join(("\x1b[31m", "\x1b[0m")) # Red
  142. if record.levelno == logging.CRITICAL:
  143. record.lvl = l.join(("\x1b[1m\x1b[31m", "\x1b[0m")) # Bold red
  144. return record
  145. if __name__ == "__main__":
  146. Builder().build()