OrderedLoader/OrderedDumper to... preserve order... plus some cleanup.

earwigbot/config/__init__.py

@@ -20,6 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from collections import OrderedDict
 from getpass import getpass
 from hashlib import sha256
 import logging
@@ -44,6 +45,7 @@ except ImportError:
 
 from earwigbot.config.formatter import BotFormatter
 from earwigbot.config.node import ConfigNode
+from earwigbot.config.ordered_yaml import OrderedLoader
 from earwigbot.config.permissions import PermissionsDB
 from earwigbot.config.script import ConfigScript
 from earwigbot.exceptions import NoConfigError
@@ -120,7 +122,7 @@ class BotConfig(object):
         filename = self._config_path
         with open(filename, 'r') as fp:
             try:
-                self._data = yaml.load(fp)
+                self._data = yaml.load(fp, OrderedLoader)
             except yaml.YAMLError:
                 print "Error parsing config file {0}:".format(filename)
                 raise
@@ -270,12 +272,12 @@ class BotConfig(object):
 
         self._load()
         data = self._data
-        self.components._load(data.get("components", {}))
-        self.wiki._load(data.get("wiki", {}))
-        self.irc._load(data.get("irc", {}))
-        self.commands._load(data.get("commands", {}))
-        self.tasks._load(data.get("tasks", {}))
-        self.metadata._load(data.get("metadata", {}))
+        self.components._load(data.get("components", OrderedDict()))
+        self.wiki._load(data.get("wiki", OrderedDict()))
+        self.irc._load(data.get("irc", OrderedDict()))
+        self.commands._load(data.get("commands", OrderedDict()))
+        self.tasks._load(data.get("tasks", OrderedDict()))
+        self.metadata._load(data.get("metadata", OrderedDict()))
 
         self._setup_logging()
         if self.is_encrypted():

earwigbot/config/node.py

@@ -20,11 +20,13 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from collections import OrderedDict
+
 __all__ = ["ConfigNode"]
 
 class ConfigNode(object):
     def __init__(self):
-        self._data = {}
+        self._data = OrderedDict()
 
     def __repr__(self):
         return self._data
@@ -99,4 +101,4 @@ class ConfigNode(object):
         return self._data.itervalues()
 
     def iteritems(self):
-        return self.__dict__.iteritems()
+        return self._data.iteritems()

earwigbot/config/ordered_yaml.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8  -*-
+#
+# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Based on:
+ * https://gist.github.com/844388
+ * http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
+with modifications.
+"""
+
+from collections import OrderedDict
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+__all__ = ["OrderedLoader", "OrderedDumper"]
+
+class OrderedLoader(yaml.Loader):
+    """A YAML loader that loads mappings into ordered dictionaries."""
+
+    def __init__(self, *args, **kwargs):
+        super(OrderedLoader, self).__init__(*args, **kwargs)
+        constructor = type(self).construct_yaml_map
+        self.add_constructor(u"tag:yaml.org,2002:map", constructor)
+        self.add_constructor(u"tag:yaml.org,2002:omap", constructor)
+
+    def construct_yaml_map(self, node):
+        data = OrderedDict()
+        yield data
+        value = self.construct_mapping(node)
+        data.update(value)
+
+    def construct_mapping(self, node, deep=False):
+        if isinstance(node, yaml.MappingNode):
+            self.flatten_mapping(node)
+        else:
+            raise yaml.constructor.ConstructorError(None, None,
+                "expected a mapping node, but found {0}".format(node.id),
+                node.start_mark)
+
+        mapping = OrderedDict()
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            try:
+                hash(key)
+            except TypeError, exc:
+                raise yaml.constructor.ConstructorError(
+                    "while constructing a mapping", node.start_mark,
+                    "found unacceptable key ({0})".format(exc),
+                    key_node.start_mark)
+            value = self.construct_object(value_node, deep=deep)
+            mapping[key] = value
+        return mapping
+
+
+class OrderedDumper(yaml.Dumper):
+    """A YAML dumper that dumps mappings into ordered dictionaries."""
+
+    def __init__(self, *args, **kwargs):
+        super(OrderedDumper, self).__init__(*args, **kwargs)
+        self.add_representer(OrderedDict, type(self).represent_dict)
+
+    def represent_mapping(self, tag, mapping, flow_style=None):
+        value = []
+        node = yaml.MappingNode(tag, value, flow_style=flow_style)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        best_style = True
+        if hasattr(mapping, "items"):
+            mapping = list(mapping.items())
+        for item_key, item_value in mapping:
+            node_key = self.represent_data(item_key)
+            node_value = self.represent_data(item_value)
+            if not (isinstance(node_key, yaml.ScalarNode) and not
+                    node_key.style):
+                best_style = False
+            if not (isinstance(node_value, yaml.ScalarNode) and not
+                    node_value.style):
+                best_style = False
+            value.append((node_key, node_value))
+        if flow_style is None:
+            if self.default_flow_style is not None:
+                node.flow_style = self.default_flow_style
+            else:
+                node.flow_style = best_style
+        return node

earwigbot/config/script.py

@@ -38,6 +38,7 @@ except ImportError:
     yaml = None
 
 from earwigbot import exceptions
+from earwigbot.config.ordered_yaml import OrderedDumper
 
 __all__ = ["ConfigScript"]
 
@@ -272,9 +273,10 @@ class ConfigScript(object):
                            'unaffiliated/nickname'.""")
             host = self._ask("Your hostname on the IRC frontend:")
             if host:
-                self.config._permissions.load()
-                self.config._permissions.add_owner(host=host)
-                self.config._permissions.add_admin(host=host)
+                permdb = self.config._permissions
+                permdb.load()
+                permdb.add_owner(host=host)
+                permdb.add_admin(host=host)
         else:
             frontend = {}
 
@@ -360,8 +362,8 @@ class ConfigScript(object):
         self._pause()
 
     def _save(self):
-        with open(self.config.path, "w") as stream:
-            yaml.dump(self.data, stream=stream, default_flow_style=False)
+        with open(self.config.path, "w") as strm:
+            yaml.dump(self.data, strm, OrderedDumper, default_flow_style=False)
 
     def make_new(self):
         """Make a new config file based on the user's input."""

earwigbot/wiki/sitesdb.py

@@ -20,6 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from collections import OrderedDict
 from cookielib import LWPCookieJar, LoadError
 import errno
 from os import chmod, path
@@ -192,7 +193,7 @@ class SitesDB(object):
         maxlag = config.wiki.get("maxlag")
         wait_between_queries = config.wiki.get("waitTime", 3)
         logger = self._logger.getChild(name)
-        search_config = config.wiki.get("search", {}).copy()
+        search_config = config.wiki.get("search", OrderedDict()).copy()
 
         if user_agent:
             user_agent = user_agent.replace("$1", __version__)
@@ -204,7 +205,7 @@ class SitesDB(object):
             search_config["exclusions_db"] = self._exclusions_db
 
         if not sql:
-            sql = config.wiki.get("sql", {}).copy()
+            sql = config.wiki.get("sql", OrderedDict()).copy()
             for key, value in sql.iteritems():
                 if isinstance(value, basestring) and "$1" in value:
                     sql[key] = value.replace("$1", name)