From 74ddc5b702d19375a407b6c87fd34f0447ba0fb3 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 29 Jul 2011 02:31:01 -0400 Subject: [PATCH] More work on wikitools, now with improved API queries and login. * Site's api_query() is much smarter. It uses a custom urllib2 URL opener with cookie support and catches URLErrors, raising its own brand new exception (SiteAPIError) when something is wrong. * The opener now uses a custom User-Agent, which is a constant in wiki.tools.constants. * Site instances automatically login via _login(), which accepts a username and password (provided via config by get_site()) and uses two api_query()s and stores the login data as cookies in self._cookiejar. Login data is not preserved between bot restarts yet. Login errors, e.g. a bad password or username, raise the new LoginError. * Site's get_user()'s username argument is now optional. If left blank, will return the current logged-in user, provided by an API query. * Misc cleanup throughout. --- wiki/tools/constants.py | 10 ++++++- wiki/tools/exceptions.py | 8 ++++++ wiki/tools/functions.py | 18 +++++++++--- wiki/tools/site.py | 74 ++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 97 insertions(+), 13 deletions(-) diff --git a/wiki/tools/constants.py b/wiki/tools/constants.py index 76a327d..6397c5d 100644 --- a/wiki/tools/constants.py +++ b/wiki/tools/constants.py @@ -3,9 +3,17 @@ """ EarwigBot's Wiki Toolset: Constants -This module defines some useful constants. +This module defines some useful constants, such as default namespace IDs for +easy lookup and our user agent. + +Import with `from wiki.tools.constants import *`. """ +import platform + +# User agent when making API queries +USER_AGENT = "EarwigBot/0.1-dev (Python/{0}; https://github.com/earwig/earwigbot)".format(platform.python_version()) + # Default namespace IDs NS_MAIN = 0 NS_TALK = 1 diff --git a/wiki/tools/exceptions.py b/wiki/tools/exceptions.py index 3e5eaf2..0620262 100644 --- a/wiki/tools/exceptions.py +++ b/wiki/tools/exceptions.py @@ -13,6 +13,14 @@ class SiteNotFoundError(WikiToolsetError): """A site matching the args given to get_site() could not be found in the config file.""" +class SiteAPIError(WikiToolsetError): + """We couldn't connect to a site's API, perhaps because the server doesn't + exist, our URL is wrong, or they're having temporary problems.""" + +class LoginError(WikiToolsetError): + """An error occured while trying to login. Perhaps the username/password is + incorrect.""" + class NamespaceNotFoundError(WikiToolsetError): """A requested namespace name or namespace ID does not exist.""" diff --git a/wiki/tools/functions.py b/wiki/tools/functions.py index 178a8e2..2618a57 100644 --- a/wiki/tools/functions.py +++ b/wiki/tools/functions.py @@ -67,10 +67,14 @@ def _get_site_object_from_dict(name, d): namespaces = d["namespaces"] except KeyError: namespaces = None + try: + login = (config.wiki["username"], config.wiki["password"]) + except KeyError: + login = (None, None) return Site(name=name, project=project, lang=lang, base_url=base_url, article_path=article_path, script_path=script_path, - sql=(sql_server, sql_db), namespaces=namespaces) + sql=(sql_server, sql_db), namespaces=namespaces, login=login) def get_site(name=None, project=None, lang=None): """Returns a Site instance based on information from our config file. @@ -86,6 +90,10 @@ def get_site(name=None, project=None, lang=None): member of config.wiki["sites"], `s`, for which s["project"] == project and s["lang"] == lang. + We will attempt to login to the site automatically + using config.wiki["username"] and config.wiki["password"] if both are + defined. + Specifying a project without a lang or a lang without a project will raise TypeError. If all three args are specified, `name` will be first tried, then `project` and `lang`. If, with any number of args, a site cannot be @@ -96,7 +104,8 @@ def get_site(name=None, project=None, lang=None): _load_config() # someone specified a project without a lang (or a lang without a project)! - if (project is None and lang is not None) or (project is not None and lang is None): + if (project is None and lang is not None) or (project is not None and + lang is None): e = "Keyword arguments 'lang' and 'project' must be specified together." raise TypeError(e) @@ -120,13 +129,14 @@ def get_site(name=None, project=None, lang=None): try: site = config.wiki["sites"][name] except KeyError: - if project is None: # implies lang is None, i.e., only name was given + if project is None: # implies lang is None, so only name was given e = "Site '{0}' not found in config.".format(name) raise SiteNotFoundError(e) for sitename, site in config.wiki["sites"].items(): if site["project"] == project and site["lang"] == lang: return _get_site_object_from_dict(sitename, site) - e = "Neither site '{0}' nor site '{1}:{2}' found in config.".format(name, project, lang) + e = "Neither site '{0}' nor site '{1}:{2}' found in config." + e.format(name, project, lang) raise SiteNotFoundError(e) else: return _get_site_object_from_dict(name, site) diff --git a/wiki/tools/site.py b/wiki/tools/site.py index 62a2ecc..f4b854f 100644 --- a/wiki/tools/site.py +++ b/wiki/tools/site.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- +from cookielib import CookieJar from json import loads from urllib import urlencode -from urllib2 import urlopen +from urllib2 import build_opener, HTTPCookieProcessor, URLError from wiki.tools.category import Category from wiki.tools.constants import * -from wiki.tools.exceptions import NamespaceNotFoundError +from wiki.tools.exceptions import * from wiki.tools.page import Page from wiki.tools.user import User @@ -17,10 +18,12 @@ class Site(object): def __init__(self, name=None, project=None, lang=None, base_url=None, article_path=None, script_path=None, sql=(None, None), - namespaces=None): + namespaces=None, login=(None, None)): """ Docstring needed """ + # attributes referring to site information, filled in by an API query + # if they are missing (and an API url is available) self._name = name self._project = project self._lang = lang @@ -30,9 +33,45 @@ class Site(object): self._sql = sql self._namespaces = namespaces - # get all of the above attributes that were not specified by the user + # set up cookiejar and URL opener for making API queries + self._cookiejar = CookieJar(cookie_file) + self._opener = build_opener(HTTPCookieProcessor(self._cookiejar)) + self._opener.addheaders = [('User-agent', USER_AGENT)] + + # use a username and password to login if they were provided + if login[0] is not None and login[1] is not None: + self._login(login[0], login[1]) + + # get all of the above attributes that were not specified as arguments self._load_attributes() + def _login(self, name, password, token="", attempt=0): + """ + Docstring needed + """ + params = {"action": "login", "lgname": name, "lgpassword": password, + "lgtoken": token} + result = self.api_query(params) + res = result["login"]["result"] + + if res == "Success": + return + elif res == "NeedToken" and attempt == 0: + token = result["login"]["token"] + return self._login(name, password, token, attempt=1) + else: + if res == "Illegal": + e = "The provided username is illegal." + elif res == "NotExists": + e = "The provided username does not exist." + elif res == "EmptyPass": + e = "No password was given." + elif res == "WrongPass" or res == "WrongPluginPass": + e = "The given password is incorrect." + else: + e = "Couldn't login; server says '{0}'.".format(res) + raise LoginError(e) + def _load_attributes(self, force=False): """ Docstring needed @@ -103,10 +142,24 @@ class Site(object): Docstring needed """ url = ''.join((self._base_url, self._script_path, "/api.php")) - params["format"] = "json" + params["format"] = "json" # this is the only format we understand data = urlencode(params) - result = urlopen(url, data).read() - return loads(result) + + try: + response = self._opener.open(url, data) + except URLError as error: + if hasattr(error, "reason"): + e = "API query at {0} failed because {1}.".format(error.geturl, + error.reason) + elif hasattr(error, "code"): + e = "API query at {0} failed; got an error code of {1}." + e = e.format(error.geturl, error.code) + else: + e = "API query failed." + raise SiteAPIError(e) + else: + result = response.read() + return loads(result) # parse as a JSON object def name(self): """ @@ -195,8 +248,13 @@ class Site(object): pagename = "{0}:{1}".format(prefix, catname) return Category(self, pagename) - def get_user(self, username): + def get_user(self, username=None): """ Docstring needed """ + if username is None: + params = {"action": "query", "meta": "userinfo"} + result = self.api_query(params) + username = result["query"]["userinfo"]["name"] + return User(self, username)