diff --git a/CHANGELOG b/CHANGELOG index 202265f..e71cebb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,16 @@ +v0.4 (released January 17, 2017): + +- Added a `--prune` flag to delete remote-tracking branches that no longer + exist on their remote after fetching. +- Added a `--bookmark-file` flag to support multiple bookmark config files. +- Added a `--cleanup` flag to remove old bookmarks that don't exist. +- Added an `--exec` flag to run a shell command on all of your repos. +- Added support for shell glob patterns and tilde expansion in bookmark files. +- Cleaned up the bookmark file format, fixing a related Windows bug. The script + will automatically migrate to the new one. +- Fixed a bug related to Python 3 compatibility. +- Fixed unicode support. + v0.3 (released June 7, 2015): - Added support for Python 3. diff --git a/LICENSE b/LICENSE index aef9ae4..5ed060b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2011-2015 Ben Kurtovic +Copyright (C) 2011-2017 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 0d4e9da..bfc125a 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,10 @@ upstreams configured. It will always skip branches where this is not possible (e.g. dirty working directory or a merge/rebase is required). Pass `--fetch-only` (or `-f`) to only fetch remotes. +After fetching, gitup will _keep_ remote-tracking branches that no longer exist +upstream. Pass `--prune` (or `-p`) to delete them, or set `fetch.prune` or +`remote..prune` in git config to do this by default. + For a full list of all command arguments and abbreviations: gitup --help diff --git a/gitup/__init__.py b/gitup/__init__.py index f031df0..f1fa294 100644 --- a/gitup/__init__.py +++ b/gitup/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2011-2015 Ben Kurtovic +# Copyright (C) 2011-2017 Ben Kurtovic # Released under the terms of the MIT License. See LICENSE for details. """ @@ -8,7 +8,7 @@ gitup: the git repository updater """ __author__ = "Ben Kurtovic" -__copyright__ = "Copyright (C) 2011-2015 Ben Kurtovic" +__copyright__ = "Copyright (C) 2011-2017 Ben Kurtovic" __license__ = "MIT License" -__version__ = "0.3" +__version__ = "0.4" __email__ = "ben.kurtovic@gmail.com" diff --git a/gitup/config.py b/gitup/config.py index f1cab54..7eeea7b 100644 --- a/gitup/config.py +++ b/gitup/config.py @@ -1,21 +1,19 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2011-2015 Ben Kurtovic +# Copyright (C) 2011-2016 Ben Kurtovic # Released under the terms of the MIT License. See LICENSE for details. from __future__ import print_function +from glob import glob import os -try: - import configparser -except ImportError: # Python 2 - import ConfigParser as configparser - from colorama import Fore, Style -__all__ = ["get_bookmarks", "add_bookmarks", "delete_bookmarks", - "list_bookmarks"] +from .migrate import run_migrations + +__all__ = ["get_default_config_path", "get_bookmarks", "add_bookmarks", + "delete_bookmarks", "list_bookmarks", "clean_bookmarks"] YELLOW = Fore.YELLOW + Style.BRIGHT RED = Fore.RED + Style.BRIGHT @@ -25,63 +23,60 @@ INDENT1 = " " * 3 def _ensure_dirs(path): """Ensure the directories within the given pathname exist.""" dirname = os.path.dirname(path) - if not os.path.exists(dirname): # Race condition, meh... + if dirname and not os.path.exists(dirname): # Race condition, meh... os.makedirs(dirname) -def _get_config_path(): - """Return the path to the configuration file.""" - xdg_cfg = os.environ.get("XDG_CONFIG_HOME") or os.path.join("~", ".config") - return os.path.join(os.path.expanduser(xdg_cfg), "gitup", "config.ini") - -def _migrate_old_config_path(): - """Migrate the old config location (~/.gitup) to the new one.""" - old_path = os.path.expanduser(os.path.join("~", ".gitup")) - if os.path.exists(old_path): - new_path = _get_config_path() - _ensure_dirs(new_path) - os.rename(old_path, new_path) - -def _load_config_file(): - """Read the config file and return a SafeConfigParser() object.""" - _migrate_old_config_path() - config = configparser.SafeConfigParser() - # Don't lowercase option names, because we are storing paths there: - config.optionxform = str - config.read(_get_config_path()) - return config - -def _save_config_file(config): - """Save config changes to the config file returned by _get_config_path.""" - _migrate_old_config_path() - cfg_path = _get_config_path() +def _load_config_file(config_path=None): + """Read the config file and return a list of bookmarks.""" + run_migrations() + cfg_path = config_path or get_default_config_path() + + try: + with open(cfg_path, "rb") as config_file: + paths = config_file.read().split(b"\n") + except IOError: + return [] + paths = [path.decode("utf8").strip() for path in paths] + return [path for path in paths if path] + +def _save_config_file(bookmarks, config_path=None): + """Save the bookmarks list to the given config file.""" + run_migrations() + cfg_path = config_path or get_default_config_path() _ensure_dirs(cfg_path) + + dump = b"\n".join(path.encode("utf8") for path in bookmarks) with open(cfg_path, "wb") as config_file: - config.write(config_file) + config_file.write(dump) + +def _normalize_path(path): + """Normalize the given path.""" + if path.startswith("~"): + return os.path.normcase(os.path.normpath(path)) + return os.path.normcase(os.path.abspath(path)) + +def get_default_config_path(): + """Return the default path to the configuration file.""" + xdg_cfg = os.environ.get("XDG_CONFIG_HOME") or os.path.join("~", ".config") + return os.path.join(os.path.expanduser(xdg_cfg), "gitup", "bookmarks") -def get_bookmarks(): +def get_bookmarks(config_path=None): """Get a list of all bookmarks, or an empty list if there are none.""" - config = _load_config_file() - try: - return [path for path, _ in config.items("bookmarks")] - except configparser.NoSectionError: - return [] + return _load_config_file(config_path) -def add_bookmarks(paths): +def add_bookmarks(paths, config_path=None): """Add a list of paths as bookmarks to the config file.""" - config = _load_config_file() - if not config.has_section("bookmarks"): - config.add_section("bookmarks") + config = _load_config_file(config_path) + paths = [_normalize_path(path) for path in paths] added, exists = [], [] for path in paths: - path = os.path.abspath(path) - if config.has_option("bookmarks", path): + if path in config: exists.append(path) else: - path_name = os.path.split(path)[1] - config.set("bookmarks", path, path_name) + config.append(path) added.append(path) - _save_config_file(config) + _save_config_file(config, config_path) if added: print(YELLOW + "Added bookmarks:") @@ -92,22 +87,22 @@ def add_bookmarks(paths): for path in exists: print(INDENT1, path) -def delete_bookmarks(paths): +def delete_bookmarks(paths, config_path=None): """Remove a list of paths from the bookmark config file.""" - config = _load_config_file() + config = _load_config_file(config_path) + paths = [_normalize_path(path) for path in paths] deleted, notmarked = [], [] - if config.has_section("bookmarks"): + if config: for path in paths: - path = os.path.abspath(path) - config_was_changed = config.remove_option("bookmarks", path) - if config_was_changed: + if path in config: + config.remove(path) deleted.append(path) else: notmarked.append(path) - _save_config_file(config) + _save_config_file(config, config_path) else: - notmarked = [os.path.abspath(path) for path in paths] + notmarked = paths if deleted: print(YELLOW + "Deleted bookmarks:") @@ -118,12 +113,32 @@ def delete_bookmarks(paths): for path in notmarked: print(INDENT1, path) -def list_bookmarks(): +def list_bookmarks(config_path=None): """Print all of our current bookmarks.""" - bookmarks = get_bookmarks() + bookmarks = _load_config_file(config_path) if bookmarks: print(YELLOW + "Current bookmarks:") for bookmark_path in bookmarks: print(INDENT1, bookmark_path) else: print("You have no bookmarks to display.") + +def clean_bookmarks(config_path=None): + """Delete any bookmarks that don't exist.""" + bookmarks = _load_config_file(config_path) + if not bookmarks: + print("You have no bookmarks to clean up.") + return + + delete = [path for path in bookmarks + if not (os.path.isdir(path) or glob(os.path.expanduser(path)))] + if not delete: + print("All of your bookmarks are valid.") + return + + bookmarks = [path for path in bookmarks if path not in delete] + _save_config_file(bookmarks, config_path) + + print(YELLOW + "Deleted bookmarks:") + for path in delete: + print(INDENT1, path) diff --git a/gitup/migrate.py b/gitup/migrate.py new file mode 100644 index 0000000..2de21f8 --- /dev/null +++ b/gitup/migrate.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2011-2016 Ben Kurtovic +# Released under the terms of the MIT License. See LICENSE for details. + +import os + +try: + from configparser import ConfigParser, NoSectionError + PY3K = True +except ImportError: # Python 2 + from ConfigParser import SafeConfigParser as ConfigParser, NoSectionError + PY3K = False + +__all__ = ["run_migrations"] + +def _get_old_path(): + """Return the old default path to the configuration file.""" + xdg_cfg = os.environ.get("XDG_CONFIG_HOME") or os.path.join("~", ".config") + return os.path.join(os.path.expanduser(xdg_cfg), "gitup", "config.ini") + +def _migrate_old_path(): + """Migrate the old config location (~/.gitup) to the new one.""" + old_path = os.path.expanduser(os.path.join("~", ".gitup")) + if not os.path.exists(old_path): + return + + temp_path = _get_old_path() + temp_dir = os.path.dirname(temp_path) + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + os.rename(old_path, temp_path) + +def _migrate_old_format(): + """Migrate the old config file format (.INI) to our custom list format.""" + old_path = _get_old_path() + if not os.path.exists(old_path): + return + + config = ConfigParser(delimiters="=") if PY3K else ConfigParser() + config.optionxform = lambda opt: opt + config.read(old_path) + + try: + bookmarks = [path for path, _ in config.items("bookmarks")] + except NoSectionError: + bookmarks = [] + if PY3K: + bookmarks = [path.encode("utf8") for path in bookmarks] + + new_path = os.path.join(os.path.split(old_path)[0], "bookmarks") + os.rename(old_path, new_path) + + with open(new_path, "wb") as handle: + handle.write(b"\n".join(bookmarks)) + +def run_migrations(): + """Run any necessary migrations to ensure the config file is up-to-date.""" + _migrate_old_path() + _migrate_old_format() diff --git a/gitup/script.py b/gitup/script.py index d353412..8f0c2f7 100644 --- a/gitup/script.py +++ b/gitup/script.py @@ -1,18 +1,26 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2011-2015 Ben Kurtovic +# Copyright (C) 2011-2016 Ben Kurtovic # Released under the terms of the MIT License. See LICENSE for details. from __future__ import print_function import argparse +import os +import sys from colorama import init as color_init, Fore, Style from . import __version__ -from .config import (get_bookmarks, add_bookmarks, delete_bookmarks, - list_bookmarks) -from .update import update_bookmarks, update_directories +from .config import (get_default_config_path, get_bookmarks, add_bookmarks, + delete_bookmarks, list_bookmarks, clean_bookmarks) +from .update import update_bookmarks, update_directories, run_command + +def _decode(path): + """Decode the given string using the system's filesystem encoding.""" + if sys.version_info.major > 2: + return path + return path.decode(sys.getfilesystemencoding()) def main(): """Parse arguments and then call the appropriate function(s).""" @@ -20,16 +28,17 @@ def main(): description="Easily update multiple git repositories at once.", epilog=""" Both relative and absolute paths are accepted by all arguments. - Direct bug reports and feature requests to: + Direct bug reports and feature requests to https://github.com/earwig/git-repo-updater.""", add_help=False) group_u = parser.add_argument_group("updating repositories") group_b = parser.add_argument_group("bookmarking") + group_a = parser.add_argument_group("advanced") group_m = parser.add_argument_group("miscellaneous") group_u.add_argument( - 'directories_to_update', nargs="*", metavar="path", + 'directories_to_update', nargs="*", metavar="path", type=_decode, help="""update all repositories in this directory (or the directory itself, if it is a repo)""") group_u.add_argument( @@ -41,16 +50,31 @@ def main(): group_u.add_argument( '-f', '--fetch-only', action="store_true", help="only fetch remotes, don't try to fast-forward any branches") + group_u.add_argument( + '-p', '--prune', action="store_true", help="""after fetching, delete + remote-tracking branches that no longer exist on their remote""") group_b.add_argument( '-a', '--add', dest="bookmarks_to_add", nargs="+", metavar="path", - help="add directory(s) as bookmarks") + type=_decode, help="add directory(s) as bookmarks") group_b.add_argument( '-d', '--delete', dest="bookmarks_to_del", nargs="+", metavar="path", + type=_decode, help="delete bookmark(s) (leaves actual directories alone)") group_b.add_argument( '-l', '--list', dest="list_bookmarks", action="store_true", help="list current bookmarks") + group_b.add_argument( + '-n', '--clean', '--cleanup', dest="clean_bookmarks", + action="store_true", help="delete any bookmarks that don't exist") + group_b.add_argument( + '-b', '--bookmark-file', nargs="?", metavar="path", type=_decode, + help="use a specific bookmark config file (default: {0})".format( + get_default_config_path())) + + group_a.add_argument( + '-e', '--exec', '--batch', dest="command", metavar="command", + help="run a shell command on all repos") group_m.add_argument( '-h', '--help', action="help", help="show this help message and exit") @@ -66,7 +90,7 @@ def main(): color_init(autoreset=True) args = parser.parse_args() - update_args = args.current_only, args.fetch_only + update_args = args.current_only, args.fetch_only, args.prune print(Style.BRIGHT + "gitup" + Style.RESET_ALL + ": the git-repo-updater") print() @@ -78,21 +102,34 @@ def main(): "upstream branch and can be safely fast-forwarded. Use " "--fetch-only to\navoid updating any branches.\n") + if args.bookmark_file: + args.bookmark_file = os.path.expanduser(args.bookmark_file) + acted = False if args.bookmarks_to_add: - add_bookmarks(args.bookmarks_to_add) + add_bookmarks(args.bookmarks_to_add, args.bookmark_file) acted = True if args.bookmarks_to_del: - delete_bookmarks(args.bookmarks_to_del) + delete_bookmarks(args.bookmarks_to_del, args.bookmark_file) acted = True if args.list_bookmarks: - list_bookmarks() + list_bookmarks(args.bookmark_file) acted = True - if args.directories_to_update: - update_directories(args.directories_to_update, update_args) + if args.clean_bookmarks: + clean_bookmarks(args.bookmark_file) acted = True - if args.update or not acted: - update_bookmarks(get_bookmarks(), update_args) + + if args.command: + if args.directories_to_update: + run_command(args.directories_to_update, args.command) + if args.update or not args.directories_to_update: + run_command(get_bookmarks(args.bookmark_file), args.command) + else: + if args.directories_to_update: + update_directories(args.directories_to_update, update_args) + acted = True + if args.update or not acted: + update_bookmarks(get_bookmarks(args.bookmark_file), update_args) def run(): """Thin wrapper for main() that catches KeyboardInterrupts.""" diff --git a/gitup/update.py b/gitup/update.py index a121a67..64991eb 100644 --- a/gitup/update.py +++ b/gitup/update.py @@ -1,17 +1,19 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2011-2015 Ben Kurtovic +# Copyright (C) 2011-2016 Ben Kurtovic # Released under the terms of the MIT License. See LICENSE for details. from __future__ import print_function +from glob import glob import os +import shlex from colorama import Fore, Style from git import RemoteReference as RemoteRef, Repo, exc from git.util import RemoteProgress -__all__ = ["update_bookmarks", "update_directories"] +__all__ = ["update_bookmarks", "update_directories", "run_command"] BOLD = Style.BRIGHT BLUE = Fore.BLUE + BOLD @@ -53,12 +55,13 @@ class _ProgressMonitor(RemoteProgress): print(str(cur_count), end=end) -def _fetch_remotes(remotes): +def _fetch_remotes(remotes, prune): """Fetch a list of remotes, displaying progress info along the way.""" def _get_name(ref): """Return the local name of a remote or tag reference.""" return ref.remote_head if isinstance(ref, RemoteRef) else ref.name + # TODO: missing branch deleted (via --prune): info = [("NEW_HEAD", "new branch", "new branches"), ("NEW_TAG", "new tag", "new tags"), ("FAST_FORWARD", "branch update", "branch updates")] @@ -72,7 +75,7 @@ def _fetch_remotes(remotes): continue try: - results = remote.fetch(progress=_ProgressMonitor()) + results = remote.fetch(progress=_ProgressMonitor(), prune=prune) except exc.GitCommandError as err: msg = err.command[0].replace("Error when fetching: ", "") if not msg.endswith("."): @@ -101,12 +104,16 @@ def _update_branch(repo, branch, is_active=False): if not upstream: print(YELLOW + "skipped:", "no upstream is tracked.") return - try: - branch.commit, upstream.commit + branch.commit except ValueError: print(YELLOW + "skipped:", "branch has no revisions.") return + try: + upstream.commit + except ValueError: + print(YELLOW + "skipped:", "upstream does not exist.") + return base = repo.git.merge_base(branch.commit, upstream.commit) if repo.commit(base) == upstream.commit: @@ -133,13 +140,15 @@ def _update_branch(repo, branch, is_active=False): repo.git.branch(branch.name, upstream.name, force=True) print(GREEN + "done", end=".\n") -def _update_repository(repo, current_only=False, fetch_only=False): +def _update_repository(repo, current_only, fetch_only, prune): """Update a single git repository by fetching remotes and rebasing/merging. The specific actions depend on the arguments given. We will fetch all remotes if *current_only* is ``False``, or only the remote tracked by the current branch if ``True``. If *fetch_only* is ``False``, we will also update all fast-forwardable branches that are tracking valid upstreams. + If *prune* is ``True``, remote-tracking branches that no longer exist on + their remote after fetching will be deleted. """ print(INDENT1, BOLD + os.path.split(repo.working_dir)[1] + ":") @@ -163,58 +172,88 @@ def _update_repository(repo, current_only=False, fetch_only=False): if not remotes: print(INDENT2, ERROR, "no remotes configured to fetch.") return - _fetch_remotes(remotes) + _fetch_remotes(remotes, prune) if not fetch_only: for branch in sorted(repo.heads, key=lambda b: b.name): _update_branch(repo, branch, branch == active) -def _update_subdirectories(path, update_args): - """Update all subdirectories that are git repos in a given directory.""" - repos = [] - for item in os.listdir(path): +def _run_command(repo, command): + """Run an arbitrary shell command on the given repository.""" + print(INDENT1, BOLD + os.path.split(repo.working_dir)[1] + ":") + + cmd = shlex.split(command) + try: + out = repo.git.execute( + cmd, with_extended_output=True, with_exceptions=False) + except exc.GitCommandNotFound as err: + print(INDENT2, ERROR, err) + return + + for line in out[1].splitlines() + out[2].splitlines(): + print(INDENT2, line) + +def _dispatch_multi(base, paths, callback, *args): + """Apply the callback to all git repos in the list of paths.""" + valid = [] + for path in paths: try: - repo = Repo(os.path.join(path, item)) + Repo(path) except (exc.InvalidGitRepositoryError, exc.NoSuchPathError): continue - repos.append(repo) + valid.append(path) + + base = os.path.abspath(base) + suffix = "" if len(valid) == 1 else "s" + print(BOLD + base, "({0} repo{1}):".format(len(valid), suffix)) - suffix = "" if len(repos) == 1 else "s" - print(BOLD + path, "({0} repo{1}):".format(len(repos), suffix)) - for repo in sorted(repos, key=lambda r: os.path.split(r.working_dir)[1]): - _update_repository(repo, *update_args) + for path in sorted(valid, key=os.path.basename): + callback(Repo(path), *args) -def _update_directory(path, update_args): - """Update a particular directory. +def _dispatch(path, callback, *args): + """Apply a callback function on each valid repo in the given path. Determine whether the directory is a git repo on its own, a directory of - git repositories, or something invalid. If the first, update the single - repository; if the second, update all repositories contained within; if the - third, print an error. + git repositories, a shell glob pattern, or something invalid. If the first, + apply the callback on it; if the second or third, apply the callback on all + repositories contained within; if the last, print an error. + + The given args are passed directly to the callback function after the repo. """ + path = os.path.expanduser(path) try: repo = Repo(path) except exc.NoSuchPathError: - print(ERROR, BOLD + path, "doesn't exist!") + paths = glob(path) + if paths: + _dispatch_multi(path, paths, callback, *args) + else: + print(ERROR, BOLD + path, "doesn't exist!") except exc.InvalidGitRepositoryError: if os.path.isdir(path): - _update_subdirectories(path, update_args) + paths = [os.path.join(path, item) for item in os.listdir(path)] + _dispatch_multi(path, paths, callback, *args) else: print(ERROR, BOLD + path, "isn't a repository!") else: print(BOLD + repo.working_dir, "(1 repo):") - _update_repository(repo, *update_args) + callback(repo, *args) def update_bookmarks(bookmarks, update_args): """Loop through and update all bookmarks.""" - if bookmarks: - for path in bookmarks: - _update_directory(path, update_args) - else: + if not bookmarks: print("You don't have any bookmarks configured! Get help with 'gitup -h'.") + return + + for path in bookmarks: + _dispatch(path, _update_repository, *update_args) def update_directories(paths, update_args): """Update a list of directories supplied by command arguments.""" for path in paths: - full_path = os.path.abspath(path) - _update_directory(full_path, update_args) + _dispatch(path, _update_repository, *update_args) + +def run_command(paths, command): + """Run an arbitrary shell command on all repos.""" + for path in paths: + _dispatch(path, _run_command, command) diff --git a/setup.py b/setup.py index 2098ffd..c2fc57b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2011-2015 Ben Kurtovic +# Copyright (C) 2011-2016 Ben Kurtovic # Released under the terms of the MIT License. See LICENSE for details. import sys @@ -19,7 +19,7 @@ setup( name = "gitup", packages = find_packages(), entry_points = {"console_scripts": ["gitup = gitup.script:run"]}, - install_requires = ["GitPython >= 1.0.1", "colorama >= 0.3.3"], + install_requires = ["GitPython >= 2.1.1", "colorama >= 0.3.7"], version = __version__, author = "Ben Kurtovic", author_email = "ben.kurtovic@gmail.com", @@ -43,6 +43,7 @@ setup( "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", "Topic :: Software Development :: Version Control" ] )