update-python-libraries: commit updates and specify update kind

This commit introduces two new features:

1. specify with --target whether major, minor or patch updates should be made
2. use --commit to create commits for each of the updates

+138 -22
+138 -22
maintainers/scripts/update-python-libraries
··· 1 1 #! /usr/bin/env nix-shell 2 - #! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ requests toolz ])' 2 + #! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ packaging requests toolz ])' -p git 3 3 4 4 """ 5 5 Update a Python package expression by passing in the `.nix` file, or the directory containing it. ··· 18 18 import re 19 19 import requests 20 20 import toolz 21 - from concurrent.futures import ThreadPoolExecutor as pool 21 + from concurrent.futures import ThreadPoolExecutor as Pool 22 + from packaging.version import Version as _Version 23 + from packaging.version import InvalidVersion 24 + from packaging.specifiers import SpecifierSet 25 + import collections 26 + import subprocess 22 27 23 28 INDEX = "https://pypi.io/pypi" 24 29 """url of PyPI""" ··· 26 31 EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl'] 27 32 """Permitted file extensions. These are evaluated from left to right and the first occurance is returned.""" 28 33 34 + PRERELEASES = False 35 + 29 36 import logging 30 37 logging.basicConfig(level=logging.INFO) 31 38 32 39 40 + class Version(_Version, collections.abc.Sequence): 41 + 42 + def __init__(self, version): 43 + super().__init__(version) 44 + # We cannot use `str(Version(0.04.21))` because that becomes `0.4.21` 45 + # https://github.com/avian2/unidecode/issues/13#issuecomment-354538882 46 + self.raw_version = version 47 + 48 + def __getitem__(self, i): 49 + return self._version.release[i] 50 + 51 + def __len__(self): 52 + return len(self._version.release) 53 + 54 + def __iter__(self): 55 + yield from self._version.release 56 + 57 + 33 58 def _get_values(attribute, text): 34 59 """Match attribute in text and return all matches. 35 60 ··· 82 107 else: 83 108 raise ValueError("request for {} failed".format(url)) 84 109 85 - def _get_latest_version_pypi(package, extension): 110 + 111 + SEMVER = { 112 + 'major' : 0, 113 + 'minor' : 1, 114 + 'patch' : 2, 115 + } 116 + 117 + 118 + def _determine_latest_version(current_version, target, versions): 119 + """Determine latest version, given `target`. 120 + """ 121 + current_version = Version(current_version) 122 + 123 + def _parse_versions(versions): 124 + for v in versions: 125 + try: 126 + yield Version(v) 127 + except InvalidVersion: 128 + pass 129 + 130 + versions = _parse_versions(versions) 131 + 132 + index = SEMVER[target] 133 + 134 + ceiling = list(current_version[0:index]) 135 + if len(ceiling) == 0: 136 + ceiling = None 137 + else: 138 + ceiling[-1]+=1 139 + ceiling = Version(".".join(map(str, ceiling))) 140 + 141 + # We do not want prereleases 142 + versions = SpecifierSet(prereleases=PRERELEASES).filter(versions) 143 + 144 + if ceiling is not None: 145 + versions = SpecifierSet(f"<{ceiling}").filter(versions) 146 + 147 + return (max(sorted(versions))).raw_version 148 + 149 + 150 + def _get_latest_version_pypi(package, extension, current_version, target): 86 151 """Get latest version and hash from PyPI.""" 87 152 url = "{}/{}/json".format(INDEX, package) 88 153 json = _fetch_page(url) 89 154 90 - version = json['info']['version'] 91 - for release in json['releases'][version]: 155 + versions = json['releases'].keys() 156 + version = _determine_latest_version(current_version, target, versions) 157 + 158 + try: 159 + releases = json['releases'][version] 160 + except KeyError as e: 161 + raise KeyError('Could not find version {} for {}'.format(version, package)) from e 162 + for release in releases: 92 163 if release['filename'].endswith(extension): 93 164 # TODO: In case of wheel we need to do further checks! 94 165 sha256 = release['digests']['sha256'] ··· 98 169 return version, sha256 99 170 100 171 101 - def _get_latest_version_github(package, extension): 172 + def _get_latest_version_github(package, extension, current_version, target): 102 173 raise ValueError("updating from GitHub is not yet supported.") 103 174 104 175 ··· 141 212 """ 142 213 if fetcher == 'fetchPypi': 143 214 try: 144 - format = _get_unique_value('format', text) 215 + src_format = _get_unique_value('format', text) 145 216 except ValueError as e: 146 - format = None # format was not given 217 + src_format = None # format was not given 147 218 148 219 try: 149 220 extension = _get_unique_value('extension', text) ··· 151 222 extension = None # extension was not given 152 223 153 224 if extension is None: 154 - if format is None: 155 - format = 'setuptools' 156 - extension = FORMATS[format] 225 + if src_format is None: 226 + src_format = 'setuptools' 227 + elif src_format == 'flit': 228 + raise ValueError("Don't know how to update a Flit package.") 229 + extension = FORMATS[src_format] 157 230 158 231 elif fetcher == 'fetchurl': 159 232 url = _get_unique_value('url', text) ··· 167 240 return extension 168 241 169 242 170 - def _update_package(path): 171 - 172 - 243 + def _update_package(path, target): 173 244 174 245 # Read the expression 175 246 with open(path, 'r') as f: ··· 186 257 187 258 extension = _determine_extension(text, fetcher) 188 259 189 - new_version, new_sha256 = _get_latest_version_pypi(pname, extension) 260 + new_version, new_sha256 = FETCHERS[fetcher](pname, extension, version, target) 190 261 191 262 if new_version == version: 192 263 logging.info("Path {}: no update available for {}.".format(path, pname)) 193 264 return False 265 + elif new_version <= version: 266 + raise ValueError("downgrade for {}.".format(pname)) 194 267 if not new_sha256: 195 268 raise ValueError("no file available for {}.".format(pname)) 196 269 ··· 202 275 203 276 logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version)) 204 277 205 - return True 278 + result = { 279 + 'path' : path, 280 + 'target': target, 281 + 'pname': pname, 282 + 'old_version' : version, 283 + 'new_version' : new_version, 284 + #'fetcher' : fetcher, 285 + } 206 286 287 + return result 207 288 208 - def _update(path): 289 + 290 + def _update(path, target): 209 291 210 292 # We need to read and modify a Nix expression. 211 293 if os.path.isdir(path): ··· 222 304 return False 223 305 224 306 try: 225 - return _update_package(path) 307 + return _update_package(path, target) 226 308 except ValueError as e: 227 309 logging.warning("Path {}: {}".format(path, e)) 228 310 return False 229 311 312 + 313 + def _commit(path, pname, old_version, new_version, **kwargs): 314 + """Commit result. 315 + """ 316 + 317 + msg = f'python: {pname}: {old_version} -> {new_version}' 318 + 319 + try: 320 + subprocess.check_call(['git', 'add', path]) 321 + subprocess.check_call(['git', 'commit', '-m', msg]) 322 + except subprocess.CalledProcessError as e: 323 + subprocess.check_call(['git', 'checkout', path]) 324 + raise subprocess.CalledProcessError(f'Could not commit {path}') from e 325 + 326 + return True 327 + 328 + 230 329 def main(): 231 330 232 331 parser = argparse.ArgumentParser() 233 332 parser.add_argument('package', type=str, nargs='+') 333 + parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major') 334 + parser.add_argument('--commit', action='store_true', help='Create a commit for each package update') 234 335 235 336 args = parser.parse_args() 337 + target = args.target 236 338 237 - packages = map(os.path.abspath, args.package) 339 + packages = list(map(os.path.abspath, args.package)) 238 340 239 - with pool() as p: 240 - count = list(p.map(_update, packages)) 341 + logging.info("Updating packages...") 241 342 242 - logging.info("{} package(s) updated".format(sum(count))) 343 + # Use threads to update packages concurrently 344 + with Pool() as p: 345 + results = list(p.map(lambda pkg: _update(pkg, target), packages)) 346 + 347 + logging.info("Finished updating packages.") 348 + 349 + # Commits are created sequentially. 350 + if args.commit: 351 + logging.info("Committing updates...") 352 + list(map(lambda x: _commit(**x), filter(bool, results))) 353 + logging.info("Finished committing updates") 354 + 355 + count = sum(map(bool, results)) 356 + logging.info("{} package(s) updated".format(count)) 357 + 358 + 243 359 244 360 if __name__ == '__main__': 245 361 main()