devscripts/make_changelog.py

   1 from __future__ import annotations
   2
   3 import enum
   4 import itertools
   5 import json
   6 import logging
   7 import re
   8 import subprocess
   9 import sys
  10 from collections import defaultdict
  11 from dataclasses import dataclass
  12 from functools import lru_cache
  13 from pathlib import Path
  14
  15 BASE_URL = 'https://github.com'
  16 LOCATION_PATH = Path(__file__).parent
  17
  18 logger = logging.getLogger(__name__)
  19
  20
  21 class CommitGroup(enum.Enum):
  22     UPSTREAM = None
  23     PRIORITY = 'Important'
  24     CORE = 'Core'
  25     EXTRACTOR = 'Extractor'
  26     DOWNLOADER = 'Downloader'
  27     POSTPROCESSOR = 'Postprocessor'
  28     MISC = 'Misc.'
  29
  30     @classmethod
  31     @lru_cache
  32     def commit_lookup(cls):
  33         return {
  34             name: group
  35             for group, names in {
  36                 cls.PRIORITY: {''},
  37                 cls.UPSTREAM: {'upstream'},
  38                 cls.CORE: {
  39                     'aes',
  40                     'cache',
  41                     'compat_utils',
  42                     'compat',
  43                     'cookies',
  44                     'core',
  45                     'dependencies',
  46                     'jsinterp',
  47                     'outtmpl',
  48                     'plugins',
  49                     'update',
  50                     'utils',
  51                 },
  52                 cls.MISC: {
  53                     'build',
  54                     'cleanup',
  55                     'devscripts',
  56                     'docs',
  57                     'misc',
  58                     'test',
  59                 },
  60                 cls.EXTRACTOR: {'extractor', 'extractors'},
  61                 cls.DOWNLOADER: {'downloader'},
  62                 cls.POSTPROCESSOR: {'postprocessor'},
  63             }.items()
  64             for name in names
  65         }
  66
  67     @classmethod
  68     def get(cls, value):
  69         result = cls.commit_lookup().get(value)
  70         if result:
  71             logger.debug(f'Mapped {value!r} => {result.name}')
  72         return result
  73
  74
  75 @dataclass
  76 class Commit:
  77     hash: str | None
  78     short: str
  79     authors: list[str]
  80
  81     def __str__(self):
  82         result = f'{self.short!r}'
  83
  84         if self.hash:
  85             result += f' ({self.hash[:7]})'
  86
  87         if self.authors:
  88             authors = ', '.join(self.authors)
  89             result += f' by {authors}'
  90
  91         return result
  92
  93
  94 @dataclass
  95 class CommitInfo:
  96     details: str | None
  97     sub_details: tuple[str, ...]
  98     message: str
  99     issues: list[str]
 100     commit: Commit
 101     fixes: list[Commit]
 102
 103     def key(self):
 104         return ((self.details or '').lower(), self.sub_details, self.message)
 105
 106
 107 class Changelog:
 108     MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE)
 109
 110     def __init__(self, groups, repo):
 111         self._groups = groups
 112         self._repo = repo
 113
 114     def __str__(self):
 115         return '\n'.join(self._format_groups(self._groups)).replace('\t', '    ')
 116
 117     def _format_groups(self, groups):
 118         for item in CommitGroup:
 119             group = groups[item]
 120             if group:
 121                 yield self.format_module(item.value, group)
 122
 123     def format_module(self, name, group):
 124         result = f'\n#### {name} changes\n' if name else '\n'
 125         return result + '\n'.join(self._format_group(group))
 126
 127     def _format_group(self, group):
 128         sorted_group = sorted(group, key=CommitInfo.key)
 129         detail_groups = itertools.groupby(sorted_group, lambda item: (item.details or '').lower())
 130         for _, items in detail_groups:
 131             items = list(items)
 132             details = items[0].details
 133             if not details:
 134                 indent = ''
 135             else:
 136                 yield f'- {details}'
 137                 indent = '\t'
 138
 139             if details == 'cleanup':
 140                 items, cleanup_misc_items = self._filter_cleanup_misc_items(items)
 141
 142             sub_detail_groups = itertools.groupby(items, lambda item: tuple(map(str.lower, item.sub_details)))
 143             for sub_details, entries in sub_detail_groups:
 144                 if not sub_details:
 145                     for entry in entries:
 146                         yield f'{indent}- {self.format_single_change(entry)}'
 147                     continue
 148
 149                 entries = list(entries)
 150                 prefix = f'{indent}- {", ".join(entries[0].sub_details)}'
 151                 if len(entries) == 1:
 152                     yield f'{prefix}: {self.format_single_change(entries[0])}'
 153                     continue
 154
 155                 yield prefix
 156                 for entry in entries:
 157                     yield f'{indent}\t- {self.format_single_change(entry)}'
 158
 159             if details == 'cleanup' and cleanup_misc_items:
 160                 yield from self._format_cleanup_misc_sub_group(cleanup_misc_items)
 161
 162     def _filter_cleanup_misc_items(self, items):
 163         cleanup_misc_items = defaultdict(list)
 164         non_misc_items = []
 165         for item in items:
 166             if self.MISC_RE.search(item.message):
 167                 cleanup_misc_items[tuple(item.commit.authors)].append(item)
 168             else:
 169                 non_misc_items.append(item)
 170
 171         return non_misc_items, cleanup_misc_items
 172
 173     def _format_cleanup_misc_sub_group(self, group):
 174         prefix = '\t- Miscellaneous'
 175         if len(group) == 1:
 176             yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}'
 177             return
 178
 179         yield prefix
 180         for message in self._format_cleanup_misc_items(group):
 181             yield f'\t\t- {message}'
 182
 183     def _format_cleanup_misc_items(self, group):
 184         for authors, infos in group.items():
 185             message = ', '.join(
 186                 self._format_message_link(None, info.commit.hash)
 187                 for info in sorted(infos, key=lambda item: item.commit.hash or ''))
 188             yield f'{message} by {self._format_authors(authors)}'
 189
 190     def format_single_change(self, info):
 191         message = self._format_message_link(info.message, info.commit.hash)
 192         if info.issues:
 193             message = f'{message} ({self._format_issues(info.issues)})'
 194
 195         if info.commit.authors:
 196             message = f'{message} by {self._format_authors(info.commit.authors)}'
 197
 198         if info.fixes:
 199             fix_message = ', '.join(f'{self._format_message_link(None, fix.hash)}' for fix in info.fixes)
 200
 201             authors = sorted({author for fix in info.fixes for author in fix.authors}, key=str.casefold)
 202             if authors != info.commit.authors:
 203                 fix_message = f'{fix_message} by {self._format_authors(authors)}'
 204
 205             message = f'{message} (With fixes in {fix_message})'
 206
 207         return message
 208
 209     def _format_message_link(self, message, hash):
 210         assert message or hash, 'Improperly defined commit message or override'
 211         message = message if message else hash[:7]
 212         return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message
 213
 214     def _format_issues(self, issues):
 215         return ', '.join(f'[#{issue}]({self.repo_url}/issues/{issue})' for issue in issues)
 216
 217     @staticmethod
 218     def _format_authors(authors):
 219         return ', '.join(f'[{author}]({BASE_URL}/{author})' for author in authors)
 220
 221     @property
 222     def repo_url(self):
 223         return f'{BASE_URL}/{self._repo}'
 224
 225
 226 class CommitRange:
 227     COMMAND = 'git'
 228     COMMIT_SEPARATOR = '-----'
 229
 230     AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE)
 231     MESSAGE_RE = re.compile(r'''
 232         (?:\[
 233             (?P<prefix>[^\]\/:,]+)
 234             (?:/(?P<details>[^\]:,]+))?
 235             (?:[:,](?P<sub_details>[^\]]+))?
 236         \]\ )?
 237         (?:(?P<sub_details_alt>`?[^:`]+`?): )?
 238         (?P<message>.+?)
 239         (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
 240         ''', re.VERBOSE | re.DOTALL)
 241     EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
 242     FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
 243     UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
 244
 245     def __init__(self, start, end, default_author=None) -> None:
 246         self._start = start
 247         self._end = end
 248         self._commits, self._fixes = self._get_commits_and_fixes(default_author)
 249         self._commits_added = []
 250
 251     def __iter__(self):
 252         return iter(itertools.chain(self._commits.values(), self._commits_added))
 253
 254     def __len__(self):
 255         return len(self._commits) + len(self._commits_added)
 256
 257     def __contains__(self, commit):
 258         if isinstance(commit, Commit):
 259             if not commit.hash:
 260                 return False
 261             commit = commit.hash
 262
 263         return commit in self._commits
 264
 265     def _is_ancestor(self, commitish):
 266         return bool(subprocess.call(
 267             [self.COMMAND, 'merge-base', '--is-ancestor', commitish, self._start]))
 268
 269     def _get_commits_and_fixes(self, default_author):
 270         result = subprocess.check_output([
 271             self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}',
 272             f'{self._start}..{self._end}' if self._start else self._end], text=True)
 273
 274         commits = {}
 275         fixes = defaultdict(list)
 276         lines = iter(result.splitlines(False))
 277         for i, commit_hash in enumerate(lines):
 278             short = next(lines)
 279             skip = short.startswith('Release ') or short == '[version] update'
 280
 281             authors = [default_author] if default_author else []
 282             for line in iter(lambda: next(lines), self.COMMIT_SEPARATOR):
 283                 match = self.AUTHOR_INDICATOR_RE.match(line)
 284                 if match:
 285                     authors = sorted(map(str.strip, line[match.end():].split(',')), key=str.casefold)
 286
 287             commit = Commit(commit_hash, short, authors)
 288             if skip and (self._start or not i):
 289                 logger.debug(f'Skipped commit: {commit}')
 290                 continue
 291             elif skip:
 292                 logger.debug(f'Reached Release commit, breaking: {commit}')
 293                 break
 294
 295             fix_match = self.FIXES_RE.search(commit.short)
 296             if fix_match:
 297                 commitish = fix_match.group(1)
 298                 fixes[commitish].append(commit)
 299
 300             commits[commit.hash] = commit
 301
 302         for commitish, fix_commits in fixes.items():
 303             if commitish in commits:
 304                 hashes = ', '.join(commit.hash[:7] for commit in fix_commits)
 305                 logger.info(f'Found fix(es) for {commitish[:7]}: {hashes}')
 306                 for fix_commit in fix_commits:
 307                     del commits[fix_commit.hash]
 308             else:
 309                 logger.debug(f'Commit with fixes not in changes: {commitish[:7]}')
 310
 311         return commits, fixes
 312
 313     def apply_overrides(self, overrides):
 314         for override in overrides:
 315             when = override.get('when')
 316             if when and when not in self and when != self._start:
 317                 logger.debug(f'Ignored {when!r}, not in commits {self._start!r}')
 318                 continue
 319
 320             override_hash = override.get('hash')
 321             if override['action'] == 'add':
 322                 commit = Commit(override.get('hash'), override['short'], override.get('authors') or [])
 323                 logger.info(f'ADD    {commit}')
 324                 self._commits_added.append(commit)
 325
 326             elif override['action'] == 'remove':
 327                 if override_hash in self._commits:
 328                     logger.info(f'REMOVE {self._commits[override_hash]}')
 329                     del self._commits[override_hash]
 330
 331             elif override['action'] == 'change':
 332                 if override_hash not in self._commits:
 333                     continue
 334                 commit = Commit(override_hash, override['short'], override['authors'])
 335                 logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}')
 336                 self._commits[commit.hash] = commit
 337
 338         self._commits = {key: value for key, value in reversed(self._commits.items())}
 339
 340     def groups(self):
 341         groups = defaultdict(list)
 342         for commit in self:
 343             upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short)
 344             if upstream_re:
 345                 commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}'
 346
 347             match = self.MESSAGE_RE.fullmatch(commit.short)
 348             if not match:
 349                 logger.error(f'Error parsing short commit message: {commit.short!r}')
 350                 continue
 351
 352             prefix, details, sub_details, sub_details_alt, message, issues = match.groups()
 353             group = None
 354             if prefix:
 355                 if prefix == 'priority':
 356                     prefix, _, details = (details or '').partition('/')
 357                     logger.debug(f'Priority: {message!r}')
 358                     group = CommitGroup.PRIORITY
 359
 360                 if not details and prefix:
 361                     if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'):
 362                         logger.debug(f'Replaced details with {prefix!r}')
 363                         details = prefix or None
 364
 365                 if details == 'common':
 366                     details = None
 367
 368                 if details:
 369                     details = details.strip()
 370
 371             else:
 372                 group = CommitGroup.CORE
 373
 374             sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.replace(':', ',')
 375             sub_details = tuple(filter(None, map(str.strip, sub_details.split(','))))
 376
 377             issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []
 378
 379             if not group:
 380                 group = CommitGroup.get(prefix.lower())
 381                 if not group:
 382                     if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
 383                         group = CommitGroup.EXTRACTOR
 384                     else:
 385                         group = CommitGroup.POSTPROCESSOR
 386                     logger.warning(f'Failed to map {commit.short!r}, selected {group.name}')
 387
 388             commit_info = CommitInfo(
 389                 details, sub_details, message.strip(),
 390                 issues, commit, self._fixes[commit.hash])
 391             logger.debug(f'Resolved {commit.short!r} to {commit_info!r}')
 392             groups[group].append(commit_info)
 393
 394         return groups
 395
 396
 397 def get_new_contributors(contributors_path, commits):
 398     contributors = set()
 399     if contributors_path.exists():
 400         with contributors_path.open() as file:
 401             for line in filter(None, map(str.strip, file)):
 402                 author, _, _ = line.partition(' (')
 403                 authors = author.split('/')
 404                 contributors.update(map(str.casefold, authors))
 405
 406     new_contributors = set()
 407     for commit in commits:
 408         for author in commit.authors:
 409             author_folded = author.casefold()
 410             if author_folded not in contributors:
 411                 contributors.add(author_folded)
 412                 new_contributors.add(author)
 413
 414     return sorted(new_contributors, key=str.casefold)
 415
 416
 417 if __name__ == '__main__':
 418     import argparse
 419
 420     parser = argparse.ArgumentParser(
 421         description='Create a changelog markdown from a git commit range')
 422     parser.add_argument(
 423         'commitish', default='HEAD', nargs='?',
 424         help='The commitish to create the range from (default: %(default)s)')
 425     parser.add_argument(
 426         '-v', '--verbosity', action='count', default=0,
 427         help='increase verbosity (can be used twice)')
 428     parser.add_argument(
 429         '-c', '--contributors', action='store_true',
 430         help='update CONTRIBUTORS file (default: %(default)s)')
 431     parser.add_argument(
 432         '--contributors-path', type=Path, default=LOCATION_PATH.parent / 'CONTRIBUTORS',
 433         help='path to the CONTRIBUTORS file')
 434     parser.add_argument(
 435         '--no-override', action='store_true',
 436         help='skip override json in commit generation (default: %(default)s)')
 437     parser.add_argument(
 438         '--override-path', type=Path, default=LOCATION_PATH / 'changelog_override.json',
 439         help='path to the changelog_override.json file')
 440     parser.add_argument(
 441         '--default-author', default='pukkandan',
 442         help='the author to use without a author indicator (default: %(default)s)')
 443     parser.add_argument(
 444         '--repo', default='yt-dlp/yt-dlp',
 445         help='the github repository to use for the operations (default: %(default)s)')
 446     args = parser.parse_args()
 447
 448     logging.basicConfig(
 449         datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}',
 450         level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr)
 451
 452     commits = CommitRange(None, args.commitish, args.default_author)
 453
 454     if not args.no_override:
 455         if args.override_path.exists():
 456             with args.override_path.open() as file:
 457                 overrides = json.load(file)
 458             commits.apply_overrides(overrides)
 459         else:
 460             logger.warning(f'File {args.override_path.as_posix()} does not exist')
 461
 462     logger.info(f'Loaded {len(commits)} commits')
 463
 464     new_contributors = get_new_contributors(args.contributors_path, commits)
 465     if new_contributors:
 466         if args.contributors:
 467             with args.contributors_path.open('a') as file:
 468                 file.writelines(f'{contributor}\n' for contributor in new_contributors)
 469         logger.info(f'New contributors: {", ".join(new_contributors)}')
 470
 471     print(Changelog(commits.groups(), args.repo))