from __future__ import annotations
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
import enum
import itertools
import json
import logging
import re
-import subprocess
-import sys
from collections import defaultdict
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
+from devscripts.utils import read_file, run_process, write_file
+
BASE_URL = 'https://github.com'
LOCATION_PATH = Path(__file__).parent
+HASH_LENGTH = 7
logger = logging.getLogger(__name__)
class CommitGroup(enum.Enum):
- UPSTREAM = None
PRIORITY = 'Important'
CORE = 'Core'
EXTRACTOR = 'Extractor'
DOWNLOADER = 'Downloader'
POSTPROCESSOR = 'Postprocessor'
+ NETWORKING = 'Networking'
MISC = 'Misc.'
@classmethod
@lru_cache
- def commit_lookup(cls):
+ def subgroup_lookup(cls):
return {
name: group
for group, names in {
- cls.PRIORITY: {''},
- cls.UPSTREAM: {'upstream'},
- cls.CORE: {
- 'aes',
- 'cache',
- 'compat_utils',
- 'compat',
- 'cookies',
- 'core',
- 'dependencies',
- 'jsinterp',
- 'outtmpl',
- 'plugins',
- 'update',
- 'utils',
- },
cls.MISC: {
'build',
+ 'ci',
'cleanup',
'devscripts',
'docs',
- 'misc',
'test',
},
- cls.EXTRACTOR: {'extractor', 'extractors'},
- cls.DOWNLOADER: {'downloader'},
- cls.POSTPROCESSOR: {'postprocessor'},
+ cls.NETWORKING: {
+ 'rh',
+ },
}.items()
for name in names
}
@classmethod
- def get(cls, value):
- result = cls.commit_lookup().get(value)
- if result:
- logger.debug(f'Mapped {value!r} => {result.name}')
+ @lru_cache
+ def group_lookup(cls):
+ result = {
+ 'fd': cls.DOWNLOADER,
+ 'ie': cls.EXTRACTOR,
+ 'pp': cls.POSTPROCESSOR,
+ 'upstream': cls.CORE,
+ }
+ result.update({item.name.lower(): item for item in iter(cls)})
return result
+ @classmethod
+ def get(cls, value: str) -> tuple[CommitGroup | None, str | None]:
+ group, _, subgroup = (group.strip().lower() for group in value.partition('/'))
+
+ result = cls.group_lookup().get(group)
+ if not result:
+ if subgroup:
+ return None, value
+ subgroup = group
+ result = cls.subgroup_lookup().get(subgroup)
+
+ return result, subgroup or None
+
@dataclass
class Commit:
result = f'{self.short!r}'
if self.hash:
- result += f' ({self.hash[:7]})'
+ result += f' ({self.hash[:HASH_LENGTH]})'
if self.authors:
authors = ', '.join(self.authors)
return ((self.details or '').lower(), self.sub_details, self.message)
+def unique(items):
+ return sorted({item.strip().lower(): item for item in items if item}.values())
+
+
class Changelog:
MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE)
+ ALWAYS_SHOWN = (CommitGroup.PRIORITY,)
- def __init__(self, groups, repo):
+ def __init__(self, groups, repo, collapsible=False):
self._groups = groups
self._repo = repo
+ self._collapsible = collapsible
def __str__(self):
return '\n'.join(self._format_groups(self._groups)).replace('\t', ' ')
def _format_groups(self, groups):
+ first = True
for item in CommitGroup:
+ if self._collapsible and item not in self.ALWAYS_SHOWN and first:
+ first = False
+ yield '\n<details><summary><h3>Changelog</h3></summary>\n'
+
group = groups[item]
if group:
yield self.format_module(item.value, group)
+ if self._collapsible:
+ yield '\n</details>'
+
def format_module(self, name, group):
result = f'\n#### {name} changes\n' if name else '\n'
return result + '\n'.join(self._format_group(group))
for _, items in detail_groups:
items = list(items)
details = items[0].details
- if not details:
- indent = ''
- else:
- yield f'- {details}'
- indent = '\t'
if details == 'cleanup':
- items, cleanup_misc_items = self._filter_cleanup_misc_items(items)
+ items = self._prepare_cleanup_misc_items(items)
+
+ prefix = '-'
+ if details:
+ if len(items) == 1:
+ prefix = f'- **{details}**:'
+ else:
+ yield f'- **{details}**'
+ prefix = '\t-'
sub_detail_groups = itertools.groupby(items, lambda item: tuple(map(str.lower, item.sub_details)))
for sub_details, entries in sub_detail_groups:
if not sub_details:
for entry in entries:
- yield f'{indent}- {self.format_single_change(entry)}'
+ yield f'{prefix} {self.format_single_change(entry)}'
continue
entries = list(entries)
- prefix = f'{indent}- {", ".join(entries[0].sub_details)}'
+ sub_prefix = f'{prefix} {", ".join(entries[0].sub_details)}'
if len(entries) == 1:
- yield f'{prefix}: {self.format_single_change(entries[0])}'
+ yield f'{sub_prefix}: {self.format_single_change(entries[0])}'
continue
- yield prefix
+ yield sub_prefix
for entry in entries:
- yield f'{indent}\t- {self.format_single_change(entry)}'
+ yield f'\t{prefix} {self.format_single_change(entry)}'
- if details == 'cleanup' and cleanup_misc_items:
- yield from self._format_cleanup_misc_sub_group(cleanup_misc_items)
-
- def _filter_cleanup_misc_items(self, items):
+ def _prepare_cleanup_misc_items(self, items):
cleanup_misc_items = defaultdict(list)
- non_misc_items = []
+ sorted_items = []
for item in items:
if self.MISC_RE.search(item.message):
cleanup_misc_items[tuple(item.commit.authors)].append(item)
else:
- non_misc_items.append(item)
-
- return non_misc_items, cleanup_misc_items
+ sorted_items.append(item)
- def _format_cleanup_misc_sub_group(self, group):
- prefix = '\t- Miscellaneous'
- if len(group) == 1:
- yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}'
- return
+ for commit_infos in cleanup_misc_items.values():
+ sorted_items.append(CommitInfo(
+ 'cleanup', ('Miscellaneous',), ', '.join(
+ self._format_message_link(None, info.commit.hash)
+ for info in sorted(commit_infos, key=lambda item: item.commit.hash or '')),
+ [], Commit(None, '', commit_infos[0].commit.authors), []))
- yield prefix
- for message in self._format_cleanup_misc_items(group):
- yield f'\t\t- {message}'
+ return sorted_items
- def _format_cleanup_misc_items(self, group):
- for authors, infos in group.items():
- message = ', '.join(
- self._format_message_link(None, info.commit.hash)
- for info in sorted(infos, key=lambda item: item.commit.hash or ''))
- yield f'{message} by {self._format_authors(authors)}'
+ def format_single_change(self, info: CommitInfo):
+ message, sep, rest = info.message.partition('\n')
+ if '[' not in message:
+ # If the message doesn't already contain markdown links, try to add a link to the commit
+ message = self._format_message_link(message, info.commit.hash)
- def format_single_change(self, info):
- message = self._format_message_link(info.message, info.commit.hash)
if info.issues:
message = f'{message} ({self._format_issues(info.issues)})'
message = f'{message} (With fixes in {fix_message})'
- return message
+ return message if not sep else f'{message}{sep}{rest}'
def _format_message_link(self, message, hash):
assert message or hash, 'Improperly defined commit message or override'
- message = message if message else hash[:7]
+ message = message if message else hash[:HASH_LENGTH]
return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message
def _format_issues(self, issues):
AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE)
MESSAGE_RE = re.compile(r'''
- (?:\[
- (?P<prefix>[^\]\/:,]+)
- (?:/(?P<details>[^\]:,]+))?
- (?:[:,](?P<sub_details>[^\]]+))?
- \]\ )?
- (?:(?P<sub_details_alt>`?[^:`]+`?): )?
+ (?:\[(?P<prefix>[^\]]+)\]\ )?
+ (?:(?P<sub_details>`?[\w.-]+`?): )?
(?P<message>.+?)
(?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
''', re.VERBOSE | re.DOTALL)
EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE)
+ REVERT_RE = re.compile(r'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})')
FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)')
- def __init__(self, start, end, default_author=None) -> None:
- self._start = start
- self._end = end
+ def __init__(self, start, end, default_author=None):
+ self._start, self._end = start, end
self._commits, self._fixes = self._get_commits_and_fixes(default_author)
self._commits_added = []
- @classmethod
- def from_single(cls, commitish='HEAD', default_author=None):
- start_commitish = cls.get_prev_tag(commitish)
- end_commitish = cls.get_next_tag(commitish)
- if start_commitish == end_commitish:
- start_commitish = cls.get_prev_tag(f'{commitish}~')
- logger.info(f'Determined range from {commitish!r}: {start_commitish}..{end_commitish}')
- return cls(start_commitish, end_commitish, default_author)
-
- @classmethod
- def get_prev_tag(cls, commitish):
- command = [cls.COMMAND, 'describe', '--tags', '--abbrev=0', '--exclude=*[^0-9.]*', commitish]
- return subprocess.check_output(command, text=True).strip()
-
- @classmethod
- def get_next_tag(cls, commitish):
- result = subprocess.run(
- [cls.COMMAND, 'describe', '--contains', '--abbrev=0', commitish],
- stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
- if result.returncode:
- return 'HEAD'
-
- return result.stdout.partition('~')[0].strip()
-
def __iter__(self):
return iter(itertools.chain(self._commits.values(), self._commits_added))
return commit in self._commits
- def _is_ancestor(self, commitish):
- return bool(subprocess.call(
- [self.COMMAND, 'merge-base', '--is-ancestor', commitish, self._start]))
-
def _get_commits_and_fixes(self, default_author):
- result = subprocess.check_output([
+ result = run_process(
self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}',
- f'{self._start}..{self._end}'], text=True)
+ f'{self._start}..{self._end}' if self._start else self._end).stdout
- commits = {}
+ commits, reverts = {}, {}
fixes = defaultdict(list)
lines = iter(result.splitlines(False))
- for line in lines:
- commit_hash = line
+ for i, commit_hash in enumerate(lines):
short = next(lines)
skip = short.startswith('Release ') or short == '[version] update'
authors = sorted(map(str.strip, line[match.end():].split(',')), key=str.casefold)
commit = Commit(commit_hash, short, authors)
- if skip:
+ if skip and (self._start or not i):
logger.debug(f'Skipped commit: {commit}')
continue
+ elif skip:
+ logger.debug(f'Reached Release commit, breaking: {commit}')
+ break
+
+ revert_match = self.REVERT_RE.fullmatch(commit.short)
+ if revert_match:
+ reverts[revert_match.group(1)] = commit
+ continue
fix_match = self.FIXES_RE.search(commit.short)
if fix_match:
commits[commit.hash] = commit
+ for commitish, revert_commit in reverts.items():
+ reverted = commits.pop(commitish, None)
+ if reverted:
+ logger.debug(f'{commitish} fully reverted {reverted}')
+ else:
+ commits[revert_commit.hash] = revert_commit
+
for commitish, fix_commits in fixes.items():
if commitish in commits:
- hashes = ', '.join(commit.hash[:7] for commit in fix_commits)
- logger.info(f'Found fix(es) for {commitish[:7]}: {hashes}')
+ hashes = ', '.join(commit.hash[:HASH_LENGTH] for commit in fix_commits)
+ logger.info(f'Found fix(es) for {commitish[:HASH_LENGTH]}: {hashes}')
for fix_commit in fix_commits:
del commits[fix_commit.hash]
else:
- logger.debug(f'Commit with fixes not in changes: {commitish[:7]}')
+ logger.debug(f'Commit with fixes not in changes: {commitish[:HASH_LENGTH]}')
return commits, fixes
for override in overrides:
when = override.get('when')
if when and when not in self and when != self._start:
- logger.debug(f'Ignored {when!r}, not in commits {self._start!r}')
+ logger.debug(f'Ignored {when!r} override')
continue
- override_hash = override.get('hash')
+ override_hash = override.get('hash') or when
if override['action'] == 'add':
commit = Commit(override.get('hash'), override['short'], override.get('authors') or [])
logger.info(f'ADD {commit}')
elif override['action'] == 'change':
if override_hash not in self._commits:
continue
- commit = Commit(override_hash, override['short'], override['authors'])
+ commit = Commit(override_hash, override['short'], override.get('authors') or [])
logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}')
self._commits[commit.hash] = commit
self._commits = {key: value for key, value in reversed(self._commits.items())}
def groups(self):
- groups = defaultdict(list)
+ group_dict = defaultdict(list)
for commit in self:
- upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short)
+ upstream_re = self.UPSTREAM_MERGE_RE.search(commit.short)
if upstream_re:
- commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}'
+ commit.short = f'[upstream] Merged with youtube-dl {upstream_re.group(1)}'
match = self.MESSAGE_RE.fullmatch(commit.short)
if not match:
logger.error(f'Error parsing short commit message: {commit.short!r}')
continue
- prefix, details, sub_details, sub_details_alt, message, issues = match.groups()
- group = None
- if prefix:
- if prefix == 'priority':
- prefix, _, details = (details or '').partition('/')
- logger.debug(f'Priority: {message!r}')
- group = CommitGroup.PRIORITY
-
- if not details and prefix:
- if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'):
- logger.debug(f'Replaced details with {prefix!r}')
- details = prefix or None
-
- if details == 'common':
- details = None
-
- if details:
- details = details.strip()
+ prefix, sub_details_alt, message, issues = match.groups()
+ issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []
+ if prefix:
+ groups, details, sub_details = zip(*map(self.details_from_prefix, prefix.split(',')))
+ group = next(iter(filter(None, groups)), None)
+ details = ', '.join(unique(details))
+ sub_details = list(itertools.chain.from_iterable(sub_details))
else:
group = CommitGroup.CORE
+ details = None
+ sub_details = []
- sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.replace(':', ',')
- sub_details = tuple(filter(None, map(str.strip, sub_details.split(','))))
-
- issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []
+ if sub_details_alt:
+ sub_details.append(sub_details_alt)
+ sub_details = tuple(unique(sub_details))
if not group:
- group = CommitGroup.get(prefix.lower())
- if not group:
- if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
- group = CommitGroup.EXTRACTOR
- else:
- group = CommitGroup.POSTPROCESSOR
- logger.warning(f'Failed to map {commit.short!r}, selected {group.name}')
+ if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
+ group = CommitGroup.EXTRACTOR
+ logger.error(f'Assuming [ie] group for {commit.short!r}')
+ else:
+ group = CommitGroup.CORE
commit_info = CommitInfo(
details, sub_details, message.strip(),
issues, commit, self._fixes[commit.hash])
+
logger.debug(f'Resolved {commit.short!r} to {commit_info!r}')
- groups[group].append(commit_info)
+ group_dict[group].append(commit_info)
+
+ return group_dict
+
+ @staticmethod
+ def details_from_prefix(prefix):
+ if not prefix:
+ return CommitGroup.CORE, None, ()
+
+ prefix, *sub_details = prefix.split(':')
+
+ group, details = CommitGroup.get(prefix)
+ if group is CommitGroup.PRIORITY and details:
+ details = details.partition('/')[2].strip()
- return groups
+ if details and '/' in details:
+ logger.error(f'Prefix is overnested, using first part: {prefix}')
+ details = details.partition('/')[0].strip()
+
+ if details == 'common':
+ details = None
+ elif group is CommitGroup.NETWORKING and details == 'rh':
+ details = 'Request Handler'
+
+ return group, details, sub_details
def get_new_contributors(contributors_path, commits):
contributors = set()
if contributors_path.exists():
- with contributors_path.open() as file:
- for line in filter(None, map(str.strip, file)):
- author, _, _ = line.partition(' (')
- authors = author.split('/')
- contributors.update(map(str.casefold, authors))
+ for line in read_file(contributors_path).splitlines():
+ author, _, _ = line.strip().partition(' (')
+ authors = author.split('/')
+ contributors.update(map(str.casefold, authors))
new_contributors = set()
for commit in commits:
parser.add_argument(
'--repo', default='yt-dlp/yt-dlp',
help='the github repository to use for the operations (default: %(default)s)')
+ parser.add_argument(
+ '--collapsible', action='store_true',
+ help='make changelog collapsible (default: %(default)s)')
args = parser.parse_args()
logging.basicConfig(
datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}',
level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr)
- commits = CommitRange.from_single(args.commitish, args.default_author)
+ commits = CommitRange(None, args.commitish, args.default_author)
if not args.no_override:
if args.override_path.exists():
- with args.override_path.open() as file:
- overrides = json.load(file)
+ overrides = json.loads(read_file(args.override_path))
commits.apply_overrides(overrides)
else:
logger.warning(f'File {args.override_path.as_posix()} does not exist')
new_contributors = get_new_contributors(args.contributors_path, commits)
if new_contributors:
if args.contributors:
- with args.contributors_path.open('a') as file:
- file.writelines(f'{contributor}\n' for contributor in new_contributors)
+ write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a')
logger.info(f'New contributors: {", ".join(new_contributors)}')
- print(Changelog(commits.groups(), args.repo))
+ print(Changelog(commits.groups(), args.repo, args.collapsible))