1 from __future__
import annotations
3 # Allow direct execution
7 sys
.path
.insert(0, os
.path
.dirname(os
.path
.dirname(os
.path
.abspath(__file__
))))
14 from collections
import defaultdict
15 from dataclasses
import dataclass
16 from functools
import lru_cache
17 from pathlib
import Path
19 from devscripts
.utils
import read_file
, run_process
, write_file
21 BASE_URL
= 'https://github.com'
22 LOCATION_PATH
= Path(__file__
).parent
25 logger
= logging
.getLogger(__name__
)
28 class CommitGroup(enum
.Enum
):
29 PRIORITY
= 'Important'
31 EXTRACTOR
= 'Extractor'
32 DOWNLOADER
= 'Downloader'
33 POSTPROCESSOR
= 'Postprocessor'
34 NETWORKING
= 'Networking'
39 def subgroup_lookup(cls
):
74 def group_lookup(cls
):
78 'pp': cls
.POSTPROCESSOR
,
81 result
.update({item.name.lower(): item for item in iter(cls)}
)
85 def get(cls
, value
: str) -> tuple[CommitGroup |
None, str |
None]:
86 group
, _
, subgroup
= (group
.strip().lower() for group
in value
.partition('/'))
88 result
= cls
.group_lookup().get(group
)
93 result
= cls
.subgroup_lookup().get(subgroup
)
95 return result
, subgroup
or None
105 result
= f
'{self.short!r}'
108 result
+= f
' ({self.hash[:HASH_LENGTH]})'
111 authors
= ', '.join(self
.authors
)
112 result
+= f
' by {authors}'
120 sub_details
: tuple[str, ...]
127 return ((self
.details
or '').lower(), self
.sub_details
, self
.message
)
131 return sorted({item.strip().lower(): item for item in items if item}
.values())
135 MISC_RE
= re
.compile(r
'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re
.IGNORECASE
)
136 ALWAYS_SHOWN
= (CommitGroup
.PRIORITY
,)
138 def __init__(self
, groups
, repo
, collapsible
=False):
139 self
._groups
= groups
141 self
._collapsible
= collapsible
144 return '\n'.join(self
._format
_groups
(self
._groups
)).replace('\t', ' ')
146 def _format_groups(self
, groups
):
148 for item
in CommitGroup
:
149 if self
._collapsible
and item
not in self
.ALWAYS_SHOWN
and first
:
151 yield '\n<details><summary><h3>Changelog</h3></summary>\n'
155 yield self
.format_module(item
.value
, group
)
157 if self
._collapsible
:
160 def format_module(self
, name
, group
):
161 result
= f
'\n#### {name} changes\n' if name
else '\n'
162 return result
+ '\n'.join(self
._format
_group
(group
))
164 def _format_group(self
, group
):
165 sorted_group
= sorted(group
, key
=CommitInfo
.key
)
166 detail_groups
= itertools
.groupby(sorted_group
, lambda item
: (item
.details
or '').lower())
167 for _
, items
in detail_groups
:
169 details
= items
[0].details
171 if details
== 'cleanup':
172 items
= self
._prepare
_cleanup
_misc
_items
(items
)
177 prefix
= f
'- **{details}**:'
179 yield f
'- **{details}**'
182 sub_detail_groups
= itertools
.groupby(items
, lambda item
: tuple(map(str.lower
, item
.sub_details
)))
183 for sub_details
, entries
in sub_detail_groups
:
185 for entry
in entries
:
186 yield f
'{prefix} {self.format_single_change(entry)}'
189 entries
= list(entries
)
190 sub_prefix
= f
'{prefix} {", ".join(entries[0].sub_details)}'
191 if len(entries
) == 1:
192 yield f
'{sub_prefix}: {self.format_single_change(entries[0])}'
196 for entry
in entries
:
197 yield f
'\t{prefix} {self.format_single_change(entry)}'
199 def _prepare_cleanup_misc_items(self
, items
):
200 cleanup_misc_items
= defaultdict(list)
203 if self
.MISC_RE
.search(item
.message
):
204 cleanup_misc_items
[tuple(item
.commit
.authors
)].append(item
)
206 sorted_items
.append(item
)
208 for commit_infos
in cleanup_misc_items
.values():
209 sorted_items
.append(CommitInfo(
210 'cleanup', ('Miscellaneous',), ', '.join(
211 self
._format
_message
_link
(None, info
.commit
.hash)
212 for info
in sorted(commit_infos
, key
=lambda item
: item
.commit
.hash or '')),
213 [], Commit(None, '', commit_infos
[0].commit
.authors
), []))
217 def format_single_change(self
, info
: CommitInfo
):
218 message
, sep
, rest
= info
.message
.partition('\n')
219 if '[' not in message
:
220 # If the message doesn't already contain markdown links, try to add a link to the commit
221 message
= self
._format
_message
_link
(message
, info
.commit
.hash)
224 message
= f
'{message} ({self._format_issues(info.issues)})'
226 if info
.commit
.authors
:
227 message
= f
'{message} by {self._format_authors(info.commit.authors)}'
230 fix_message
= ', '.join(f
'{self._format_message_link(None, fix.hash)}' for fix
in info
.fixes
)
232 authors
= sorted({author for fix in info.fixes for author in fix.authors}
, key
=str.casefold
)
233 if authors
!= info
.commit
.authors
:
234 fix_message
= f
'{fix_message} by {self._format_authors(authors)}'
236 message
= f
'{message} (With fixes in {fix_message})'
238 return message
if not sep
else f
'{message}{sep}{rest}'
240 def _format_message_link(self
, message
, hash):
241 assert message
or hash, 'Improperly defined commit message or override'
242 message
= message
if message
else hash[:HASH_LENGTH
]
243 return f
'[{message}]({self.repo_url}/commit/{hash})' if hash else message
245 def _format_issues(self
, issues
):
246 return ', '.join(f
'[#{issue}]({self.repo_url}/issues/{issue})' for issue
in issues
)
249 def _format_authors(authors
):
250 return ', '.join(f
'[{author}]({BASE_URL}/{author})' for author
in authors
)
254 return f
'{BASE_URL}/{self._repo}'
259 COMMIT_SEPARATOR
= '-----'
261 AUTHOR_INDICATOR_RE
= re
.compile(r
'Authored by:? ', re
.IGNORECASE
)
262 MESSAGE_RE
= re
.compile(r
'''
263 (?:\[(?P<prefix>[^\]]+)\]\ )?
264 (?:(?P<sub_details>`?[\w.-]+`?): )?
266 (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
267 ''', re
.VERBOSE | re
.DOTALL
)
268 EXTRACTOR_INDICATOR_RE
= re
.compile(r
'(?:Fix|Add)\s+Extractors?', re
.IGNORECASE
)
269 REVERT_RE
= re
.compile(r
'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})')
270 FIXES_RE
= re
.compile(r
'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})')
271 UPSTREAM_MERGE_RE
= re
.compile(r
'Update to ytdl-commit-([\da-f]+)')
273 def __init__(self
, start
, end
, default_author
=None):
274 self
._start
, self
._end
= start
, end
275 self
._commits
, self
._fixes
= self
._get
_commits
_and
_fixes
(default_author
)
276 self
._commits
_added
= []
279 return iter(itertools
.chain(self
._commits
.values(), self
._commits
_added
))
282 return len(self
._commits
) + len(self
._commits
_added
)
284 def __contains__(self
, commit
):
285 if isinstance(commit
, Commit
):
290 return commit
in self
._commits
292 def _get_commits_and_fixes(self
, default_author
):
293 result
= run_process(
294 self
.COMMAND
, 'log', f
'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}',
295 f
'{self._start}..{self._end}' if self
._start
else self
._end
).stdout
297 commits
, reverts
= {}, {}
298 fixes
= defaultdict(list)
299 lines
= iter(result
.splitlines(False))
300 for i
, commit_hash
in enumerate(lines
):
302 skip
= short
.startswith('Release ') or short
== '[version] update'
304 authors
= [default_author
] if default_author
else []
305 for line
in iter(lambda: next(lines
), self
.COMMIT_SEPARATOR
):
306 match
= self
.AUTHOR_INDICATOR_RE
.match(line
)
308 authors
= sorted(map(str.strip
, line
[match
.end():].split(',')), key
=str.casefold
)
310 commit
= Commit(commit_hash
, short
, authors
)
311 if skip
and (self
._start
or not i
):
312 logger
.debug(f
'Skipped commit: {commit}')
315 logger
.debug(f
'Reached Release commit, breaking: {commit}')
318 revert_match
= self
.REVERT_RE
.fullmatch(commit
.short
)
320 reverts
[revert_match
.group(1)] = commit
323 fix_match
= self
.FIXES_RE
.search(commit
.short
)
325 commitish
= fix_match
.group(1)
326 fixes
[commitish
].append(commit
)
328 commits
[commit
.hash] = commit
330 for commitish
, revert_commit
in reverts
.items():
331 reverted
= commits
.pop(commitish
, None)
333 logger
.debug(f
'{commitish} fully reverted {reverted}')
335 commits
[revert_commit
.hash] = revert_commit
337 for commitish
, fix_commits
in fixes
.items():
338 if commitish
in commits
:
339 hashes
= ', '.join(commit
.hash[:HASH_LENGTH
] for commit
in fix_commits
)
340 logger
.info(f
'Found fix(es) for {commitish[:HASH_LENGTH]}: {hashes}')
341 for fix_commit
in fix_commits
:
342 del commits
[fix_commit
.hash]
344 logger
.debug(f
'Commit with fixes not in changes: {commitish[:HASH_LENGTH]}')
346 return commits
, fixes
348 def apply_overrides(self
, overrides
):
349 for override
in overrides
:
350 when
= override
.get('when')
351 if when
and when
not in self
and when
!= self
._start
:
352 logger
.debug(f
'Ignored {when!r} override')
355 override_hash
= override
.get('hash') or when
356 if override
['action'] == 'add':
357 commit
= Commit(override
.get('hash'), override
['short'], override
.get('authors') or [])
358 logger
.info(f
'ADD {commit}')
359 self
._commits
_added
.append(commit
)
361 elif override
['action'] == 'remove':
362 if override_hash
in self
._commits
:
363 logger
.info(f
'REMOVE {self._commits[override_hash]}')
364 del self
._commits
[override_hash
]
366 elif override
['action'] == 'change':
367 if override_hash
not in self
._commits
:
369 commit
= Commit(override_hash
, override
['short'], override
.get('authors') or [])
370 logger
.info(f
'CHANGE {self._commits[commit.hash]} -> {commit}')
371 self
._commits
[commit
.hash] = commit
373 self
._commits
= {key: value for key, value in reversed(self._commits.items())}
376 group_dict
= defaultdict(list)
378 upstream_re
= self
.UPSTREAM_MERGE_RE
.search(commit
.short
)
380 commit
.short
= f
'[upstream] Merged with youtube-dl {upstream_re.group(1)}'
382 match
= self
.MESSAGE_RE
.fullmatch(commit
.short
)
384 logger
.error(f
'Error parsing short commit message: {commit.short!r}')
387 prefix
, sub_details_alt
, message
, issues
= match
.groups()
388 issues
= [issue
.strip()[1:] for issue
in issues
.split(',')] if issues
else []
391 groups
, details
, sub_details
= zip(*map(self
.details_from_prefix
, prefix
.split(',')))
392 group
= next(iter(filter(None, groups
)), None)
393 details
= ', '.join(unique(details
))
394 sub_details
= list(itertools
.chain
.from_iterable(sub_details
))
396 group
= CommitGroup
.CORE
401 sub_details
.append(sub_details_alt
)
402 sub_details
= tuple(unique(sub_details
))
405 if self
.EXTRACTOR_INDICATOR_RE
.search(commit
.short
):
406 group
= CommitGroup
.EXTRACTOR
408 group
= CommitGroup
.POSTPROCESSOR
409 logger
.warning(f
'Failed to map {commit.short!r}, selected {group.name.lower()}')
411 commit_info
= CommitInfo(
412 details
, sub_details
, message
.strip(),
413 issues
, commit
, self
._fixes
[commit
.hash])
415 logger
.debug(f
'Resolved {commit.short!r} to {commit_info!r}')
416 group_dict
[group
].append(commit_info
)
421 def details_from_prefix(prefix
):
423 return CommitGroup
.CORE
, None, ()
425 prefix
, *sub_details
= prefix
.split(':')
427 group
, details
= CommitGroup
.get(prefix
)
428 if group
is CommitGroup
.PRIORITY
and details
:
429 details
= details
.partition('/')[2].strip()
431 if details
and '/' in details
:
432 logger
.error(f
'Prefix is overnested, using first part: {prefix}')
433 details
= details
.partition('/')[0].strip()
435 if details
== 'common':
437 elif group
is CommitGroup
.NETWORKING
and details
== 'rh':
438 details
= 'Request Handler'
440 return group
, details
, sub_details
443 def get_new_contributors(contributors_path
, commits
):
445 if contributors_path
.exists():
446 for line
in read_file(contributors_path
).splitlines():
447 author
, _
, _
= line
.strip().partition(' (')
448 authors
= author
.split('/')
449 contributors
.update(map(str.casefold
, authors
))
451 new_contributors
= set()
452 for commit
in commits
:
453 for author
in commit
.authors
:
454 author_folded
= author
.casefold()
455 if author_folded
not in contributors
:
456 contributors
.add(author_folded
)
457 new_contributors
.add(author
)
459 return sorted(new_contributors
, key
=str.casefold
)
462 if __name__
== '__main__':
465 parser
= argparse
.ArgumentParser(
466 description
='Create a changelog markdown from a git commit range')
468 'commitish', default
='HEAD', nargs
='?',
469 help='The commitish to create the range from (default: %(default)s)')
471 '-v', '--verbosity', action
='count', default
=0,
472 help='increase verbosity (can be used twice)')
474 '-c', '--contributors', action
='store_true',
475 help='update CONTRIBUTORS file (default: %(default)s)')
477 '--contributors-path', type=Path
, default
=LOCATION_PATH
.parent
/ 'CONTRIBUTORS',
478 help='path to the CONTRIBUTORS file')
480 '--no-override', action
='store_true',
481 help='skip override json in commit generation (default: %(default)s)')
483 '--override-path', type=Path
, default
=LOCATION_PATH
/ 'changelog_override.json',
484 help='path to the changelog_override.json file')
486 '--default-author', default
='pukkandan',
487 help='the author to use without a author indicator (default: %(default)s)')
489 '--repo', default
='yt-dlp/yt-dlp',
490 help='the github repository to use for the operations (default: %(default)s)')
492 '--collapsible', action
='store_true',
493 help='make changelog collapsible (default: %(default)s)')
494 args
= parser
.parse_args()
497 datefmt
='%Y-%m-%d %H-%M-%S', format
='{asctime} | {levelname:<8} | {message}',
498 level
=logging
.WARNING
- 10 * args
.verbosity
, style
='{', stream
=sys
.stderr
)
500 commits
= CommitRange(None, args
.commitish
, args
.default_author
)
502 if not args
.no_override
:
503 if args
.override_path
.exists():
504 overrides
= json
.loads(read_file(args
.override_path
))
505 commits
.apply_overrides(overrides
)
507 logger
.warning(f
'File {args.override_path.as_posix()} does not exist')
509 logger
.info(f
'Loaded {len(commits)} commits')
511 new_contributors
= get_new_contributors(args
.contributors_path
, commits
)
513 if args
.contributors
:
514 write_file(args
.contributors_path
, '\n'.join(new_contributors
) + '\n', mode
='a')
515 logger
.info(f
'New contributors: {", ".join(new_contributors)}')
517 print(Changelog(commits
.groups(), args
.repo
, args
.collapsible
))