From 8f349dcf8eaf0643f525bfdb66cc894bd5f67ece Mon Sep 17 00:00:00 2001 From: deltazefiro Date: Wed, 26 Nov 2025 18:55:18 +0800 Subject: [PATCH] ci: Add script to generate translation contributor summary from git commits --- .../scripts/summarize_translation_contrib.py | 387 ++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100755 .github/scripts/summarize_translation_contrib.py diff --git a/.github/scripts/summarize_translation_contrib.py b/.github/scripts/summarize_translation_contrib.py new file mode 100755 index 0000000..d085fd9 --- /dev/null +++ b/.github/scripts/summarize_translation_contrib.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +""" +Generate translation contributor summary from git commits. +Default: from last tag to HEAD. +""" + +import re +import sys +import os +import subprocess +import argparse +import json +import asyncio +from collections import defaultdict +from typing import Dict, List, Set, Tuple, Optional + + +def get_last_tag() -> str: + """Get the most recent git tag.""" + try: + result = subprocess.run( + ["git", "describe", "--tags", "--abbrev=0"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError: + return None + + +def get_commit_list(start: str, end: str) -> List[str]: + """Get list of commits between start and end.""" + commit_range = f"{start}..{end}" if start else end + + try: + result = subprocess.run( + ["git", "log", "--format=%H", "--grep=Translation", commit_range], + capture_output=True, + text=True, + check=True, + ) + commits = [ + line.strip() for line in result.stdout.strip().split("\n") if line.strip() + ] + return commits + except subprocess.CalledProcessError as e: + print(f"Error: Failed to get commit list", file=sys.stderr) + print(f"Git error: {e.stderr}", file=sys.stderr) + sys.exit(1) + + +def get_commit_message(commit_hash: str) -> str: + """Get the full commit message.""" + try: + result = subprocess.run( + ["git", "show", "--format=%B", "--no-patch", commit_hash], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + except subprocess.CalledProcessError: + return "" + + +def extract_github_username_from_email(email: str) -> Optional[str]: + """Extract GitHub username from email address.""" + # GitHub noreply format: 25982450+VisionR1@users.noreply.github.com + pattern = r"(\d+\+)?([^@]+)@users\.noreply\.github\.com" + match = re.match(pattern, email) + if match: + return match.group(2) + + # Hosted Weblate noreply format + pattern = r"([^@]+)@users\.noreply\.hosted\.weblate\.org" + match = re.match(pattern, email) + if match: + return match.group(1) + + return None + + +# Global cache for API lookups +_email_to_username_cache = {} + + +async def lookup_github_user_by_email( + session, semaphore, email: str, verbose: bool = False +) -> Tuple[str, Optional[str]]: + """ + Look up GitHub username by email using GitHub API with async/await. + Returns (email, username) tuple. + """ + # Check cache first + if email in _email_to_username_cache: + return (email, _email_to_username_cache[email]) + + # Skip obvious non-GitHub emails + if email.endswith("@weblate.org") or "@" not in email: + _email_to_username_cache[email] = None + return (email, None) + + async with semaphore: + try: + # Use GitHub search API + query = email.replace("@", "%40").replace("+", "%2B") + url = f"https://api.github.com/search/users?q={query}+in:email" + + headers = { + "Accept": "application/vnd.github.v3+json", + "User-Agent": "Amarok-Translation-Contrib-Script", + } + + # Add GitHub token if available from environment + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"token {github_token}" + + # Use urllib for simplicity (no external dependencies) + import urllib.request + + req = urllib.request.Request(url, headers=headers) + + # Run in executor to avoid blocking + loop = asyncio.get_event_loop() + response = await loop.run_in_executor( + None, lambda: urllib.request.urlopen(req, timeout=10) + ) + + data = json.loads(response.read().decode()) + + if data.get("total_count", 0) > 0: + items = data.get("items", []) + if items: + username = items[0].get("login") + _email_to_username_cache[email] = username + if verbose: + print(f" API: {email} -> @{username}", file=sys.stderr) + return (email, username) + + _email_to_username_cache[email] = None + return (email, None) + + except Exception as e: + if verbose and "403" in str(e): + print(f" API: Rate limit or auth issue for {email}", file=sys.stderr) + _email_to_username_cache[email] = None + return (email, None) + + +async def batch_lookup_emails( + emails: Set[str], verbose: bool = False, max_concurrent: int = 1 +) -> Dict[str, Optional[str]]: + """ + Batch lookup multiple emails concurrently with semaphore. + Returns dict mapping email to username. + """ + semaphore = asyncio.Semaphore(max_concurrent) + + # Create a dummy session object (we're using urllib in executor) + session = None + + tasks = [ + lookup_github_user_by_email(session, semaphore, email, verbose) + for email in emails + ] + + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Build result dict + email_to_username = {} + for result in results: + if isinstance(result, tuple): + email, username = result + email_to_username[email] = username + else: + # Handle exceptions + if verbose: + print(f" API: Error during lookup: {result}", file=sys.stderr) + + return email_to_username + + +def extract_co_authors(commit_msg: str) -> List[Tuple[str, str]]: + """Extract co-authors from commit message.""" + pattern = r"Co-authored-by:\s*(.+?)\s*<(.+?)>" + matches = re.findall(pattern, commit_msg) + return matches + + +def extract_language_from_line(line: str) -> Optional[str]: + """Extract language from a commit message line.""" + pattern = r"translation(?:\s+update)?:\s+([^(]+?)\s*\(by" + match = re.search(pattern, line) + if match: + return match.group(1).strip() + return None + + +def parse_translation_commits( + commit_list: List[str], verbose: bool = False +) -> Dict[str, Set[Tuple[str, Optional[str]]]]: + """ + Parse translation commits and organize contributors by language. + Returns: {language: {(name, github_username), ...}} + """ + contributors_by_lang = defaultdict(set) + email_to_name = {} + email_to_languages_map = defaultdict(set) + emails_to_lookup = set() + + # First pass: collect all emails and their mappings + for commit_hash in commit_list: + commit_msg = get_commit_message(commit_hash) + if not commit_msg: + continue + + lines = commit_msg.split("\n") + email_to_languages = defaultdict(set) + + # Map emails to languages + for line in lines: + language = extract_language_from_line(line) + if language: + email_pattern = r"<(.+?)>" + email_match = re.search(email_pattern, line) + if email_match: + email = email_match.group(1) + email_to_languages[email].add(language) + + # Extract co-authors + co_authors = extract_co_authors(commit_msg) + + for name, email in co_authors: + # Skip bot accounts + if email in ["hosted@weblate.org", "noreply-addon-languages@weblate.org"]: + continue + + email_to_name[email] = name + languages = email_to_languages.get(email, set()) + for lang in languages: + email_to_languages_map[email].add(lang) + + # Check if we need to lookup this email + github_username = extract_github_username_from_email(email) + if not github_username and email not in _email_to_username_cache: + emails_to_lookup.add(email) + + # Batch lookup all emails concurrently + if emails_to_lookup: + if verbose: + print( + f"Looking up {len(emails_to_lookup)} emails via GitHub API...", + file=sys.stderr, + ) + + email_to_username_results = asyncio.run( + batch_lookup_emails(emails_to_lookup, verbose=verbose) + ) + + # Second pass: build contributors by language + for email, name in email_to_name.items(): + # Try to extract username from email format first + github_username = extract_github_username_from_email(email) + + # If not found, use cached result from API lookup + if not github_username: + github_username = _email_to_username_cache.get(email) + + languages = email_to_languages_map.get(email, set()) + + for language in languages: + contributors_by_lang[language].add((name, github_username)) + + return contributors_by_lang + + +def format_output( + contributors_by_lang: Dict[str, Set[Tuple[str, Optional[str]]]], +) -> str: + """Format output as a single line for commit messages.""" + if not contributors_by_lang: + return "" + + parts = [] + + # Sort languages alphabetically + for language in sorted(contributors_by_lang.keys()): + contributors = contributors_by_lang[language] + + # Sort contributors and format mentions + mentions = [] + for name, gh_username in sorted(contributors, key=lambda x: x[0]): + if gh_username: + mentions.append(f"@{gh_username}") + else: + mentions.append(name) + + if mentions: + contributor_str = ", ".join(mentions) + parts.append(f"{language} (by {contributor_str})") + + if not parts: + return "" + + return "Update community translations: " + ", ".join(parts) + + +def main(): + parser = argparse.ArgumentParser( + description="Generate translation contributor summary for commit messages", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # From last tag to HEAD (default) + %(prog)s + + # From specific tag/commit to HEAD + %(prog)s --start v1.0.0 + + # Between two tags/commits + %(prog)s --start v1.0.0 --end v2.0.0 + + # From specific commit + %(prog)s --start abc123 + + # Verbose mode + %(prog)s --verbose + """, + ) + + parser.add_argument("--start", help="Start commit/tag (default: last tag)") + + parser.add_argument("--end", default="HEAD", help="End commit/tag (default: HEAD)") + + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Verbose output (shows API lookups and processing details). Set GITHUB_TOKEN environment variable for higher API rate limits.", + ) + + args = parser.parse_args() + + # Determine start point + start = args.start + if not start: + start = get_last_tag() + if not start: + print("Error: No tags found and no --start specified", file=sys.stderr) + sys.exit(1) + if args.verbose: + print(f"Using last tag as start: {start}", file=sys.stderr) + + if args.verbose: + print(f"Processing commits from {start} to {args.end}...", file=sys.stderr) + + # Get commit list + commit_list = get_commit_list(start, args.end) + + if not commit_list: + if args.verbose: + print("No translation commits found in range", file=sys.stderr) + sys.exit(0) + + if args.verbose: + print(f"Found {len(commit_list)} translation commit(s)", file=sys.stderr) + + # Parse commits + contributors_by_lang = parse_translation_commits(commit_list, verbose=args.verbose) + + if not contributors_by_lang: + if args.verbose: + print("No contributors found", file=sys.stderr) + sys.exit(0) + + # Format and print output + output = format_output(contributors_by_lang) + if output: + print(output) + elif args.verbose: + print("No output generated", file=sys.stderr) + + +if __name__ == "__main__": + main()