From ccd169caed4b5b63d3f60ddc8c7fb45a14e32857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 18 Jan 2023 09:16:31 +0100 Subject: [PATCH] GH-14997: [Release] Ensure archery release tasks works with both new style GitHub issues and old style JIRA issues (#33615) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I've decided to do all the archery release tasks on a single PR: * Closes: #14997 * Closes: #14999 * Closes: #15002 Authored-by: Raúl Cumplido Signed-off-by: Raúl Cumplido --- dev/archery/archery/release/cli.py | 41 +-- dev/archery/archery/release/core.py | 258 ++++++++++++------ dev/archery/archery/release/reports.py | 7 +- .../archery/release/tests/test_release.py | 91 +++--- .../archery/templates/release_changelog.md.j2 | 4 + .../archery/templates/release_curation.txt.j2 | 20 +- dev/archery/setup.py | 2 +- 7 files changed, 278 insertions(+), 145 deletions(-) diff --git a/dev/archery/archery/release/cli.py b/dev/archery/archery/release/cli.py index 4fbf93861e6f1..ed15dcb1ed6dc 100644 --- a/dev/archery/archery/release/cli.py +++ b/dev/archery/archery/release/cli.py @@ -20,34 +20,33 @@ import click from ..utils.cli import validate_arrow_sources -from .core import Jira, CachedJira, Release +from .core import IssueTracker, Release @click.group('release') @click.option("--src", metavar="", default=None, callback=validate_arrow_sources, help="Specify Arrow source directory.") -@click.option("--jira-cache", type=click.Path(), default=None, - help="File path to cache queried JIRA issues per version.") +@click.option('--github-token', '-t', default=None, + envvar="CROSSBOW_GITHUB_TOKEN", + help='OAuth token for GitHub authentication') @click.pass_obj -def release(obj, src, jira_cache): +def release(obj, src, github_token): """Release releated commands.""" - jira = Jira() - if jira_cache is not None: - jira = CachedJira(jira_cache, jira=jira) - obj['jira'] = jira + obj['issue_tracker'] = IssueTracker(github_token=github_token) obj['repo'] = src.path -@release.command('curate', help="Lists release related Jira issues.") +@release.command('curate', help="Lists release related issues.") @click.argument('version') @click.option('--minimal/--full', '-m/-f', - help="Only show actionable Jira issues.", default=False) + help="Only show actionable issues.", default=False) @click.pass_obj def release_curate(obj, version, minimal): """Release curation.""" - release = Release.from_jira(version, jira=obj['jira'], repo=obj['repo']) + release = Release(version, repo=obj['repo'], + issue_tracker=obj['issue_tracker']) curation = release.curate(minimal) click.echo(curation.render('console')) @@ -64,10 +63,10 @@ def release_changelog(): @click.pass_obj def release_changelog_add(obj, version): """Prepend the changelog with the current release""" - jira, repo = obj['jira'], obj['repo'] + repo, issue_tracker = obj['repo'], obj['issue_tracker'] # just handle the current version - release = Release.from_jira(version, jira=jira, repo=repo) + release = Release(version, repo=repo, issue_tracker=issue_tracker) if release.is_released: raise ValueError('This version has been already released!') @@ -87,10 +86,10 @@ def release_changelog_add(obj, version): @click.pass_obj def release_changelog_generate(obj, version, output): """Generate the changelog of a specific release.""" - jira, repo = obj['jira'], obj['repo'] + repo, issue_tracker = obj['repo'], obj['issue_tracker'] # just handle the current version - release = Release.from_jira(version, jira=jira, repo=repo) + release = Release(version, repo=repo, issue_tracker=issue_tracker) changelog = release.changelog() output.write(changelog.render('markdown')) @@ -100,13 +99,15 @@ def release_changelog_generate(obj, version, output): @click.pass_obj def release_changelog_regenerate(obj): """Regeneretate the whole CHANGELOG.md file""" - jira, repo = obj['jira'], obj['repo'] + issue_tracker, repo = obj['issue_tracker'], obj['repo'] changelogs = [] + issue_tracker = IssueTracker(issue_tracker=issue_tracker) - for version in jira.project_versions('ARROW'): + for version in issue_tracker.project_versions(): if not version.released: continue - release = Release.from_jira(version, jira=jira, repo=repo) + release = Release(version, repo=repo, + issue_tracker=issue_tracker) click.echo('Querying changelog for version: {}'.format(version)) changelogs.append(release.changelog()) @@ -129,7 +130,9 @@ def release_cherry_pick(obj, version, dry_run, recreate): """ Cherry pick commits. """ - release = Release.from_jira(version, jira=obj['jira'], repo=obj['repo']) + issue_tracker = obj['issue_tracker'] + release = Release(version, + repo=obj['repo'], issue_tracker=issue_tracker) if not dry_run: release.cherry_pick_commits(recreate_branch=recreate) diff --git a/dev/archery/archery/release/core.py b/dev/archery/archery/release/core.py index 03eceb80a1034..822d408f88c8c 100644 --- a/dev/archery/archery/release/core.py +++ b/dev/archery/archery/release/core.py @@ -21,16 +21,16 @@ import os import pathlib import re -import shelve import warnings from git import Repo +from github import Github from jira import JIRA from semver import VersionInfo as SemVer from ..utils.source import ArrowSources from ..utils.logger import logger -from .reports import ReleaseCuration, JiraChangelog +from .reports import ReleaseCuration, ReleaseChangelog def cached_property(fn): @@ -58,13 +58,29 @@ def from_jira(cls, jira_version): release_date=getattr(jira_version, 'releaseDate', None) ) + @classmethod + def from_milestone(cls, milestone): + return cls.parse( + milestone.title, + released=milestone.state == "closed", + release_date=milestone.due_on + ) + + +ORIGINAL_ARROW_REGEX = re.compile( + r"\*This issue was originally created as " + + r"\[(?PARROW\-(?P(\d+)))\]" +) + class Issue: - def __init__(self, key, type, summary): + def __init__(self, key, type, summary, github_issue=None): self.key = key self.type = type self.summary = summary + self.github_issue_id = getattr(github_issue, "number", None) + self._github_issue = github_issue @classmethod def from_jira(cls, jira_issue): @@ -74,13 +90,49 @@ def from_jira(cls, jira_issue): summary=jira_issue.fields.summary ) + @classmethod + def from_github(cls, github_issue): + original_jira = cls.original_jira_id(github_issue) + key = original_jira or github_issue.number + return cls( + key=key, + type=next( + iter( + [ + label.name for label in github_issue.labels + if label.name.startswith("Type:") + ] + ), None), + summary=github_issue.title, + github_issue=github_issue + ) + @property def project(self): + if isinstance(self.key, int): + return 'GH' return self.key.split('-')[0] @property def number(self): - return int(self.key.split('-')[1]) + if isinstance(self.key, str): + return int(self.key.split('-')[1]) + else: + return self.key + + @cached_property + def is_pr(self): + return bool(self._github_issue and self._github_issue.pull_request) + + @classmethod + def original_jira_id(cls, github_issue): + # All migrated issues contain body + if not github_issue.body: + return None + matches = ORIGINAL_ARROW_REGEX.search(github_issue.body) + if matches: + values = matches.groupdict() + return values['issue'] class Jira(JIRA): @@ -88,54 +140,54 @@ class Jira(JIRA): def __init__(self, url='https://issues.apache.org/jira'): super().__init__(url) - def project_version(self, version_string, project='ARROW'): - # query version from jira to populated with additional metadata - versions = {str(v): v for v in self.project_versions(project)} - return versions[version_string] + def issue(self, key): + return Issue.from_jira(super().issue(key)) + + +class IssueTracker: + + def __init__(self, github_token=None): + github = Github(github_token) + self.github_repo = github.get_repo('apache/arrow') - def project_versions(self, project): + def project_version(self, version_string): + for milestone in self.project_versions(): + if milestone == version_string: + return milestone + + def project_versions(self): versions = [] - for v in super().project_versions(project): + milestones = self.github_repo.get_milestones(state="all") + for milestone in milestones: try: - versions.append(Version.from_jira(v)) + versions.append(Version.from_milestone(milestone)) except ValueError: # ignore invalid semantic versions like JS-0.4.0 continue return sorted(versions, reverse=True) - def issue(self, key): - return Issue.from_jira(super().issue(key)) - - def project_issues(self, version, project='ARROW'): - query = "project={} AND fixVersion={}".format(project, version) - issues = super().search_issues(query, maxResults=False) - return list(map(Issue.from_jira, issues)) - - -class CachedJira: - - def __init__(self, cache_path, jira=None): - self.jira = jira or Jira() - self.cache_path = cache_path + def _milestone_from_semver(self, semver): + milestones = self.github_repo.get_milestones(state="all") + for milestone in milestones: + try: + if milestone.title == semver: + return milestone + except ValueError: + # ignore invalid semantic versions like JS-0.3.0 + continue - def __getattr__(self, name): - attr = getattr(self.jira, name) - return self._cached(name, attr) if callable(attr) else attr + def project_issues(self, version): + issues = self.github_repo.get_issues( + milestone=self._milestone_from_semver(version), + state="all") + return list(map(Issue.from_github, issues)) - def _cached(self, name, method): - def wrapper(*args, **kwargs): - key = str((name, args, kwargs)) - with shelve.open(self.cache_path) as cache: - try: - result = cache[key] - except KeyError: - cache[key] = result = method(*args, **kwargs) - return result - return wrapper + def issue(self, key): + return Issue.from_github(self.github_repo.get_issue(key)) _TITLE_REGEX = re.compile( - r"(?P(?P(ARROW|PARQUET))\-\d+)?\s*:?\s*" + r"(?P(?P(ARROW|PARQUET|GH))\-(?P(\d+)))?\s*:?\s*" r"(?P(MINOR))?\s*:?\s*" r"(?P\[.*\])?\s*(?P.*)" ) @@ -145,9 +197,10 @@ def wrapper(*args, **kwargs): class CommitTitle: def __init__(self, summary, project=None, issue=None, minor=None, - components=None): + components=None, issue_id=None): self.project = project self.issue = issue + self.issue_id = issue_id self.components = components or [] self.summary = summary self.minor = bool(minor) @@ -186,6 +239,7 @@ def parse(cls, headline): values['summary'], project=values.get('project'), issue=values.get('issue'), + issue_id=values.get('issue_id'), minor=values.get('minor'), components=components ) @@ -230,7 +284,8 @@ def title(self): class Release: - def __new__(self, version, jira=None, repo=None): + def __new__(self, version, repo=None, github_token=None, + issue_tracker=None): if isinstance(version, str): version = Version.parse(version) elif not isinstance(version, Version): @@ -250,15 +305,7 @@ def __new__(self, version, jira=None, repo=None): return super().__new__(klass) - def __init__(self, version, jira, repo): - if jira is None: - jira = Jira() - elif isinstance(jira, str): - jira = Jira(jira) - elif not isinstance(jira, (Jira, CachedJira)): - raise TypeError("`jira` argument must be a server url or a valid " - "Jira instance") - + def __init__(self, version, repo, issue_tracker): if repo is None: arrow = ArrowSources.find() repo = Repo(arrow.path) @@ -269,13 +316,14 @@ def __init__(self, version, jira, repo): "instance") if isinstance(version, str): - version = jira.project_version(version, project='ARROW') + version = issue_tracker.project_version(version) + elif not isinstance(version, Version): raise TypeError(version) self.version = version - self.jira = jira self.repo = repo + self.issue_tracker = issue_tracker def __repr__(self): if self.version.released: @@ -284,10 +332,6 @@ def __repr__(self): status = "pending" return f"<{self.__class__.__name__} {self.version!r} {status}>" - @staticmethod - def from_jira(version, jira=None, repo=None): - return Release(version, jira, repo) - @property def is_released(self): return self.version.released @@ -322,7 +366,8 @@ def previous(self): # first release doesn't have a previous one return None else: - return Release.from_jira(previous, jira=self.jira, repo=self.repo) + return Release(previous, repo=self.repo, + issue_tracker=self.issue_tracker) @cached_property def next(self): @@ -332,13 +377,21 @@ def next(self): raise ValueError("There is no upcoming release set in JIRA after " f"version {self.version}") upcoming = self.siblings[position - 1] - return Release.from_jira(upcoming, jira=self.jira, repo=self.repo) + return Release(upcoming, repo=self.repo, + issue_tracker=self.issue_tracker) @cached_property def issues(self): - issues = self.jira.project_issues(self.version, project='ARROW') + issues = self.issue_tracker.project_issues( + self.version + ) return {i.key: i for i in issues} + @cached_property + def github_issue_ids(self): + return {v.github_issue_id for v in self.issues.values() + if v.github_issue_id} + @cached_property def commits(self): """ @@ -351,7 +404,11 @@ def commits(self): lower = self.repo.tags[self.previous.tag] if self.version.released: - upper = self.repo.tags[self.tag] + try: + upper = self.repo.tags[self.tag] + except IndexError: + warnings.warn(f"Release tag `{self.tag}` doesn't exist.") + return [] else: try: upper = self.repo.branches[self.branch] @@ -362,6 +419,10 @@ def commits(self): commit_range = f"{lower}..{upper}" return list(map(Commit, self.repo.iter_commits(commit_range))) + @cached_property + def jira_instance(self): + return Jira() + @cached_property def default_branch(self): default_branch_name = os.getenv("ARCHERY_DEFAULT_BRANCH") @@ -388,7 +449,7 @@ def default_branch(self): # The last token is the default branch name default_branch_name = origin_head_name_tokenized[-1] - except KeyError: + except (KeyError, IndexError): # Use a hard-coded default value to set default_branch_name # TODO: ARROW-18011 to track changing the hard coded default # value from "master" to "main". @@ -403,29 +464,43 @@ def default_branch(self): return default_branch_name def curate(self, minimal=False): - # handle commits with parquet issue key specially and query them from - # jira and add it to the issues + # handle commits with parquet issue key specially release_issues = self.issues - - within, outside, nojira, parquet = [], [], [], [] + within, outside, noissue, parquet, minor = [], [], [], [], [] for c in self.commits: if c.issue is None: - nojira.append(c) - elif c.issue in release_issues: - within.append((release_issues[c.issue], c)) + if c.title.minor: + minor.append(c) + else: + noissue.append(c) + elif c.project == 'GH': + if int(c.issue_id) in release_issues: + within.append((release_issues[int(c.issue_id)], c)) + else: + outside.append( + (self.issue_tracker.issue(int(c.issue_id)), c)) + elif c.project == 'ARROW': + if c.issue in release_issues: + within.append((release_issues[c.issue], c)) + else: + outside.append((self.jira_instance.issue(c.issue), c)) elif c.project == 'PARQUET': - parquet.append((self.jira.issue(c.issue), c)) + parquet.append((self.jira_instance.issue(c.issue), c)) else: - outside.append((self.jira.issue(c.issue), c)) + warnings.warn( + f'Issue {c.issue} is not MINOR nor pertains to GH' + + ', ARROW or PARQUET') + outside.append((c.issue, c)) # remaining jira tickets within_keys = {i.key for i, c in within} + # Take into account that some issues milestoned are prs nopatch = [issue for key, issue in release_issues.items() - if key not in within_keys] + if key not in within_keys and issue.is_pr is False] return ReleaseCuration(release=self, within=within, outside=outside, - nojira=nojira, parquet=parquet, nopatch=nopatch, - minimal=minimal) + noissue=noissue, parquet=parquet, + nopatch=nopatch, minimal=minimal, minor=minor) def changelog(self): issue_commit_pairs = [] @@ -451,16 +526,26 @@ def changelog(self): 'Task': 'New Features and Improvements', 'Test': 'Bug Fixes', 'Wish': 'New Features and Improvements', + 'Type: bug': 'Bug Fixes', + 'Type: enhancement': 'New Features and Improvements', + 'Type: task': 'New Features and Improvements', + 'Type: test': 'Bug Fixes', + 'Type: usage': 'New Features and Improvements', } categories = defaultdict(list) for issue, commit in issue_commit_pairs: - categories[issue_types[issue.type]].append((issue, commit)) + try: + categories[issue_types[issue.type]].append((issue, commit)) + except KeyError: + # If issue or pr don't have a type assume task. + # Currently the label for type is not mandatory on GitHub. + categories[issue_types['Type: task']].append((issue, commit)) # sort issues by the issue key in ascending order for issues in categories.values(): issues.sort(key=lambda pair: (pair[0].project, pair[0].number)) - return JiraChangelog(release=self, categories=categories) + return ReleaseChangelog(release=self, categories=categories) def commits_to_pick(self, exclude_already_applied=True): # collect commits applied on the default branch since the root of the @@ -481,10 +566,18 @@ def commits_to_pick(self, exclude_already_applied=True): # iterate over the commits applied on the main branch and filter out # the ones that are included in the jira release - patches_to_pick = [c for c in commits if - c.issue in self.issues and - c.title not in already_applied] - + patches_to_pick = [] + for c in commits: + key = c.issue + # For the release we assume all issues that have to be + # cherry-picked are merged with the GH issue id instead of the + # JIRA ARROW one. That's why we use github_issues along with + # issues. This is only to correct the mapping for migrated issues. + if c.issue and c.issue.startswith("GH-"): + key = int(c.issue_id) + if ((key in self.github_issue_ids or key in self.issues) and + c.title not in already_applied): + patches_to_pick.append(c) return reversed(patches_to_pick) def cherry_pick_commits(self, recreate_branch=True): @@ -525,7 +618,7 @@ def siblings(self): Filter only the major releases. """ # handle minor releases before 1.0 as major releases - return [v for v in self.jira.project_versions('ARROW') + return [v for v in self.issue_tracker.project_versions() if v.patch == 0 and (v.major == 0 or v.minor == 0)] @@ -544,7 +637,8 @@ def siblings(self): """ Filter the major and minor releases. """ - return [v for v in self.jira.project_versions('ARROW') if v.patch == 0] + return [v for v in self.issue_tracker.project_versions() + if v.patch == 0] class PatchRelease(Release): @@ -562,4 +656,4 @@ def siblings(self): """ No filtering, consider all releases. """ - return self.jira.project_versions('ARROW') + return self.issue_tracker.project_versions() diff --git a/dev/archery/archery/release/reports.py b/dev/archery/archery/release/reports.py index 43093487c023f..4299eaa7ede48 100644 --- a/dev/archery/archery/release/reports.py +++ b/dev/archery/archery/release/reports.py @@ -27,14 +27,15 @@ class ReleaseCuration(JinjaReport): 'release', 'within', 'outside', - 'nojira', + 'noissue', 'parquet', 'nopatch', - 'minimal' + 'minimal', + 'minor' ] -class JiraChangelog(JinjaReport): +class ReleaseChangelog(JinjaReport): templates = { 'markdown': 'release_changelog.md.j2', 'html': 'release_changelog.html.j2' diff --git a/dev/archery/archery/release/tests/test_release.py b/dev/archery/archery/release/tests/test_release.py index 1283b4bcb4f74..22b43c7cb3bc4 100644 --- a/dev/archery/archery/release/tests/test_release.py +++ b/dev/archery/archery/release/tests/test_release.py @@ -19,13 +19,29 @@ from archery.release.core import ( Release, MajorRelease, MinorRelease, PatchRelease, - Jira, Version, Issue, CommitTitle, Commit + IssueTracker, Version, Issue, CommitTitle, Commit ) from archery.testing import DotDict # subset of issues per revision _issues = { + "3.0.0": [ + Issue("GH-9784", type="Bug", summary="[C++] Title"), + Issue("GH-9767", type="New Feature", summary="[Crossbow] Title"), + Issue("GH-1231", type="Bug", summary="[Java] Title"), + Issue("GH-1244", type="Bug", summary="[C++] Title"), + Issue("GH-1301", type="Bug", summary="[Python][Archery] Title") + ], + "2.0.0": [ + Issue("ARROW-9784", type="Bug", summary="[Java] Title"), + Issue("ARROW-9767", type="New Feature", summary="[Crossbow] Title"), + Issue("GH-1230", type="Bug", summary="[Dev] Title"), + Issue("ARROW-9694", type="Bug", summary="[Release] Title"), + Issue("ARROW-5643", type="Bug", summary="[Go] Title"), + Issue("GH-1243", type="Bug", summary="[Python] Title"), + Issue("GH-1300", type="Bug", summary="[CI][Archery] Title") + ], "1.0.1": [ Issue("ARROW-9684", type="Bug", summary="[C++] Title"), Issue("ARROW-9667", type="New Feature", summary="[Crossbow] Title"), @@ -62,13 +78,14 @@ } -class FakeJira(Jira): +class FakeIssueTracker(IssueTracker): def __init__(self): pass - def project_versions(self, project='ARROW'): + def project_versions(self): return [ + Version.parse("4.0.0", released=False), Version.parse("3.0.0", released=False), Version.parse("2.0.0", released=False), Version.parse("1.1.0", released=False), @@ -82,16 +99,16 @@ def project_versions(self, project='ARROW'): Version.parse("0.15.0", released=True), ] - def project_issues(self, version, project='ARROW'): + def project_issues(self, version): return _issues[str(version)] @pytest.fixture -def fake_jira(): - return FakeJira() +def fake_issue_tracker(): + return FakeIssueTracker() -def test_version(fake_jira): +def test_version(fake_issue_tracker): v = Version.parse("1.2.5") assert str(v) == "1.2.5" assert v.major == 1 @@ -109,7 +126,7 @@ def test_version(fake_jira): assert v.release_date == "2020-01-01" -def test_issue(fake_jira): +def test_issue(fake_issue_tracker): i = Issue("ARROW-1234", type='Bug', summary="title") assert i.key == "ARROW-1234" assert i.type == "Bug" @@ -212,78 +229,78 @@ def test_commit_title(): assert t.minor is False -def test_release_basics(fake_jira): - r = Release.from_jira("1.0.0", jira=fake_jira) +def test_release_basics(fake_issue_tracker): + r = Release("1.0.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r, MajorRelease) assert r.is_released is True assert r.branch == 'maint-1.0.0' assert r.tag == 'apache-arrow-1.0.0' - r = Release.from_jira("1.1.0", jira=fake_jira) + r = Release("1.1.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r, MinorRelease) assert r.is_released is False assert r.branch == 'maint-1.x.x' assert r.tag == 'apache-arrow-1.1.0' # minor releases before 1.0 are treated as major releases - r = Release.from_jira("0.17.0", jira=fake_jira) + r = Release("0.17.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r, MajorRelease) assert r.is_released is True assert r.branch == 'maint-0.17.0' assert r.tag == 'apache-arrow-0.17.0' - r = Release.from_jira("0.17.1", jira=fake_jira) + r = Release("0.17.1", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r, PatchRelease) assert r.is_released is True assert r.branch == 'maint-0.17.x' assert r.tag == 'apache-arrow-0.17.1' -def test_previous_and_next_release(fake_jira): - r = Release.from_jira("3.0.0", jira=fake_jira) +def test_previous_and_next_release(fake_issue_tracker): + r = Release("4.0.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r.previous, MajorRelease) - assert r.previous.version == Version.parse("2.0.0") + assert r.previous.version == Version.parse("3.0.0") with pytest.raises(ValueError, match="There is no upcoming release set"): assert r.next - r = Release.from_jira("2.0.0", jira=fake_jira) + r = Release("3.0.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r.previous, MajorRelease) assert isinstance(r.next, MajorRelease) - assert r.previous.version == Version.parse("1.0.0") - assert r.next.version == Version.parse("3.0.0") + assert r.previous.version == Version.parse("2.0.0") + assert r.next.version == Version.parse("4.0.0") - r = Release.from_jira("1.1.0", jira=fake_jira) + r = Release("1.1.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r.previous, MajorRelease) assert isinstance(r.next, MajorRelease) assert r.previous.version == Version.parse("1.0.0") assert r.next.version == Version.parse("2.0.0") - r = Release.from_jira("1.0.0", jira=fake_jira) + r = Release("1.0.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r.next, MajorRelease) assert isinstance(r.previous, MajorRelease) assert r.previous.version == Version.parse("0.17.0") assert r.next.version == Version.parse("2.0.0") - r = Release.from_jira("0.17.0", jira=fake_jira) + r = Release("0.17.0", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r.previous, MajorRelease) assert r.previous.version == Version.parse("0.16.0") - r = Release.from_jira("0.15.2", jira=fake_jira) + r = Release("0.15.2", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r.previous, PatchRelease) assert isinstance(r.next, MajorRelease) assert r.previous.version == Version.parse("0.15.1") assert r.next.version == Version.parse("0.16.0") - r = Release.from_jira("0.15.1", jira=fake_jira) + r = Release("0.15.1", repo=None, issue_tracker=fake_issue_tracker) assert isinstance(r.previous, MajorRelease) assert isinstance(r.next, PatchRelease) assert r.previous.version == Version.parse("0.15.0") assert r.next.version == Version.parse("0.15.2") -def test_release_issues(fake_jira): +def test_release_issues(fake_issue_tracker): # major release issues - r = Release.from_jira("1.0.0", jira=fake_jira) + r = Release("1.0.0", repo=None, issue_tracker=fake_issue_tracker) assert r.issues.keys() == set([ "ARROW-300", "ARROW-4427", @@ -295,7 +312,7 @@ def test_release_issues(fake_jira): "ARROW-8973" ]) # minor release issues - r = Release.from_jira("0.17.0", jira=fake_jira) + r = Release("0.17.0", repo=None, issue_tracker=fake_issue_tracker) assert r.issues.keys() == set([ "ARROW-2882", "ARROW-2587", @@ -305,7 +322,7 @@ def test_release_issues(fake_jira): "ARROW-1636", ]) # patch release issues - r = Release.from_jira("1.0.1", jira=fake_jira) + r = Release("1.0.1", repo=None, issue_tracker=fake_issue_tracker) assert r.issues.keys() == set([ "ARROW-9684", "ARROW-9667", @@ -315,6 +332,16 @@ def test_release_issues(fake_jira): "ARROW-9609", "ARROW-9606" ]) + r = Release("2.0.0", repo=None, issue_tracker=fake_issue_tracker) + assert r.issues.keys() == set([ + "ARROW-9784", + "ARROW-9767", + "GH-1230", + "ARROW-9694", + "ARROW-5643", + "GH-1243", + "GH-1300" + ]) @pytest.mark.parametrize(('version', 'ncommits'), [ @@ -323,8 +350,8 @@ def test_release_issues(fake_jira): ("0.17.0", 569), ("0.15.1", 41) ]) -def test_release_commits(fake_jira, version, ncommits): - r = Release.from_jira(version, jira=fake_jira) +def test_release_commits(fake_issue_tracker, version, ncommits): + r = Release(version, repo=None, issue_tracker=fake_issue_tracker) assert len(r.commits) == ncommits for c in r.commits: assert isinstance(c, Commit) @@ -332,8 +359,8 @@ def test_release_commits(fake_jira, version, ncommits): assert c.url.endswith(c.hexsha) -def test_maintenance_patch_selection(fake_jira): - r = Release.from_jira("0.17.1", jira=fake_jira) +def test_maintenance_patch_selection(fake_issue_tracker): + r = Release("0.17.1", repo=None, issue_tracker=fake_issue_tracker) shas_to_pick = [ c.hexsha for c in r.commits_to_pick(exclude_already_applied=False) diff --git a/dev/archery/archery/templates/release_changelog.md.j2 b/dev/archery/archery/templates/release_changelog.md.j2 index 0c9efbc42f7dc..0eedb217a8b84 100644 --- a/dev/archery/archery/templates/release_changelog.md.j2 +++ b/dev/archery/archery/templates/release_changelog.md.j2 @@ -23,7 +23,11 @@ ## {{ category }} {% for issue, commit in issue_commit_pairs -%} +{% if issue.project in ('ARROW', 'PARQUET') -%} * [{{ issue.key }}](https://issues.apache.org/jira/browse/{{ issue.key }}) - {{ commit.title.to_string(with_issue=False) if commit else issue.summary | md }} +{% else -%} +* [GH-{{ issue.key }}](https://github.com/apache/arrow/issues/{{ issue.key }}) - {{ commit.title.to_string(with_issue=False) if commit else issue.summary | md }} +{% endif -%} {% endfor %} {% endfor %} diff --git a/dev/archery/archery/templates/release_curation.txt.j2 b/dev/archery/archery/templates/release_curation.txt.j2 index 4f524d001ce3f..0796f451625f1 100644 --- a/dev/archery/archery/templates/release_curation.txt.j2 +++ b/dev/archery/archery/templates/release_curation.txt.j2 @@ -17,26 +17,30 @@ # under the License. #} {%- if not minimal -%} -Total number of JIRA tickets assigned to version {{ release.version }}: {{ release.issues|length }} +### Total number of GitHub tickets assigned to version {{ release.version }}: {{ release.issues|length }} -Total number of applied patches since version {{ release.previous.version }}: {{ release.commits|length }} +### Total number of applied patches since version {{ release.previous.version }}: {{ release.commits|length }} -Patches with assigned issue in version {{ release.version }}: +### Patches with assigned issue in version {{ release.version }}: {{ within|length }} {% for issue, commit in within -%} - {{ commit.url }} {{ commit.title }} {% endfor %} {% endif -%} -Patches with assigned issue outside of version {{ release.version }}: +### Patches with assigned issue outside of version {{ release.version }}: {{ outside|length }} {% for issue, commit in outside -%} - {{ commit.url }} {{ commit.title }} {% endfor %} {% if not minimal -%} -Patches in version {{ release.version }} without a linked issue: -{% for commit in nojira -%} +### Minor patches in version {{ release.version }}: {{ minor|length }} +{% for commit in minor -%} - {{ commit.url }} {{ commit.title }} {% endfor %} -JIRA issues in version {{ release.version }} without a linked patch: +### Patches in version {{ release.version }} without a linked issue: +{% for commit in noissue -%} + - {{ commit.url }} {{ commit.title }} +{% endfor %} +### JIRA issues in version {{ release.version }} without a linked patch: {{ nopatch|length }} {% for issue in nopatch -%} - - https://issues.apache.org/jira/browse/{{ issue.key }} + - https://github.com/apache/arrow/issues/{{ issue.key }} {% endfor %} {%- endif -%} \ No newline at end of file diff --git a/dev/archery/setup.py b/dev/archery/setup.py index 4b13608cf833b..51f066c9ede40 100755 --- a/dev/archery/setup.py +++ b/dev/archery/setup.py @@ -31,7 +31,7 @@ 'lint': ['numpydoc==1.1.0', 'autopep8', 'flake8', 'cmake_format==0.6.13'], 'benchmark': ['pandas'], 'docker': ['ruamel.yaml', 'python-dotenv'], - 'release': [jinja_req, 'jira', 'semver', 'gitpython'], + 'release': ['pygithub', jinja_req, 'jira', 'semver', 'gitpython'], 'crossbow': ['github3.py', jinja_req, 'pygit2>=1.6.0', 'requests', 'ruamel.yaml', 'setuptools_scm'], 'crossbow-upload': ['github3.py', jinja_req, 'ruamel.yaml',