Skip to content

Commit 3dc59ec

Browse files
achingbrainNargonathgr2m
authoredMay 27, 2023
fix: use retry and throttle octokit plugins (#487)
Co-authored-by: Jonas Pauthier <jonas.pauthier@gmail.com> Co-authored-by: Gregor Martynus <39992+gr2m@users.noreply.github.com>
·
v11.0.1v8.0.8
1 parent 94a0a7b commit 3dc59ec

23 files changed

+1745
-18442
lines changed
 

‎lib/add-channel.js

+4-4
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ module.exports = async (pluginConfig, context) => {
1414
} = context;
1515
const {githubToken, githubUrl, githubApiPathPrefix, proxy} = resolveConfig(pluginConfig, context);
1616
const {owner, repo} = parseGithubUrl(repositoryUrl);
17-
const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
17+
const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
1818
let releaseId;
1919

2020
const release = {owner, repo, name, prerelease: isPrerelease(branch), tag_name: gitTag};
@@ -24,14 +24,14 @@ module.exports = async (pluginConfig, context) => {
2424
try {
2525
({
2626
data: {id: releaseId},
27-
} = await github.repos.getReleaseByTag({owner, repo, tag: gitTag}));
27+
} = await octokit.request('GET /repos/{owner}/{repo}/releases/tags/{tag}', {owner, repo, tag: gitTag}));
2828
} catch (error) {
2929
if (error.status === 404) {
3030
logger.log('There is no release for tag %s, creating a new one', gitTag);
3131

3232
const {
3333
data: {html_url: url},
34-
} = await github.repos.createRelease({...release, body: notes});
34+
} = await octokit.request('POST /repos/{owner}/{repo}/releases', {...release, body: notes});
3535

3636
logger.log('Published GitHub release: %s', url);
3737
return {url, name: RELEASE_NAME};
@@ -44,7 +44,7 @@ module.exports = async (pluginConfig, context) => {
4444

4545
const {
4646
data: {html_url: url},
47-
} = await github.repos.updateRelease({...release, release_id: releaseId});
47+
} = await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', {...release, release_id: releaseId});
4848

4949
logger.log('Updated GitHub release: %s', url);
5050

‎lib/definitions/rate-limit.js

-27
This file was deleted.

‎lib/definitions/retry.js

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/**
2+
* Default exponential backoff configuration for retries.
3+
*/
4+
const RETRY_CONF = {
5+
// By default, Octokit does not retry on 404s.
6+
// But we want to retry on 404s to account for replication lag.
7+
doNotRetry: [400, 401, 403, 422],
8+
};
9+
10+
module.exports = {RETRY_CONF};

‎lib/definitions/throttle.js

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/**
2+
* Default configuration for throttle.
3+
* @see https://github.com/octokit/plugin-throttling.js#options
4+
*/
5+
const THROTTLE_CONF = {};
6+
7+
module.exports = {THROTTLE_CONF};

‎lib/fail.js

+14-6
Original file line numberDiff line numberDiff line change
@@ -22,26 +22,34 @@ module.exports = async (pluginConfig, context) => {
2222
if (failComment === false || failTitle === false) {
2323
logger.log('Skip issue creation.');
2424
} else {
25-
const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
25+
const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
2626
// In case the repo changed name, get the new `repo`/`owner` as the search API will not follow redirects
27-
const [owner, repo] = (await github.repos.get(parseGithubUrl(repositoryUrl))).data.full_name.split('/');
27+
const {data: repoData} = await octokit.request('GET /repos/{owner}/{repo}', parseGithubUrl(repositoryUrl));
28+
const [owner, repo] = repoData.full_name.split('/');
2829
const body = failComment ? template(failComment)({branch, errors}) : getFailComment(branch, errors);
29-
const [srIssue] = await findSRIssues(github, failTitle, owner, repo);
30+
const [srIssue] = await findSRIssues(octokit, failTitle, owner, repo);
3031

3132
if (srIssue) {
3233
logger.log('Found existing semantic-release issue #%d.', srIssue.number);
3334
const comment = {owner, repo, issue_number: srIssue.number, body};
3435
debug('create comment: %O', comment);
3536
const {
3637
data: {html_url: url},
37-
} = await github.issues.createComment(comment);
38+
} = await octokit.request('POST /repos/{owner}/{repo}/issues/{issue_number}/comments', comment);
3839
logger.log('Added comment to issue #%d: %s.', srIssue.number, url);
3940
} else {
40-
const newIssue = {owner, repo, title: failTitle, body: `${body}\n\n${ISSUE_ID}`, labels: labels || [], assignees};
41+
const newIssue = {
42+
owner,
43+
repo,
44+
title: failTitle,
45+
body: `${body}\n\n${ISSUE_ID}`,
46+
labels: labels || [],
47+
assignees,
48+
};
4149
debug('create issue: %O', newIssue);
4250
const {
4351
data: {html_url: url, number},
44-
} = await github.issues.create(newIssue);
52+
} = await octokit.request('POST /repos/{owner}/{repo}/issues', newIssue);
4553
logger.log('Created issue #%d: %s.', number, url);
4654
}
4755
}

‎lib/find-sr-issues.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
const {ISSUE_ID} = require('./definitions/constants');
22

3-
module.exports = async (github, title, owner, repo) => {
3+
module.exports = async (octokit, title, owner, repo) => {
44
const {
55
data: {items: issues},
6-
} = await github.search.issuesAndPullRequests({
6+
} = await octokit.request('GET /search/issues', {
77
q: `in:title+repo:${owner}/${repo}+type:issue+state:open+${title}`,
88
});
99

‎lib/get-client.js

+3-44
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,12 @@
1-
const {memoize, get} = require('lodash');
2-
const {Octokit} = require('@octokit/rest');
3-
const pRetry = require('p-retry');
4-
const Bottleneck = require('bottleneck');
51
const urljoin = require('url-join');
62
const HttpProxyAgent = require('http-proxy-agent');
73
const HttpsProxyAgent = require('https-proxy-agent');
84

9-
const {RETRY_CONF, RATE_LIMITS, GLOBAL_RATE_LIMIT} = require('./definitions/rate-limit');
10-
11-
/**
12-
* Http error status for which to not retry.
13-
*/
14-
const SKIP_RETRY_CODES = new Set([400, 401, 403]);
15-
16-
/**
17-
* Create or retrieve the throttler function for a given rate limit group.
18-
*
19-
* @param {Array} rate The rate limit group.
20-
* @param {String} limit The rate limits per API endpoints.
21-
* @param {Bottleneck} globalThrottler The global throttler.
22-
*
23-
* @return {Bottleneck} The throller function for the given rate limit group.
24-
*/
25-
const getThrottler = memoize((rate, globalThrottler) =>
26-
new Bottleneck({minTime: get(RATE_LIMITS, rate)}).chain(globalThrottler)
27-
);
5+
const SemanticReleaseOctokit = require('./semantic-release-octokit');
286

297
module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => {
308
const baseUrl = githubUrl && urljoin(githubUrl, githubApiPathPrefix);
31-
const globalThrottler = new Bottleneck({minTime: GLOBAL_RATE_LIMIT});
32-
const github = new Octokit({
9+
const octokit = new SemanticReleaseOctokit({
3310
auth: `token ${githubToken}`,
3411
baseUrl,
3512
request: {
@@ -41,23 +18,5 @@ module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => {
4118
},
4219
});
4320

44-
github.hook.wrap('request', (request, options) => {
45-
const access = options.method === 'GET' ? 'read' : 'write';
46-
const rateCategory = options.url.startsWith('/search') ? 'search' : 'core';
47-
const limitKey = [rateCategory, RATE_LIMITS[rateCategory][access] && access].filter(Boolean).join('.');
48-
49-
return pRetry(async () => {
50-
try {
51-
return await getThrottler(limitKey, globalThrottler).wrap(request)(options);
52-
} catch (error) {
53-
if (SKIP_RETRY_CODES.has(error.status)) {
54-
throw new pRetry.AbortError(error);
55-
}
56-
57-
throw error;
58-
}
59-
}, RETRY_CONF);
60-
});
61-
62-
return github;
21+
return octokit;
6322
};

‎lib/publish.js

+11-5
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ module.exports = async (pluginConfig, context) => {
2020
} = context;
2121
const {githubToken, githubUrl, githubApiPathPrefix, proxy, assets} = resolveConfig(pluginConfig, context);
2222
const {owner, repo} = parseGithubUrl(repositoryUrl);
23-
const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
23+
const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
2424
const release = {
2525
owner,
2626
repo,
@@ -37,7 +37,7 @@ module.exports = async (pluginConfig, context) => {
3737
if (!assets || assets.length === 0) {
3838
const {
3939
data: {html_url: url, id: releaseId},
40-
} = await github.repos.createRelease(release);
40+
} = await octokit.request('POST /repos/{owner}/{repo}/releases', release);
4141

4242
logger.log('Published GitHub release: %s', url);
4343
return {url, name: RELEASE_NAME, id: releaseId};
@@ -49,7 +49,7 @@ module.exports = async (pluginConfig, context) => {
4949

5050
const {
5151
data: {upload_url: uploadUrl, id: releaseId},
52-
} = await github.repos.createRelease(draftRelease);
52+
} = await octokit.request('POST /repos/{owner}/{repo}/releases', draftRelease);
5353

5454
// Append assets to the release
5555
const globbedAssets = await globAssets(context, assets);
@@ -74,6 +74,7 @@ module.exports = async (pluginConfig, context) => {
7474

7575
const fileName = template(asset.name || path.basename(filePath))(context);
7676
const upload = {
77+
method: 'POST',
7778
url: uploadUrl,
7879
data: await readFile(path.resolve(cwd, filePath)),
7980
name: fileName,
@@ -92,14 +93,19 @@ module.exports = async (pluginConfig, context) => {
9293

9394
const {
9495
data: {browser_download_url: downloadUrl},
95-
} = await github.repos.uploadReleaseAsset(upload);
96+
} = await octokit.request(upload);
9697
logger.log('Published file %s', downloadUrl);
9798
})
9899
);
99100

100101
const {
101102
data: {html_url: url},
102-
} = await github.repos.updateRelease({owner, repo, release_id: releaseId, draft: false});
103+
} = await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', {
104+
owner,
105+
repo,
106+
release_id: releaseId,
107+
draft: false,
108+
});
103109

104110
logger.log('Published GitHub release: %s', url);
105111
return {url, name: RELEASE_NAME, id: releaseId};

‎lib/semantic-release-octokit.js

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/* istanbul ignore file */
2+
3+
// If maintaining @octokit/core and the separate plugins gets to cumbersome
4+
// then the `octokit` package can be used which has all these plugins included.
5+
// However the `octokit` package has a lot of other things we don't care about.
6+
// We use only the bits we need to minimize the size of the package.
7+
const {Octokit} = require('@octokit/core');
8+
const {paginateRest} = require('@octokit/plugin-paginate-rest');
9+
const {retry} = require('@octokit/plugin-retry');
10+
const {throttling} = require('@octokit/plugin-throttling');
11+
12+
const {RETRY_CONF} = require('./definitions/retry');
13+
const {THROTTLE_CONF} = require('./definitions/throttle');
14+
const {version} = require('../package.json');
15+
16+
const onRetry = (retryAfter, options, octokit, retryCount) => {
17+
octokit.log.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
18+
19+
if (retryCount <= RETRY_CONF.retries) {
20+
octokit.log.debug(`Will retry after ${retryAfter}.`);
21+
return true;
22+
}
23+
};
24+
25+
const SemanticReleaseOctokit = Octokit.plugin(paginateRest, retry, throttling).defaults({
26+
userAgent: `@semantic-release/github v${version}`,
27+
retry: RETRY_CONF,
28+
throttle: {
29+
...THROTTLE_CONF,
30+
onRateLimit: onRetry,
31+
onSecondaryRateLimit: onRetry,
32+
},
33+
});
34+
35+
module.exports = SemanticReleaseOctokit;

‎lib/success.js

+33-17
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,10 @@ module.exports = async (pluginConfig, context) => {
3232
addReleases,
3333
} = resolveConfig(pluginConfig, context);
3434

35-
const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
35+
const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
3636
// In case the repo changed name, get the new `repo`/`owner` as the search API will not follow redirects
37-
const [owner, repo] = (await github.repos.get(parseGithubUrl(repositoryUrl))).data.full_name.split('/');
37+
const {data: repoData} = await octokit.request('GET /repos/{owner}/{repo}', parseGithubUrl(repositoryUrl));
38+
const [owner, repo] = repoData.full_name.split('/');
3839

3940
const errors = [];
4041

@@ -46,15 +47,27 @@ module.exports = async (pluginConfig, context) => {
4647
const shas = commits.map(({hash}) => hash);
4748

4849
const searchQueries = getSearchQueries(`repo:${owner}/${repo}+type:pr+is:merged`, shas).map(
49-
async (q) => (await github.search.issuesAndPullRequests({q})).data.items
50+
async (q) => (await octokit.request('GET /search/issues', {q})).data.items
5051
);
5152

52-
const prs = await pFilter(
53-
uniqBy(flatten(await Promise.all(searchQueries)), 'number'),
54-
async ({number}) =>
55-
(await github.pulls.listCommits({owner, repo, pull_number: number})).data.find(({sha}) => shas.includes(sha)) ||
56-
shas.includes((await github.pulls.get({owner, repo, pull_number: number})).data.merge_commit_sha)
57-
);
53+
const searchQueriesResults = await Promise.all(searchQueries);
54+
const uniqueSearchQueriesResults = uniqBy(flatten(searchQueriesResults), 'number');
55+
const prs = await pFilter(uniqueSearchQueriesResults, async ({number}) => {
56+
const commits = await octokit.paginate('GET /repos/{owner}/{repo}/pulls/{pull_number}/commits', {
57+
owner,
58+
repo,
59+
pull_number: number,
60+
});
61+
const matchingCommit = commits.find(({sha}) => shas.includes(sha));
62+
if (matchingCommit) return matchingCommit;
63+
64+
const {data: pullRequest} = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}', {
65+
owner,
66+
repo,
67+
pull_number: number,
68+
});
69+
return shas.includes(pullRequest.merge_commit_sha);
70+
});
5871

5972
debug(
6073
'found pull requests: %O',
@@ -87,17 +100,15 @@ module.exports = async (pluginConfig, context) => {
87100
debug('create comment: %O', comment);
88101
const {
89102
data: {html_url: url},
90-
} = await github.issues.createComment(comment);
103+
} = await octokit.request('POST /repos/{owner}/{repo}/issues/{issue_number}/comments', comment);
91104
logger.log('Added comment to issue #%d: %s', issue.number, url);
92105

93106
if (releasedLabels) {
94107
const labels = releasedLabels.map((label) => template(label)(context));
95-
// Don’t use .issues.addLabels for GHE < 2.16 support
96-
// https://github.com/semantic-release/github/issues/138
97-
await github.request('POST /repos/:owner/:repo/issues/:number/labels', {
108+
await octokit.request('POST /repos/{owner}/{repo}/issues/{issue_number}/labels', {
98109
owner,
99110
repo,
100-
number: issue.number,
111+
issue_number: issue.number,
101112
data: labels,
102113
});
103114
logger.log('Added labels %O to issue #%d', labels, issue.number);
@@ -120,7 +131,7 @@ module.exports = async (pluginConfig, context) => {
120131
if (failComment === false || failTitle === false) {
121132
logger.log('Skip closing issue.');
122133
} else {
123-
const srIssues = await findSRIssues(github, failTitle, owner, repo);
134+
const srIssues = await findSRIssues(octokit, failTitle, owner, repo);
124135

125136
debug('found semantic-release issues: %O', srIssues);
126137

@@ -132,7 +143,7 @@ module.exports = async (pluginConfig, context) => {
132143
debug('closing issue: %O', updateIssue);
133144
const {
134145
data: {html_url: url},
135-
} = await github.issues.update(updateIssue);
146+
} = await octokit.request('PATCH /repos/{owner}/{repo}/issues/{issue_number}', updateIssue);
136147
logger.log('Closed issue #%d: %s.', issue.number, url);
137148
} catch (error) {
138149
errors.push(error);
@@ -153,7 +164,12 @@ module.exports = async (pluginConfig, context) => {
153164
addReleases === 'top'
154165
? additionalReleases.concat('\n---\n', nextRelease.notes)
155166
: nextRelease.notes.concat('\n---\n', additionalReleases);
156-
await github.repos.updateRelease({owner, repo, release_id: ghRelaseId, body: newBody});
167+
await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', {
168+
owner,
169+
repo,
170+
release_id: ghRelaseId,
171+
body: newBody,
172+
});
157173
}
158174
}
159175
}

0 commit comments

Comments
 (0)
Please sign in to comment.