Skip to content

Commit

Permalink
feat(sdk): generate an sdk with spec from npm (#1678)
Browse files Browse the repository at this point in the history
## Proposed change

The goal is to be able to generate an SDK with the necessary setup to
fetch the spec file form an npm registry

## Related issues

- 🐛 Fixes #(issue)
- 🚀 Feature #(issue)

<!-- Please make sure to follow the contributing guidelines on
https://github.com/amadeus-digital/Otter/blob/main/CONTRIBUTING.md -->
  • Loading branch information
mrednic-1A committed May 14, 2024
2 parents 96b9447 + 196526e commit 89fcb9b
Show file tree
Hide file tree
Showing 38 changed files with 602 additions and 86 deletions.
6 changes: 4 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"build:lint": "yarn nx run-many --target=build --projects=eslint-plugin --parallel $(yarn get:cpus-number)",
"build:swagger-gen": "yarn nx run-many --target=build-swagger --parallel $(yarn get:cpus-number)",
"prepare:publish": "yarn prepare-publish \"$(yarn workspaces:list)\" --append dist",
"publish": "yarn run prepare:publish && yarn nx run-many --target=publish --parallel $(yarn get:cpus-number) --nx-bail",
"publish": "yarn run prepare:publish && yarn nx run-many --target=publish --exclude=tag:private --parallel $(yarn get:cpus-number) --nx-bail",
"publish:extensions": "yarn nx run-many --target=publish-extension --parallel $(yarn get:cpus-number)",
"publish:extensions:affected": "yarn nx affected --target=publish-extension --parallel $(yarn get:cpus-number)",
"lint": "yarn nx run-many --target=lint --parallel $(yarn get:cpus-number)",
Expand All @@ -41,7 +41,8 @@
"verdaccio:start-persistent": "docker run -d -it --rm --name verdaccio -p 4873:4873 -v \"$(yarn get:current-dir)/.verdaccio/conf\":/verdaccio/conf -v \"$(yarn get:current-dir)/.verdaccio/storage\":/verdaccio/storage:z verdaccio/verdaccio",
"verdaccio:clean": "rimraf -g \".verdaccio/storage/@{o3r,ama-sdk,ama-terasu}\"",
"verdaccio:login": "yarn cpy --cwd=./.verdaccio/conf .npmrc . --rename=.npmrc-logged && npx --yes npm-cli-login -u verdaccio -p verdaccio -e test@test.com -r http://127.0.0.1:4873 --config-path \".verdaccio/conf/.npmrc-logged\"",
"verdaccio:publish": "yarn verdaccio:clean && yarn set:version 999.0.$(node -e 'process.stdout.write(String(Date.now()))') --include \"!**/!(dist)/package.json\" --include !package.json && yarn verdaccio:login && yarn run publish --userconfig \".verdaccio/conf/.npmrc-logged\" --tag=latest --@o3r:registry=http://127.0.0.1:4873 --@ama-sdk:registry=http://127.0.0.1:4873 --@ama-terasu:registry=http://127.0.0.1:4873",
"verdaccio:prepare-publish": "yarn verdaccio:clean && yarn set:version 999.0.$(node -e 'process.stdout.write(String(Date.now()))') --include \"!**/!(dist)/package.json\" --include !package.json && yarn verdaccio:login && replace-in-files --regex=\"private(.*)true\" --replacement=private\\$1false '**/dist/package.json' && yarn run prepare:publish",
"verdaccio:publish": "yarn run verdaccio:prepare-publish && yarn nx run-many --target=publish --parallel $(yarn get:cpus-number) --nx-bail --userconfig \".verdaccio/conf/.npmrc-logged\" --tag=latest --@o3r:registry=http://127.0.0.1:4873 --@ama-sdk:registry=http://127.0.0.1:4873 --@ama-terasu:registry=http://127.0.0.1:4873",
"verdaccio:stop": "docker container stop $(docker ps -a -q --filter=\"name=verdaccio\")",
"verdaccio:all": "yarn verdaccio:stop && yarn verdaccio:start && yarn verdaccio:publish",
"watch:vscode-extension": "yarn nx run vscode-extension:compile:watch",
Expand Down Expand Up @@ -238,6 +239,7 @@
"postcss-scss": "~4.0.9",
"react": "^18.0.0",
"react-dom": "^18.0.0",
"replace-in-files-cli": "^2.2.0",
"rimraf": "^5.0.1",
"sass": "~1.76.0",
"sass-loader": "^14.0.0",
Expand Down
12 changes: 12 additions & 0 deletions packages/@ama-sdk/core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,3 +69,15 @@ function petApiFactory() {
```

> *Note*: Adding a third-party logging service is optional. If undefined, the fallback is the console logger.
### CLI

This package also comes with CLI scripts that can facilitate the upgrade and publication of an SDK.
Use --help on each command for more information

| Script | Description |
|-----------------------------|------------------------------------------------------------------------------------------------|
| amasdk-clear-index | Remove the index files that are no longer necessary after the deletion of the associated model |
| amasdk-files-pack | Prepare the dist folder for publication |
| amasdk-update-spec-from-npm | Update the OpenAPI spec from an NPM package |

10 changes: 10 additions & 0 deletions packages/@ama-sdk/core/cli/clear-index.cts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,21 @@
* Remove deleted models' exports
*/

import * as minimist from 'minimist';
import { promises as fs, statSync } from 'node:fs';
import { resolve } from 'node:path';

const argv = minimist(process.argv.slice(2));
const { help } = argv;
const baseDir = resolve(process.cwd(), 'src', 'models', 'base');

if (help) {
console.log(`Remove the index files that are no longer necessary after the deletion of the associated model.
Usage: amasdk-clear-index
`);
process.exit(0);
}

void (async () => {
const models = await fs.readdir(baseDir);
const shouldRemoveModels = (
Expand Down
18 changes: 15 additions & 3 deletions packages/@ama-sdk/core/cli/files-pack.cts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,17 @@ import type { PackageJson } from 'type-fest';
const argv = minimist(process.argv.slice(2));
const distFolder = argv.dist || 'dist';
const baseDir = argv.cwd && path.resolve(process.cwd(), argv.cwd) || process.cwd();
const {watch, noExports} = argv;
const {help, watch, noExports} = argv;

if (help) {
console.log(`Prepare the dist folder for publication. This will copy necessary files from src and update the exports in package.json.
Usage: amasdk-files-pack [--exports] [--watch]
--exports Update the exports in package.json. (Default: true)
--watch Watch for files changes and run the updates
`);
process.exit(0);
}

const files = [
{glob: 'README.md', cwdForCopy: baseDir},
Expand Down Expand Up @@ -71,10 +81,12 @@ void (async () => {
return watch ?
import('chokidar')
.then((chokidar) => chokidar.watch(glob, {cwd: baseDir}))
.then((watcher) => watcher.on('all', (event, file) => {
.then((watcher) => watcher.on('all', async (event, file) => {
if (event !== 'unlink' && event !== 'unlinkDir') {
copyToDist(file, cwdForCopy);
return updateExports();
if (!noExports) {
await updateExports();
}
}
})) :
globby.sync(glob)
Expand Down
74 changes: 74 additions & 0 deletions packages/@ama-sdk/core/cli/update-spec-from-npm.cts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/usr/bin/env node

/*
* Update the OpenAPI spec from an NPM package
*/

import * as minimist from 'minimist';
import { existsSync } from 'node:fs';
import { createRequire } from 'node:module';
import { extname, join } from 'node:path';
import { copyFile, readFile } from 'node:fs/promises';
import type { PackageJson } from 'type-fest';
import type { OpenApiToolsConfiguration } from '../src/fwk/open-api-tools-configuration';

const argv = minimist(process.argv.slice(2));
const packageName = argv._[0];
const { help, output, 'package-path': packagePath, quiet } = argv;
const openApiConfigDefaultPath = './openapitools.json';
const supportedExtensions = ['json', 'yaml', 'yml'];
const noop = () => undefined;
const logger = quiet ? {error: noop, warn: noop, log: noop, info: noop, debug: noop} : console;

if (help) {
console.log(`This script can be used to update your local spec file from a given locally installed npm package.
Usage: amasdk-update-spec-from-npm <package-name> [--package-path] [--output] [--quiet]
package-name The full identifier of the npm package (e.g. @my-scope/my-package)
--package-path The relative path inside the npm package where to find the spec file (default: './openapi.yml')
--output The path where the spec file should be copied (default: './openapi.yml')
--quiet Don't log anything
`);
process.exit(0);
}

if (!packageName) {
logger.error('Need to provide packageName, use `amasdk-update-spec-from-npm --help` for more information');
process.exit(-1);
}

void (async () => {
let specSourcePath;
const appRequire = createRequire(join(process.cwd(), 'package.json'));
const packageJsonPath = appRequire.resolve(`${packageName}/package.json`);
if (!packagePath) {
const packageJson = JSON.parse(await readFile(packageJsonPath, {encoding: 'utf8'})) as PackageJson;
const exportMatcher = new RegExp(`openapi\\.(?:${supportedExtensions.join('|')})$`);
const matchingExport = packageJson.exports && Object.keys(packageJson.exports).find((exportPath) => exportMatcher.test(exportPath));
if (matchingExport) {
specSourcePath = appRequire.resolve(`${packageName}/${matchingExport}`);
}
} else {
specSourcePath = packageJsonPath.replace(/package.json$/, packagePath);
}
if (!specSourcePath || !existsSync(specSourcePath)) {
logger.error(`Unable to find source spec from ${packageName}, please make sure it is correctly exported in package.json`);
process.exit(-2);
}

let specDestinationPath = output;
if (!specDestinationPath) {
const specSourceExtension = extname(specSourcePath);
specDestinationPath = `./openapi${specSourceExtension}`;
if (existsSync(openApiConfigDefaultPath)) {
const openApiConfig = JSON.parse(await readFile(openApiConfigDefaultPath, {encoding: 'utf8'})) as OpenApiToolsConfiguration;
const generators = Object.values(openApiConfig['generator-cli']?.generators ?? {});
if (generators.length === 1 && generators[0].inputSpec && extname(generators[0].inputSpec) === specSourceExtension) {
specDestinationPath = generators[0].inputSpec;
}
}
}

logger.info(`Updating spec file from "${specSourcePath}" to "${specDestinationPath}" (CWD: "${process.cwd()}")`);
await copyFile(specSourcePath, specDestinationPath);
})();
3 changes: 2 additions & 1 deletion packages/@ama-sdk/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@
"schematics": "./collection.json",
"bin": {
"amasdk-clear-index": "./dist/cli/clear-index.cjs",
"amasdk-files-pack": "./dist/cli/files-pack.cjs"
"amasdk-files-pack": "./dist/cli/files-pack.cjs",
"amasdk-update-spec-from-npm": "./dist/cli/update-spec-from-npm.cjs"
}
}
1 change: 1 addition & 0 deletions packages/@ama-sdk/core/src/fwk/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ export * from './errors';
export * from './ignore-enum.type';
export * from './logger';
export * from './mocks/index';
export * from './open-api-tools-configuration';
export * from './Reviver';
30 changes: 30 additions & 0 deletions packages/@ama-sdk/core/src/fwk/open-api-tools-configuration.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/** Configuration of an Open API generator */
export interface OpenApiToolsGenerator {
/** Location of the OpenAPI spec, as URL or file */
inputSpec: string;
/** Output path for the generated SDK */
output: string;
/** Generator to use */
generatorName: string;
/** Path to configuration file. It can be JSON or YAML */
config?: string;
/** Sets specified global properties */
globalProperty?: string | Record<string, any>;
}

/** Global configuration of Open API generators */
export interface OpenApiToolsGeneratorCli {
/** Open API version */
version: string;
/** Location of the generator JAR file */
storageDir?: string;
/** Generators configuration */
generators: Record<string, OpenApiToolsGenerator>;
}

/** Global configuration of Open API Tools */
export interface OpenApiToolsConfiguration {
/** Generators CLI configuration */
// eslint-disable-next-line @typescript-eslint/naming-convention
'generator-cli': OpenApiToolsGeneratorCli;
}
5 changes: 5 additions & 0 deletions packages/@ama-sdk/core/tsconfig.cli.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@
"rootDir": ".",
"tsBuildInfoFile": "build/.tsbuildinfo.cli"
},
"references": [
{
"path": "./tsconfig.build.json"
}
],
"include": [
"cli/**/*.cts"
],
Expand Down
12 changes: 10 additions & 2 deletions packages/@ama-sdk/create/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,19 @@ npm create @ama-sdk typescript <project-name> -- --package-manager=yarn [...opti

## Options list

- `--spec-path`: Path to the swagger/open-api specification used to generate the SDK
- `--package-manager`: Node package manager to be used (`npm` and `yarn` are available).
- `--debug --no-dry-run`: Enable schematics debug mode (dry-run is not currently supported).
- `--o3r-metrics`: Enable or disable the collection of anonymous data for Otter
- `--exact-o3r-version` : use a pinned version for [otter packages](https://github.com/AmadeusITGroup/otter/blob/main/docs/README.md).

- `--spec-path`: Path to the swagger/open-api specification used to generate the SDK
- `--spec-package-name`: The npm package name where the spec file can be fetched
- `--spec-package-path`: The path inside the package where to find the spec file
- `--spec-package-version`: The version to target for the npm package where the spec file can be fetched
- `--spec-package-registry`: The npm registry where the spec file can be fetched

> [!NOTE]
> If `--spec-path` or `--spec-package-name` is specified, the SDK will be generated based on this specification at the creation time.
> [!NOTE]
> If the `--spec-path` is specified, the SDK will be generated based on this specification at the creation time.
> > `--spec-package-registry` option assumes that the authentication is set up globally (See [npm setup](https://docs.npmjs.com/cli/v8/configuring-npm/npmrc#auth-related-configuration), [yarn setup](https://yarnpkg.com/configuration/yarnrc#npmRegistries))
21 changes: 21 additions & 0 deletions packages/@ama-sdk/create/src/index.it.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,27 @@ describe('Create new sdk command', () => {
).not.toThrow();
expect(() => packageManagerRun({script: 'build'}, { ...execAppOptions, cwd: sdkPackagePath })).not.toThrow();
expect(existsSync(path.join(sdkPackagePath, 'src', 'models', 'base', 'pet', 'pet.reviver.ts'))).toBeTruthy();
expect(() => packageManagerRun({script: 'spec:upgrade'}, { ...execAppOptions, cwd: sdkPackagePath })).not.toThrow();
});

test('should generate a full SDK when the specification is provided as npm dependency', () => {
expect(() =>
packageManagerCreate({
script: '@ama-sdk',
args: [
'typescript',
sdkPackageName,
'--package-manager', packageManager,
'--spec-package-name', '@ama-sdk/showcase-sdk',
'--spec-package-path', 'openapi.yml',
'--spec-package-version', o3rEnvironment.testEnvironment.o3rVersion,
'--spec-package-registry', o3rEnvironment.testEnvironment.packageManagerConfig.registry
]
}, execAppOptions)
).not.toThrow();
expect(() => packageManagerRun({script: 'build'}, { ...execAppOptions, cwd: sdkPackagePath })).not.toThrow();
expect(existsSync(path.join(sdkPackagePath, 'src', 'models', 'base', 'pet', 'pet.reviver.ts'))).toBeTruthy();
expect(() => packageManagerRun({script: 'spec:upgrade'}, { ...execAppOptions, cwd: sdkPackagePath })).not.toThrow();
});

test('should generate an SDK with no package scope', () => {
Expand Down
36 changes: 31 additions & 5 deletions packages/@ama-sdk/create/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,12 @@ const getYarnVersion = () => {
}
};

const schematicArgs = [
if (argv['spec-path'] && argv['spec-package-name']) {
console.error('--spec-path cannot be set with --spec-package-name');
process.exit(-4);
}

const commonSchematicArgs = [
argv.debug !== undefined ? `--debug=${argv.debug as string}` : '--debug=false', // schematics enable debug mode per default when using schematics with relative path
...(name ? ['--name', name] : []),
'--package', pck,
Expand All @@ -74,21 +79,42 @@ const resolveTargetDirectory = resolve(process.cwd(), targetDirectory);

const run = () => {
const isSpecRelativePath = !!argv['spec-path'] && !parse(argv['spec-path']).root;
const shellSchematicArgs = [
...commonSchematicArgs,
...(argv['spec-package-name'] ? ['--spec-package-name', argv['spec-package-name']] : []),
...(argv['spec-package-registry'] ? ['--spec-package-registry', argv['spec-package-registry']] : []),
...(argv['spec-package-path'] ? ['--spec-package-path', argv['spec-package-path']] : []),
...(argv['spec-package-version'] ? ['--spec-package-version', argv['spec-package-version']] : [])
];
const coreSchematicArgs = [
...commonSchematicArgs,
'--spec-path', argv['spec-package-name'] ? './openapi.yml' : isSpecRelativePath ? relative(resolveTargetDirectory, resolve(process.cwd(), argv['spec-path'])) : argv['spec-path']
];

const runner = process.platform === 'win32' ? `${packageManager}.cmd` : packageManager;
const steps: { args: string[]; cwd?: string; runner?: string }[] = [
{ args: [binPath, `${schematicsPackage}:typescript-shell`, ...schematicArgs, '--directory', targetDirectory] },
{ args: [binPath, `${schematicsPackage}:typescript-shell`, ...shellSchematicArgs, '--directory', targetDirectory] },
...(
packageManager === 'yarn'
? [{ runner, args: ['set', 'version', getYarnVersion()], cwd: resolveTargetDirectory }]
: []
),
...(argv['spec-path'] ? [{
...(argv['spec-package-name'] ? [{
runner,
args: [
'exec',
'amasdk-update-spec-from-npm',
argv['spec-package-name'],
...packageManager === 'npm' ? ['--'] : [],
'--package-path', argv['spec-package-path']
],
cwd: resolveTargetDirectory
}] : []),
...((argv['spec-path'] || argv['spec-package-name']) ? [{
args: [
binPath,
`${schematicsPackage}:typescript-core`,
...schematicArgs,
'--spec-path', isSpecRelativePath ? relative(resolveTargetDirectory, resolve(process.cwd(), argv['spec-path'])) : argv['spec-path']
...coreSchematicArgs
],
cwd: resolveTargetDirectory
}] : [])
Expand Down
5 changes: 3 additions & 2 deletions packages/@ama-sdk/schematics/schematics/ng-add/index.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import type { OpenApiToolsConfiguration } from '@ama-sdk/core';
import { isJsonObject } from '@angular-devkit/core';
import { chain, externalSchematic, Rule, SchematicContext, Tree } from '@angular-devkit/schematics';
import * as path from 'node:path';
import { readFileSync } from 'node:fs';
import { lastValueFrom } from 'rxjs';
import type { PackageJson } from 'type-fest';
import type { JsonObject, PackageJson } from 'type-fest';
import { DevInstall } from '../helpers/node-install';

const packageJsonPath = '/package.json';
Expand Down Expand Up @@ -79,7 +80,7 @@ const createOpenApiToolsConfig: Rule = (tree) => {
const openApiDefaultStorageDir = '.openapi-generator';
if (tree.exists(openApiConfigPath)) {
// eslint-disable-next-line @typescript-eslint/naming-convention
const openapitoolsConfig = tree.readJson(openApiConfigPath) as { 'generator-cli'?: { storageDir?: string; version?: string } } || {};
const openapitoolsConfig = tree.readJson(openApiConfigPath) as JsonObject & OpenApiToolsConfiguration;
openapitoolsConfig['generator-cli'] = {storageDir: openApiDefaultStorageDir, ...openapitoolsConfig['generator-cli'], version: openApiGeneratorVersion};
tree.overwrite(openApiConfigPath, JSON.stringify(openapitoolsConfig));
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import type { OpenApiToolsConfiguration } from '@ama-sdk/core';
import { Tree } from '@angular-devkit/schematics';
import { SchematicTestRunner, UnitTestTree } from '@angular-devkit/schematics/testing';
import * as path from 'node:path';
import { LOCAL_SPEC_FILENAME, SPEC_JSON_EXTENSION, SPEC_YAML_EXTENSION } from './index';
import type { JsonObject } from 'type-fest';

const collectionPath = path.join(__dirname, '..', '..', '..', 'collection.json');

Expand Down Expand Up @@ -33,7 +35,7 @@ describe('Typescript Core Generator', () => {
const tree = await runner.runSchematic('typescript-core', {
specPath: path.join(__dirname, '..', '..', '..', 'testing', 'MOCK_swagger.yaml')
}, baseTree);
const content: any = tree.readJson('/openapitools.json');
const content = tree.readJson('/openapitools.json') as JsonObject & OpenApiToolsConfiguration;

expect(content['generator-cli'].generators['test-sdk-sdk'].inputSpec.endsWith(`${LOCAL_SPEC_FILENAME}.${SPEC_YAML_EXTENSION}`)).toBe(true);
expect(tree.exists(`/${LOCAL_SPEC_FILENAME}.${SPEC_YAML_EXTENSION}`)).toBe(true);
Expand All @@ -44,7 +46,7 @@ describe('Typescript Core Generator', () => {
const tree = await runner.runSchematic('typescript-core', {
specPath: path.join(__dirname, '..', '..', '..', 'testing', 'MOCK_swagger.json')
}, baseTree);
const content: any = tree.readJson('/openapitools.json');
const content = tree.readJson('/openapitools.json') as JsonObject & OpenApiToolsConfiguration;

expect(content['generator-cli'].generators['test-sdk-sdk'].inputSpec.endsWith(`${LOCAL_SPEC_FILENAME}.${SPEC_JSON_EXTENSION}`)).toBe(true);
expect(tree.exists(`/${LOCAL_SPEC_FILENAME}.${SPEC_JSON_EXTENSION}`)).toBe(true);
Expand Down

0 comments on commit 89fcb9b

Please sign in to comment.