From 2a55474f9725f6907a5f17fa68cd13d76e8d2f9d Mon Sep 17 00:00:00 2001 From: Bazyli Brzoska Date: Thu, 1 Jun 2023 00:54:29 -0700 Subject: [PATCH] fix: missing default exports due to limitations of ESM fixes #11 --- package.json | 8 ++++---- src/codegen/generateByModel.ts | 2 +- src/encoding/cl100k_base.ts | 2 +- src/encoding/p50k_base.ts | 2 +- src/encoding/p50k_edit.ts | 2 +- src/encoding/r50k_base.ts | 2 +- src/main.ts | 2 ++ src/model/ada.ts | 2 ++ src/model/babbage.ts | 2 ++ src/model/code-cushman-001.ts | 2 ++ src/model/code-cushman-002.ts | 2 ++ src/model/code-davinci-001.ts | 2 ++ src/model/code-davinci-002.ts | 2 ++ src/model/code-davinci-edit-001.ts | 2 ++ src/model/code-search-ada-code-001.ts | 2 ++ src/model/code-search-babbage-code-001.ts | 2 ++ src/model/curie.ts | 2 ++ src/model/cushman-codex.ts | 2 ++ src/model/davinci-codex.ts | 2 ++ src/model/davinci.ts | 2 ++ src/model/text-ada-001.ts | 2 ++ src/model/text-babbage-001.ts | 2 ++ src/model/text-curie-001.ts | 2 ++ src/model/text-davinci-001.ts | 2 ++ src/model/text-davinci-002.ts | 2 ++ src/model/text-davinci-003.ts | 2 ++ src/model/text-davinci-edit-001.ts | 2 ++ src/model/text-embedding-ada-002.ts | 2 ++ src/model/text-search-ada-doc-001.ts | 2 ++ src/model/text-search-babbage-doc-001.ts | 2 ++ src/model/text-search-curie-doc-001.ts | 2 ++ src/model/text-search-davinci-doc-001.ts | 2 ++ src/model/text-similarity-ada-001.ts | 2 ++ src/model/text-similarity-babbage-001.ts | 2 ++ src/model/text-similarity-curie-001.ts | 2 ++ src/model/text-similarity-davinci-001.ts | 2 ++ 36 files changed, 69 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 99d4699..f348a5b 100644 --- a/package.json +++ b/package.json @@ -78,10 +78,10 @@ "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json", "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json", "build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base", - "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'export=api' --env 'filename=cl100k_base.js'", - "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'export=api' --env 'filename=p50k_base.js'", - "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'export=api' --env 'filename=p50k_edit.js'", - "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'export=api' --env 'filename=r50k_base.js'", + "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'filename=cl100k_base.js'", + "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'filename=p50k_base.js'", + "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'filename=p50k_edit.js'", + "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'filename=r50k_base.js'", "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs", "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"", "postinstallDev": "yarn prepare", diff --git a/src/codegen/generateByModel.ts b/src/codegen/generateByModel.ts index d0cbf8f..690a72a 100644 --- a/src/codegen/generateByModel.ts +++ b/src/codegen/generateByModel.ts @@ -23,7 +23,7 @@ await Promise.all( `getEncodingApi('cl100k_base'`, `getEncodingApiForModel('${modelName}'`, ) - : `export * from '../encoding/${encoding}.js'\n` + : `// eslint-disable-next-line no-restricted-exports, import/no-default-export\nexport { default } from '../encoding/${encoding}.js'\nexport * from '../encoding/${encoding}.js'\n` await fs.writeFile( path.join(__dirname, `../model/${modelName}.ts`), content, diff --git a/src/encoding/cl100k_base.ts b/src/encoding/cl100k_base.ts index 82f6803..78fed60 100644 --- a/src/encoding/cl100k_base.ts +++ b/src/encoding/cl100k_base.ts @@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js' export * from '../specialTokens.js' -export const api = GptEncoding.getEncodingApi('cl100k_base', () => +const api = GptEncoding.getEncodingApi('cl100k_base', () => convertTokenBytePairEncodingFromTuples(encoder), ) const { diff --git a/src/encoding/p50k_base.ts b/src/encoding/p50k_base.ts index 776c596..d25fd10 100644 --- a/src/encoding/p50k_base.ts +++ b/src/encoding/p50k_base.ts @@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js' export * from '../specialTokens.js' -export const api = GptEncoding.getEncodingApi('p50k_base', () => +const api = GptEncoding.getEncodingApi('p50k_base', () => convertTokenBytePairEncodingFromTuples(encoder), ) const { diff --git a/src/encoding/p50k_edit.ts b/src/encoding/p50k_edit.ts index 3595979..b36df84 100644 --- a/src/encoding/p50k_edit.ts +++ b/src/encoding/p50k_edit.ts @@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js' export * from '../specialTokens.js' -export const api = GptEncoding.getEncodingApi('p50k_edit', () => +const api = GptEncoding.getEncodingApi('p50k_edit', () => convertTokenBytePairEncodingFromTuples(encoder), ) const { diff --git a/src/encoding/r50k_base.ts b/src/encoding/r50k_base.ts index b687c8c..3f9e0d2 100644 --- a/src/encoding/r50k_base.ts +++ b/src/encoding/r50k_base.ts @@ -5,7 +5,7 @@ import { GptEncoding } from '../GptEncoding.js' export * from '../specialTokens.js' -export const api = GptEncoding.getEncodingApi('r50k_base', () => +const api = GptEncoding.getEncodingApi('r50k_base', () => convertTokenBytePairEncodingFromTuples(encoder), ) const { diff --git a/src/main.ts b/src/main.ts index f0b17e7..2dcc9a5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from './encoding/cl100k_base.js' export * from './encoding/cl100k_base.js' diff --git a/src/model/ada.ts b/src/model/ada.ts index ce51f37..5270dfb 100644 --- a/src/model/ada.ts +++ b/src/model/ada.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/babbage.ts b/src/model/babbage.ts index ce51f37..5270dfb 100644 --- a/src/model/babbage.ts +++ b/src/model/babbage.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/code-cushman-001.ts b/src/model/code-cushman-001.ts index 02f07ed..7f63b02 100644 --- a/src/model/code-cushman-001.ts +++ b/src/model/code-cushman-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/code-cushman-002.ts b/src/model/code-cushman-002.ts index 02f07ed..7f63b02 100644 --- a/src/model/code-cushman-002.ts +++ b/src/model/code-cushman-002.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/code-davinci-001.ts b/src/model/code-davinci-001.ts index 02f07ed..7f63b02 100644 --- a/src/model/code-davinci-001.ts +++ b/src/model/code-davinci-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/code-davinci-002.ts b/src/model/code-davinci-002.ts index 02f07ed..7f63b02 100644 --- a/src/model/code-davinci-002.ts +++ b/src/model/code-davinci-002.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/code-davinci-edit-001.ts b/src/model/code-davinci-edit-001.ts index 0cffe0f..fee82cf 100644 --- a/src/model/code-davinci-edit-001.ts +++ b/src/model/code-davinci-edit-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_edit.js' export * from '../encoding/p50k_edit.js' diff --git a/src/model/code-search-ada-code-001.ts b/src/model/code-search-ada-code-001.ts index ce51f37..5270dfb 100644 --- a/src/model/code-search-ada-code-001.ts +++ b/src/model/code-search-ada-code-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/code-search-babbage-code-001.ts b/src/model/code-search-babbage-code-001.ts index ce51f37..5270dfb 100644 --- a/src/model/code-search-babbage-code-001.ts +++ b/src/model/code-search-babbage-code-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/curie.ts b/src/model/curie.ts index ce51f37..5270dfb 100644 --- a/src/model/curie.ts +++ b/src/model/curie.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/cushman-codex.ts b/src/model/cushman-codex.ts index 02f07ed..7f63b02 100644 --- a/src/model/cushman-codex.ts +++ b/src/model/cushman-codex.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/davinci-codex.ts b/src/model/davinci-codex.ts index 02f07ed..7f63b02 100644 --- a/src/model/davinci-codex.ts +++ b/src/model/davinci-codex.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/davinci.ts b/src/model/davinci.ts index ce51f37..5270dfb 100644 --- a/src/model/davinci.ts +++ b/src/model/davinci.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-ada-001.ts b/src/model/text-ada-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-ada-001.ts +++ b/src/model/text-ada-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-babbage-001.ts b/src/model/text-babbage-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-babbage-001.ts +++ b/src/model/text-babbage-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-curie-001.ts b/src/model/text-curie-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-curie-001.ts +++ b/src/model/text-curie-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-davinci-001.ts b/src/model/text-davinci-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-davinci-001.ts +++ b/src/model/text-davinci-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-davinci-002.ts b/src/model/text-davinci-002.ts index 02f07ed..7f63b02 100644 --- a/src/model/text-davinci-002.ts +++ b/src/model/text-davinci-002.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/text-davinci-003.ts b/src/model/text-davinci-003.ts index 02f07ed..7f63b02 100644 --- a/src/model/text-davinci-003.ts +++ b/src/model/text-davinci-003.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_base.js' export * from '../encoding/p50k_base.js' diff --git a/src/model/text-davinci-edit-001.ts b/src/model/text-davinci-edit-001.ts index 0cffe0f..fee82cf 100644 --- a/src/model/text-davinci-edit-001.ts +++ b/src/model/text-davinci-edit-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/p50k_edit.js' export * from '../encoding/p50k_edit.js' diff --git a/src/model/text-embedding-ada-002.ts b/src/model/text-embedding-ada-002.ts index 6ba161e..95ae7c2 100644 --- a/src/model/text-embedding-ada-002.ts +++ b/src/model/text-embedding-ada-002.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/cl100k_base.js' export * from '../encoding/cl100k_base.js' diff --git a/src/model/text-search-ada-doc-001.ts b/src/model/text-search-ada-doc-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-search-ada-doc-001.ts +++ b/src/model/text-search-ada-doc-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-search-babbage-doc-001.ts b/src/model/text-search-babbage-doc-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-search-babbage-doc-001.ts +++ b/src/model/text-search-babbage-doc-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-search-curie-doc-001.ts b/src/model/text-search-curie-doc-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-search-curie-doc-001.ts +++ b/src/model/text-search-curie-doc-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-search-davinci-doc-001.ts b/src/model/text-search-davinci-doc-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-search-davinci-doc-001.ts +++ b/src/model/text-search-davinci-doc-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-similarity-ada-001.ts b/src/model/text-similarity-ada-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-similarity-ada-001.ts +++ b/src/model/text-similarity-ada-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-similarity-babbage-001.ts b/src/model/text-similarity-babbage-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-similarity-babbage-001.ts +++ b/src/model/text-similarity-babbage-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-similarity-curie-001.ts b/src/model/text-similarity-curie-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-similarity-curie-001.ts +++ b/src/model/text-similarity-curie-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js' diff --git a/src/model/text-similarity-davinci-001.ts b/src/model/text-similarity-davinci-001.ts index ce51f37..5270dfb 100644 --- a/src/model/text-similarity-davinci-001.ts +++ b/src/model/text-similarity-davinci-001.ts @@ -1 +1,3 @@ +// eslint-disable-next-line no-restricted-exports, import/no-default-export +export { default } from '../encoding/r50k_base.js' export * from '../encoding/r50k_base.js'