28594_usage_tracking

@@ -5,6 +5,46 @@ const cors = require('cors')
 require('dotenv').config()
 require('dotenv').config()
 const rateLimit = require('express-rate-limit')
 const rateLimit = require('express-rate-limit')
 const anchorme = require("anchorme").default;
 const anchorme = require("anchorme").default;
+const tiktoken = require('@dqbd/tiktoken');
+const tiktokenModels = [
+  'text-davinci-003',
+  'text-davinci-002',
+  'text-davinci-001',
+  'text-curie-001',
+  'text-babbage-001',
+  'text-ada-001',
+  'davinci',
+  'curie',
+  'babbage',
+  'ada',
+  'code-davinci-002',
+  'code-davinci-001',
+  'code-cushman-002',
+  'code-cushman-001',
+  'davinci-codex',
+  'cushman-codex',
+  'text-davinci-edit-001',
+  'code-davinci-edit-001',
+  'text-embedding-ada-002',
+  'text-similarity-davinci-001',
+  'text-similarity-curie-001',
+  'text-similarity-babbage-001',
+  'text-similarity-ada-001',
+  'text-search-davinci-doc-001',
+  'text-search-curie-doc-001',
+  'text-search-babbage-doc-001',
+  'text-search-ada-doc-001',
+  'code-search-babbage-code-001',
+  'code-search-ada-code-001',
+  'gpt2',
+  'gpt-4',
+  'gpt-4-0314',
+  'gpt-4-32k',
+  'gpt-4-32k-0314',
+  'gpt-3.5-turbo',
+  'gpt-3.5-turbo-0301'
+];
+const encoding_for_model = tiktoken.encoding_for_model;
 
 
 // Open AI Configuration
 // Open AI Configuration
 // console.log(process.env.OPENAI_API_ORG)
 // console.log(process.env.OPENAI_API_ORG)

@@ -9,6 +9,7 @@
   "author": "",
   "author": "",
   "license": "ISC",
   "license": "ISC",
   "dependencies": {
   "dependencies": {
+    "@dqbd/tiktoken": "^1.0.7",
     "anchorme": "^2.1.2",
     "anchorme": "^2.1.2",
     "body-parser": "^1.20.1",
     "body-parser": "^1.20.1",
     "cors": "^2.8.5",
     "cors": "^2.8.5",