28594_usage_tracking

@@ -46,7 +46,6 @@ const tiktokenModels = [
   'gpt-3.5-turbo-0301'
 ];
 const encoding_for_model = tiktoken.encoding_for_model;
-const user_secret_id = process.env.USER_SECRET_ID ? process.env.USER_SECRET_ID : "aiwp_logged_in";
 
 // Open AI Configuration
 // console.log(process.env.OPENAI_API_ORG)
@@ -71,6 +70,16 @@ app.use(cors())
 app.use(require('morgan')('dev'))
 app.use(rateLimiter)
 
+const cleanString = async (input) => {
+  var output = "";
+  for (var i=0; i<input.length; i++) {
+      if (input.charCodeAt(i) <= 127) {
+          output += input.charAt(i);
+      }
+  }
+  return output;
+};
+
 // Routing
 
 // Primary Open AI Route
@@ -124,8 +133,8 @@ app.post('/api', async (req, res) => {
 		let enc = null;
 		try {
 			enc = encoding_for_model(tiktokenModels.includes(currentModel) ? currentModel : 'gpt-3.5-turbo');
-			usage.prompt_tokens = (enc.encode(query_prompt)).length;
-			usage.completion_tokens = (enc.encode(input)).length;
+			usage.prompt_tokens = (enc.encode(await cleanString(query_prompt))).length;
+			usage.completion_tokens = (enc.encode(await cleanString(input))).length;
 			usage.total_tokens = usage.prompt_tokens + usage.completion_tokens;
 		} catch (e) {
 			console.log('Error encoding prompt text', e);