#31531 - Apply tokenization logic on other LLMs

@@ -280,6 +280,7 @@ async function runOpensource(req, res) {
   const validate = await validation(aiwp_app_id, req, res);
   const validate = await validation(aiwp_app_id, req, res);
   if(!validate) return;
   if(!validate) return;
   const { IS_FREE_USER, aiwp_logged_in, TRIED_USAGE} = validate;
   const { IS_FREE_USER, aiwp_logged_in, TRIED_USAGE} = validate;
+  let usage = {};
   
   
   try {
   try {
     let error_msg = "";
     let error_msg = "";
@@ -304,6 +305,7 @@ async function runOpensource(req, res) {
       input = response.data.choices[0].message.content
       input = response.data.choices[0].message.content
     }
     }
 
 
+    usage = response.data.usage
   } catch (e) {
   } catch (e) {
     let error_msg = e.response.data.error.message ? e.response.data.error.message : '';
     let error_msg = e.response.data.error.message ? e.response.data.error.message : '';
     if (error_msg.indexOf('maximum context length') >= 0) {
     if (error_msg.indexOf('maximum context length') >= 0) {
@@ -312,20 +314,10 @@ async function runOpensource(req, res) {
       // console.log(e.response);
       // console.log(e.response);
     }
     }
   } finally {
   } finally {
-
-    let usage = {};
-    let enc = null;
-    try {
-      enc = encodingForModel('gpt-3.5-turbo');
-      usage.prompt_tokens = (enc.encode(query_prompt)).length;
-      usage.completion_tokens = (enc.encode(input)).length;
-      usage.total_tokens = usage.prompt_tokens + usage.completion_tokens;
-    } catch (e) {
-      console.log('Error encoding prompt text', e);
-    }
     let usage_params = {
     let usage_params = {
       "aiwp_logged_in": aiwp_logged_in, "app": "chatbot+", "prompt_token": usage.prompt_tokens, "total_token": usage.total_tokens, "aiwp_app_id":aiwp_app_id , "usage_tries": TRIED_USAGE
       "aiwp_logged_in": aiwp_logged_in, "app": "chatbot+", "prompt_token": usage.prompt_tokens, "total_token": usage.total_tokens, "aiwp_app_id":aiwp_app_id , "usage_tries": TRIED_USAGE
     };
     };
+
     if(IS_FREE_USER) {
     if(IS_FREE_USER) {
 			await setUsage(usage_params);
 			await setUsage(usage_params);
 		} else {
 		} else {