[FIX] fix interntrain get_loglikelihood (#1584)

opencompass/models/interntrain.py

@@ -288,7 +288,7 @@ def _convert_dtype(self, default_dtype, model_dtype=None):
         else:
             raise NotImplementedError(f'Unknown model dtype {model_dtype}')
 
-    def get_token_len(self, prompt: str) -> int:
+    def get_token_len(self, prompt: str, use_bos=None, use_eos=None) -> int:
         """Get lengths of the tokenized strings.
 
         Args:
@@ -297,7 +297,7 @@ def get_token_len(self, prompt: str) -> int:
         Returns:
             int: Length of the input tokens
         """
-        tokens = self.tokenizer(prompt, use_bos=True, use_eos=True)
+        tokens = self.tokenizer(prompt, use_bos=use_bos, use_eos=use_eos)
         return len(tokens)
 
     def generate(self,
@@ -391,7 +391,7 @@ def get_loglikelihood(self, input_texts: List[str],
             for input_text, cont in zip(input_texts, conts)
         ]
         replaced_lens = [
-            len(self.encode(input_text)[0]) for input_text in replaced_texts
+            self.get_token_len(input_text) for input_text in replaced_texts
         ]
         loglikelihoods = []
         for nloss, nlen, rlen in zip(loss, lens, replaced_lens):