:Billing

# Before you run the sample code, run the pip install tiktoken command.
from dashscope import get_tokenizer  # Make sure that DashScope SDK for Python V1.14.0 or later is used.

# Obtain the tokenizer object. Only Qwen models are supported.
tokenizer = get_tokenizer('qwen-turbo')

input_str = 'Qwen provides powerful capabilities. '

# Split the string into tokens and convert the tokens into token IDs.
tokens = tokenizer.encode(input_str)
print(f"IDs of tokens split from the string: {tokens}")
print(f"Total tokens: {len(tokens)}")

# Convert the token IDs into a string and display the string.
for i in range(len(tokens)):
    print(f"String converted from token IDs {tokens[i]}: {tokenizer.decode(tokens[i])}")

// Copyright (c) Alibaba, Inc. and its affiliates.
// Make sure that DashScope SDK for Java V2.13.0 or later is used.
import java.util.List;
import com.alibaba.dashscope.exception.NoSpecialTokenExists;
import com.alibaba.dashscope.exception.UnSupportedSpecialTokenMode;
import com.alibaba.dashscope.tokenizers.Tokenizer;
import com.alibaba.dashscope.tokenizers.TokenizerFactory;

public class Main {
  public static void testEncodeOrdinary(){
    Tokenizer tokenizer = TokenizerFactory.qwen();
    String prompt ="How long does it take for you to walk a thousand miles?  ";
    // Encode the string with no special tokens.
    List<Integer> ids = tokenizer.encodeOrdinary(prompt);
    System.out.println(ids);
    String decodedString = tokenizer.decode(ids);
    assert decodedString == prompt;
  }

  public static void testEncode() throws NoSpecialTokenExists, UnSupportedSpecialTokenMode{
    Tokenizer tokenizer = TokenizerFactory.qwen();
    String prompt = "<|im_start|>system\nYour are a helpful assistant.<|im_end|>\n<|im_start|>user\nSanFrancisco is a<|im_end|>\n<|im_start|>assistant\n";
    // Encode the string with special tokens <|im_start|> and <|im_end|>.
    List<Integer> ids = tokenizer.encode(prompt, "all");
    // 24 tokens [151644, 8948, 198, 7771, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 23729, 80328, 9464, 374, 264, 151645, 198, 151644, 77091, 198]
    String decodedString = tokenizer.decode(ids);
    System.out.println(ids);
    assert decodedString == prompt;

  }

  public static void main(String[] args) {
      try {
        testEncodeOrdinary();
        testEncode();
      } catch (NoSpecialTokenExists | UnSupportedSpecialTokenMode e) {
        e.printStackTrace();
      }
  }
}