llama3.1 and intervl optimal configuration (#90)

Author: shfunc

language_models/llama3_8b/app.py

@@ -6,25 +6,37 @@
     from transformers import AutoModelForCausalLM, AutoTokenizer
 
 # Model parameters
-MODEL_NAME = "meta-llama/Meta-Llama-3-8B-Instruct"
+MODEL_NAME = "meta-llama/Meta-Llama-3.1-8B-Instruct"
 MAX_LENGTH = 512
-TEMPERATURE = 1.0
-TOP_P = 0.95
-TOP_K = 40
-REPETITION_PENALTY = 1.0
-NO_REPEAT_NGRAM_SIZE = 0
-DO_SAMPLE = True
+TEMPERATURE = 0.7
+TOP_P = 0.9
+TOP_K = 50
+REPETITION_PENALTY = 1.05
+NO_REPEAT_NGRAM_SIZE = 2
+DO_SAMPLE = True 
+NUM_BEAMS = 1
+EARLY_STOPPING = True
 
-CACHE_PATH = "./cached_models"
+BEAM_VOLUME_PATH = "./cached_models"
 
 
 # This runs once when the container first starts
 def load_models():
-    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, cache_dir=CACHE_PATH)
+    tokenizer = AutoTokenizer.from_pretrained(
+        MODEL_NAME, 
+        cache_dir=BEAM_VOLUME_PATH,
+        padding_side='left'
+    )
     tokenizer.pad_token = tokenizer.eos_token
     model = AutoModelForCausalLM.from_pretrained(
-        MODEL_NAME, device_map="auto", torch_dtype=torch.float16, cache_dir=CACHE_PATH
+        MODEL_NAME, 
+        device_map="auto", 
+        torch_dtype=torch.float16, 
+        cache_dir=BEAM_VOLUME_PATH,
+        use_cache=True,
+        low_cpu_mem_usage=True
     )
+    model.eval()
     return model, tokenizer
 
 
@@ -38,22 +50,25 @@ def load_models():
             "huggingface_hub[hf-transfer]",
         ]
     )
-    .with_envs("HF_HUB_ENABLE_HF_TRANSFER=1")
+    .with_envs({
+        "HF_HUB_ENABLE_HF_TRANSFER": "1",
+        "TOKENIZERS_PARALLELISM": "false",
+        "CUDA_VISIBLE_DEVICES": "0",
+    })
 )
 
 
 @endpoint(
     secrets=["HF_TOKEN"],
     on_start=load_models,
-    name="meta-llama-3-8b-instruct",
+    name="meta-llama-3.1-8b-instruct",
     cpu=2,
-    memory="32Gi",
-    gpu_count=2,
+    memory="16Gi",
     gpu="A10G",
     volumes=[
         Volume(
             name="cached_models",
-            mount_path=CACHE_PATH,
+            mount_path=BEAM_VOLUME_PATH,
         )
     ],
     image=image,
@@ -68,30 +83,43 @@ def generate_text(context, **inputs):
         return {"error": "Please provide messages for text generation."}
 
     generate_args = {
-        "max_length": inputs.get("max_tokens", MAX_LENGTH),
+        "max_new_tokens": inputs.get("max_tokens", MAX_LENGTH),
         "temperature": inputs.get("temperature", TEMPERATURE),
         "top_p": inputs.get("top_p", TOP_P),
         "top_k": inputs.get("top_k", TOP_K),
         "repetition_penalty": inputs.get("repetition_penalty", REPETITION_PENALTY),
         "no_repeat_ngram_size": inputs.get(
             "no_repeat_ngram_size", NO_REPEAT_NGRAM_SIZE
         ),
+        "num_beams": inputs.get("num_beams", NUM_BEAMS),
+        "early_stopping": inputs.get("early_stopping", EARLY_STOPPING),
         "do_sample": inputs.get("do_sample", DO_SAMPLE),
         "use_cache": True,
         "eos_token_id": tokenizer.eos_token_id,
         "pad_token_id": tokenizer.pad_token_id,
     }
 
-    model_inputs = tokenizer.apply_chat_template(
+    model_inputs_str = tokenizer.apply_chat_template(
         messages, tokenize=False, add_generation_prompt=True
     )
-    inputs = tokenizer(model_inputs, return_tensors="pt", padding=True)
-    input_ids = inputs["input_ids"].to("cuda")
-    attention_mask = inputs["attention_mask"].to("cuda")
+    
+    tokenized_inputs = tokenizer(
+        model_inputs_str, 
+        return_tensors="pt", 
+        padding=True, 
+        truncation=True, 
+        max_length=2048
+    )
+    input_ids = tokenized_inputs["input_ids"].to("cuda")
+    attention_mask = tokenized_inputs["attention_mask"].to("cuda")
+    input_ids_length = input_ids.shape[-1]
 
     with torch.no_grad():
         outputs = model.generate(
-            input_ids=input_ids, attention_mask=attention_mask, **generate_args
+            input_ids=input_ids, 
+            attention_mask=attention_mask, 
+            **generate_args
         )
-        output_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        new_tokens = outputs[0][input_ids_length:]
+        output_text = tokenizer.decode(new_tokens, skip_special_tokens=True)
         return {"output": output_text}

vllm/chat.py

@@ -86,7 +86,7 @@ def process_user_input(
         self, user_input: str, img_link: Optional[str] = None, stream: bool = False
     ) -> str:
         """Process user input and return assistant's response."""
-        if self.model == "OpenGVLab/InternVL2_5-8B" and img_link:
+        if self.model == "OpenGVLab/InternVL3-8B-AWQ" and img_link:
             self.conversation_history.append(
                 {
                     "role": "user",
@@ -178,7 +178,7 @@ def chat() -> None:
 
             # Handle image input for vision models
             img_link = None
-            if model == "OpenGVLab/InternVL2_5-8B":
+            if model == "OpenGVLab/InternVL3-8B-AWQ":
                 img_link = Prompt.ask(
                     "[bold yellow]Image link (press enter to skip)[/bold yellow]"
                 )

vllm/models.py

@@ -1,27 +1,28 @@
 from beam.integrations import VLLM, VLLMArgs
 from beam import Image
 
-INTERNVL2_5 = "OpenGVLab/InternVL2_5-8B"
+INTERNVL3_AWQ = "OpenGVLab/InternVL3-8B-AWQ"
 YI_CODER_CHAT = "01-ai/Yi-Coder-9B-Chat"
 MISTRAL_INSTRUCT = "mistralai/Mistral-7B-Instruct-v0.3"
 DEEPSEEK_R1 = "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
 
 internvl = VLLM(
-    name=INTERNVL2_5.split("/")[-1],
-    cpu=8,
-    memory="32Gi",
+    name=INTERNVL3_AWQ.split("/")[-1],
+    cpu=4,
+    memory="16Gi",
     gpu="A10G",
-    gpu_count=2,
+    gpu_count=1,
     image=(Image(python_version="python3.12")).add_python_packages(
         ["vllm==0.6.4.post1"]
     ),
     vllm_args=VLLMArgs(
-        model=INTERNVL2_5,
-        served_model_name=[INTERNVL2_5],
+        model=INTERNVL3_AWQ,
+        served_model_name=[INTERNVL3_AWQ],
         trust_remote_code=True,
         max_model_len=4096,
-        gpu_memory_utilization=0.95,
+        gpu_memory_utilization=0.90,
         limit_mm_per_prompt={"image": 2},
+        quantization="awq",
     ),
 )