update basic.py

examples/offline_inference/basic/basic.py

@@ -13,14 +13,14 @@ if __name__ == '__main__':
     # Create a sampling params object.
     sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=16)
 
-# Create an LLM.
-llm = LLM(model="facebook/opt-125m")
-# Generate texts from the prompts. The output is a list of RequestOutput objects
-# that contain the prompt, generated text, and other information.
-outputs = llm.generate(prompts, sampling_params)
-# Print the outputs.
-print("\nGenerated Outputs:\n" + "-" * 60)
-for output in outputs:
+    # Create an LLM.
+    llm = LLM(model="facebook/opt-125m",tensor_parallel_size=1, dtype="float16",trust_remote_code=True, enforce_eager=True)
+    # Generate texts from the prompts. The output is a list of RequestOutput objects
+    # that contain the prompt, generated text, and other information.
+    outputs = llm.generate(prompts, sampling_params)
+    # Print the outputs.
+    print("\nGenerated Outputs:\n" + "-" * 60)
+    for output in outputs:
         prompt = output.prompt
         generated_text = output.outputs[0].text
         print(f"Prompt:    {prompt!r}")