Add wikitext example

examples/quant_custom_data.py

@@ -10,6 +10,7 @@ quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version":
 model = AutoAWQForCausalLM.from_pretrained(model_path)
 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
 
+# Define data loading methods
 def load_dolly():
     data = load_dataset('databricks/databricks-dolly-15k', split="train")
 
@@ -20,8 +21,12 @@ def load_dolly():
     concatenated = data.map(concatenate_data)
     return [text for text in concatenated["text"]]
 
+def load_wikitext():
+    data = load_dataset('wikitext', 'wikitext-2-raw-v1', split="train")
+    return [text for text in data["text"] if text.strip() != '' and len(text.split(' ')) > 20]
+
 # Quantize
-model.quantize(tokenizer, quant_config=quant_config, calib_data=load_dolly())
+model.quantize(tokenizer, quant_config=quant_config, calib_data=load_wikitext())
 
 # Save quantized model
 model.save_quantized(quant_path)