pip3 install --upgrade transformers optimum

pip3 uninstall -y auto-gptq
git clone https://github.com/PanQiWei/AutoGPTQ
cd AutoGPTQ
git checkout v0.5.1
pip3 install .

model_name_or_path = "TheBloke/Mixtral-8x7B-Instruct-v0.1-GPTQ"
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, GPTQConfig
from auto_gptq import AutoGPTQForCausalLM

model_name_or_path = args.model_dir
# To use a different branch, change revision
# For example: revision="gptq-4bit-32g-actorder_True"
model = AutoGPTQForCausalLM.from_quantized(model_name_or_path,
        model_basename="model",
        use_safetensors=True,
        trust_remote_code=False,
        device="cuda:0",
        use_triton=False,
        disable_exllama=False,
        disable_exllamav2=True,
        quantize_config=None)

tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True, trust_remote_code=False)

prompt = "Why Lion is King of Jungle?"
prompt_template=f'''<s>[INST] {prompt} [/INST]
'''

print("\
\
*** Generate:")

input_ids = tokenizer(prompt_template, return_tensors='pt').input_ids.cuda()
output = model.generate(inputs=input_ids, temperature=0.7, do_sample=True, top_p=0.95, top_k=40, max_new_tokens=512)
print(tokenizer.decode(output[0]))