Update moss_inference.py

2023-04-24 21:33:50 +08:00
parent e2c3359f1d
commit d7431d4021
1 changed files with 1 additions and 0 deletions
@@ -348,6 +348,7 @@ if __name__ == "__main__":
    # Create an Inference instance with the specified model directory.
    infer = Inference(model_dir="fnlp/moss-moon-003-sft", device_map="auto")

+    # ！！！如果需要运行量化版本，请以以下方式load模型！！！
    # If you need to load a quantized model, please instead load the model and then pass it into Inference.__init__.
    # model = MossForCausalLM.from_pretrained("fnlp/moss-moon-003-sft-int4").half().cuda()
    # infer = Inference(model, device_map="auto")