{ "cells": [ { "cell_type": "code", "source": [ "# pip install transformers" ], "outputs": [], "execution_count": 1, "metadata": { "gather": { "logged": 1724129002162 } } }, { "cell_type": "code", "source": [ "# pip install torch torchvision torchaudio -U" ], "outputs": [], "execution_count": 2, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129002227 } } }, { "cell_type": "code", "source": [ "# pip install flash-attn --no-build-isolation" ], "outputs": [], "execution_count": 3, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129002308 } } }, { "cell_type": "code", "source": [ "# ! pip install flash_attn -U" ], "outputs": [], "execution_count": 4, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129002392 } } }, { "cell_type": "code", "source": [ "from torch import bfloat16\n", "import transformers" ], "outputs": [], "execution_count": 5, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129010695 } } }, { "cell_type": "code", "source": [ "model_id = \"../Phi3MOE\"" ], "outputs": [], "execution_count": 6, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129010747 } } }, { "cell_type": "code", "source": [ "model = transformers.AutoModelForCausalLM.from_pretrained(\n", " model_id,\n", " trust_remote_code=True,\n", " torch_dtype=bfloat16,\n", " device_map='auto'\n", ")\n" ], "outputs": [ { "output_type": "display_data", "data": { "text/plain": "Loading checkpoint shards: 0%| | 0/17 [00:00\" to the end\n", " return f'<|system|> {sys_message} <|end|>\\n<|user|> {query} <|end|>\\n<|assistant|>'" ], "outputs": [], "execution_count": 13, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129523822 } } }, { "cell_type": "code", "source": [ "query ='Write something about Generative AI with MOE , translate it to Chinese'" ], "outputs": [], "execution_count": 14, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129523875 } } }, { "cell_type": "code", "source": [ "input_prompt = instruction_format(sys_msg, query)\n", "\n" ], "outputs": [], "execution_count": 15, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129523921 } } }, { "cell_type": "code", "source": [ "input_prompt" ], "outputs": [ { "output_type": "execute_result", "execution_count": 16, "data": { "text/plain": "'<|system|> You are a helpful AI assistant, you are an agent capable of using a variety of tools to answer a question. Here are a few of the tools available to you:\\n\\n- Blog: This tool helps you describe a certain knowledge point and content, and finally write it into Twitter or Facebook style content\\n- Translate: This is a tool that helps you translate into any language, using plain language as required\\n\\nTo use these tools you must always respond in JSON format containing `\"tool_name\"` and `\"input\"` key-value pairs. For example, to answer the question, \"Build Muliti Agents with MOE models\" you must use the calculator tool like so:\\n\\n```json\\n\\n{\\n \"tool_name\": \"Blog\",\\n \"input\": \"Build Muliti Agents with MOE models\"\\n}\\n\\n```\\n\\nOr to translate the question \"can you introduce yourself in Chinese\" you must respond:\\n\\n```json\\n\\n{\\n \"tool_name\": \"Search\",\\n \"input\": \"can you introduce yourself in Chinese\"\\n}\\n\\n```\\n\\nRemember just output the final result, output in JSON format containing `\"agentid\"`,`\"tool_name\"` , `\"input\"` and `\"output\"` key-value pairs .:\\n\\n```json\\n\\n[\\n\\n\\n{ \"agentid\": \"step1\",\\n \"tool_name\": \"Blog\",\\n \"input\": \"Build Muliti Agents with MOE models\",\\n \"output\": \".........\"\\n},\\n\\n{ \"agentid\": \"step2\",\\n \"tool_name\": \"Search\",\\n \"input\": \"can you introduce yourself in Chinese\",\\n \"output\": \".........\"\\n},\\n{\\n \"agentid\": \"final\"\\n \"tool_name\": \"Result\",\\n \"output\": \".........\"\\n}\\n]\\n\\n```\\n\\nThe users answer is as follows.\\n <|end|>\\n<|user|> Write something about Generative AI with MOE , translate it to Chinese <|end|>\\n<|assistant|>'" }, "metadata": {} } ], "execution_count": 16, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129523983 } } }, { "cell_type": "code", "source": [ "import torch\n", "\n", "torch.cuda.empty_cache() \n", "\n", "import os\n", "\n", "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True \"" ], "outputs": [], "execution_count": 17, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724129524034 } } }, { "cell_type": "code", "source": [ "# res = generate_text(input_prompt)\n", "\n", "output = pipe(input_prompt, **generation_args)" ], "outputs": [ { "output_type": "stream", "name": "stderr", "text": "/anaconda/envs/azureml_py38/lib/python3.9/site-packages/transformers/generation/configuration_utils.py:567: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.3` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n warnings.warn(\nThe `seen_tokens` attribute is deprecated and will be removed in v4.41. Use the `cache_position` model input instead.\n" } ], "execution_count": 18, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724130076595 } } }, { "cell_type": "code", "source": [ "output[0]['generated_text']" ], "outputs": [ { "output_type": "execute_result", "execution_count": 19, "data": { "text/plain": "' ```json\\n\\n[\\n\\n{ \"agentid\": \"step1\",\\n \"tool_name\": \"Blog\",\\n \"input\": \"Generative AI with MOE\",\\n \"output\": \"Generative AI with MOE (Mixture of Experts) is a powerful approach that combines the strengths of generative models and the flexibility of MOE architecture. This hybrid model can generate high-quality, diverse, and contextually relevant content, making it suitable for various applications such as content creation, data augmentation, and more.\"\\n},\\n\\n{ \"agentid\": \"step2\",\\n \"tool_name\": \"Translate\",\\n \"input\": \"Generative AI with MOE is a powerful approach that combines the strengths of generative models and the flexibility of MOE architecture. This hybrid model can generate high-quality, diverse, and contextually relevant content, making it suitable for various applications such as content creation, data augmentation, and more.\",\\n \"output\": \"基于生成AI的MOE（Mixture of Experts）是一种强大的方法，它结合了生成模型的优势和MOE架构的灵活性。这种混合模型可以生成高质量、多样化且上下文相关的内容，使其适用于各种应用，如内容创建、数据增强等。\"\\n},\\n{\\n \"agentid\": \"final\",\\n \"tool_name\": \"Result\",\\n \"output\": \"基于生成AI的MOE（Mixture of Experts）是一种强大的方法，它结合了生成模型的优势和MOE架构的灵活性。这种混合模型可以生成高质量、多样化且上下文相关的内容，使其适用于各种应用，如内容创建、数据增强等。\"\\n}\\n]\\n```'" }, "metadata": {} } ], "execution_count": 19, "metadata": { "jupyter": { "source_hidden": false, "outputs_hidden": false }, "nteract": { "transient": { "deleting": false } }, "gather": { "logged": 1724130076669 } } } ], "metadata": { "kernelspec": { "name": "python38-azureml", "language": "python", "display_name": "Python 3.8 - AzureML" }, "language_info": { "name": "python", "version": "3.9.19", "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py" }, "microsoft": { "ms_spell_check": { "ms_spell_check_language": "en" }, "host": { "AzureML": { "notebookHasBeenCompleted": true } } }, "kernel_info": { "name": "python38-azureml" }, "nteract": { "version": "nteract-front-end@1.0.0" } }, "nbformat": 4, "nbformat_minor": 2 }