{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6687628a-472e-495a-82c0-b1dacde99e56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pip in /usr/lib/python3/dist-packages (22.0.2)\n",
      "Collecting pip\n",
      "  Using cached pip-24.2-py3-none-any.whl (1.8 MB)\n",
      "Installing collected packages: pip\n",
      "  Attempting uninstall: pip\n",
      "    Found existing installation: pip 22.0.2\n",
      "    Not uninstalling pip at /usr/lib/python3/dist-packages, outside environment /usr\n",
      "    Can't uninstall 'pip'. No files were found to uninstall.\n",
      "Successfully installed pip-24.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting unsloth@ git+https://github.com/unslothai/unsloth.git (from unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Cloning https://github.com/unslothai/unsloth.git to /tmp/pip-install-42mbmn6k/unsloth_a3b84356d92343808c0187bf6996ed5b\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/unslothai/unsloth.git /tmp/pip-install-42mbmn6k/unsloth_a3b84356d92343808c0187bf6996ed5b\n",
      "  Resolved https://github.com/unslothai/unsloth.git to commit 976d11a10d54383aeb7a692c69e01151a20bfd72\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: bitsandbytes>=0.43.3 in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.43.3)\n",
      "Collecting torch (from bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached torch-2.4.0-cp310-cp310-manylinux1_x86_64.whl.metadata (26 kB)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2.0.1)\n",
      "Collecting xformers@ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl (20.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.8/20.8 MB\u001b[0m \u001b[31m49.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (24.1)\n",
      "Collecting tyro (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading tyro-0.8.10-py3-none-any.whl.metadata (8.4 kB)\n",
      "Collecting transformers>=4.43.2 (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading transformers-4.44.2-py3-none-any.whl.metadata (43 kB)\n",
      "Requirement already satisfied: datasets>=2.16.0 in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2.20.0)\n",
      "Collecting sentencepiece>=0.2.0 (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (4.66.5)\n",
      "Collecting psutil (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n",
      "Collecting wheel>=0.42.0 (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached wheel-0.44.0-py3-none-any.whl.metadata (2.3 kB)\n",
      "Requirement already satisfied: accelerate>=0.26.1 in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.33.0)\n",
      "Requirement already satisfied: trl!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,>=0.7.9 in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.8.6)\n",
      "Requirement already satisfied: peft!=0.11.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.12.0)\n",
      "Collecting protobuf<4.0.0 (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (679 bytes)\n",
      "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.24.5)\n",
      "Collecting hf-transfer (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading hf_transfer-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.7 kB)\n",
      "Collecting numpy (from bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.26.1->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (6.0.2)\n",
      "Collecting safetensors>=0.3.1 (from accelerate>=0.26.1->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading safetensors-0.4.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (3.15.4)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (17.0.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2.2.2)\n",
      "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2.32.3)\n",
      "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.5.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.5.0,>=2023.1.0->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2024.5.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (3.10.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (4.12.2)\n",
      "Collecting sympy (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached sympy-1.13.2-py3-none-any.whl.metadata (12 kB)\n",
      "Collecting networkx (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached networkx-3.3-py3-none-any.whl.metadata (5.1 kB)\n",
      "Collecting jinja2 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached jinja2-3.1.4-py3-none-any.whl.metadata (2.6 kB)\n",
      "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cudnn-cu12==9.1.0.70 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-curand-cu12==10.3.2.106 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-nccl-cu12==2.20.5 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n",
      "Collecting nvidia-nvtx-cu12==12.1.105 (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n",
      "Requirement already satisfied: triton==3.0.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (3.0.0)\n",
      "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting regex!=2019.12.17 (from transformers>=4.43.2->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
      "Collecting tokenizers<0.20,>=0.19 (from transformers>=4.43.2->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
      "Collecting docstring-parser>=0.16 (from tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading docstring_parser-0.16-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting rich>=11.1.0 (from tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading rich-13.8.0-py3-none-any.whl.metadata (18 kB)\n",
      "Collecting shtab>=1.5.6 (from tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading shtab-1.7.1-py3-none-any.whl.metadata (7.3 kB)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2.3.5)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (4.0.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2.2.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2024.7.4)\n",
      "Collecting markdown-it-py>=2.2.0 (from rich>=11.1.0->tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading markdown_it_py-3.0.0-py3-none-any.whl.metadata (6.9 kB)\n",
      "Collecting pygments<3.0.0,>=2.13.0 (from rich>=11.1.0->tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading pygments-2.18.0-py3-none-any.whl.metadata (2.5 kB)\n",
      "Collecting MarkupSafe>=2.0 (from jinja2->torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.0 kB)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (2024.1)\n",
      "Collecting mpmath<1.4,>=1.1.0 (from sympy->torch->bitsandbytes>=0.43.3->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Using cached mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n",
      "Collecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[cu121-torch240]@ git+https://github.com/unslothai/unsloth.git) (1.16.0)\n",
      "Using cached numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)\n",
      "Downloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m44.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hUsing cached torch-2.4.0-cp310-cp310-manylinux1_x86_64.whl (797.2 MB)\n",
      "Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
      "Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
      "Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
      "Using cached nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl (664.8 MB)\n",
      "Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
      "Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
      "Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
      "Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
      "Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
      "Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
      "Downloading transformers-4.44.2-py3-none-any.whl (9.5 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.5/9.5 MB\u001b[0m \u001b[31m54.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading tyro-0.8.10-py3-none-any.whl (105 kB)\n",
      "Using cached wheel-0.44.0-py3-none-any.whl (67 kB)\n",
      "Downloading hf_transfer-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m53.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (290 kB)\n",
      "Downloading docstring_parser-0.16-py3-none-any.whl (36 kB)\n",
      "Downloading regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (776 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m776.5/776.5 kB\u001b[0m \u001b[31m43.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading rich-13.8.0-py3-none-any.whl (241 kB)\n",
      "Downloading safetensors-0.4.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (435 kB)\n",
      "Downloading shtab-1.7.1-py3-none-any.whl (14 kB)\n",
      "Downloading tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hUsing cached jinja2-3.1.4-py3-none-any.whl (133 kB)\n",
      "Using cached networkx-3.3-py3-none-any.whl (1.7 MB)\n",
      "Using cached sympy-1.13.2-py3-none-any.whl (6.2 MB)\n",
      "Downloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\n",
      "Downloading MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\n",
      "Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
      "Downloading pygments-2.18.0-py3-none-any.whl (1.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_x86_64.whl (19.7 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.7/19.7 MB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
      "Building wheels for collected packages: unsloth\n",
      "  Building wheel for unsloth (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for unsloth: filename=unsloth-2024.8-py3-none-any.whl size=145817 sha256=92e0c6d3a259cb764556ff6a2c44d1b2ebc196f4c251a847297893d053a9dbbf\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-0p8aot_2/wheels/ed/d4/e9/76fb290ee3df0a5fc21ce5c2c788e29e9607a2353d8342fd0d\n",
      "Successfully built unsloth\n",
      "Installing collected packages: sentencepiece, mpmath, wheel, unsloth, sympy, shtab, safetensors, regex, pygments, psutil, protobuf, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, numpy, networkx, mdurl, MarkupSafe, hf-transfer, docstring-parser, nvidia-cusparse-cu12, nvidia-cudnn-cu12, markdown-it-py, jinja2, tokenizers, rich, nvidia-cusolver-cu12, tyro, transformers, torch, xformers\n",
      "  Attempting uninstall: wheel\n",
      "    Found existing installation: wheel 0.37.1\n",
      "    Uninstalling wheel-0.37.1:\n",
      "      Successfully uninstalled wheel-0.37.1\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 2.0.1\n",
      "    Uninstalling numpy-2.0.1:\n",
      "      Successfully uninstalled numpy-2.0.1\n",
      "  Attempting uninstall: xformers\n",
      "    Found existing installation: xformers 0.0.26.post1\n",
      "    Uninstalling xformers-0.0.26.post1:\n",
      "      Successfully uninstalled xformers-0.0.26.post1\n",
      "Successfully installed MarkupSafe-2.1.5 docstring-parser-0.16 hf-transfer-0.1.8 jinja2-3.1.4 markdown-it-py-3.0.0 mdurl-0.1.2 mpmath-1.3.0 networkx-3.3 numpy-1.26.4 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.6.68 nvidia-nvtx-cu12-12.1.105 protobuf-3.20.3 psutil-6.0.0 pygments-2.18.0 regex-2024.7.24 rich-13.8.0 safetensors-0.4.4 sentencepiece-0.2.0 shtab-1.7.1 sympy-1.13.2 tokenizers-0.19.1 torch-2.4.0 transformers-4.44.2 tyro-0.8.10 unsloth-2024.8 wheel-0.44.0 xformers-0.0.27.post2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: xformers in /usr/local/lib/python3.10/dist-packages (0.0.27.post2)\n",
      "Requirement already satisfied: trl in /usr/local/lib/python3.10/dist-packages (0.8.6)\n",
      "Requirement already satisfied: peft in /usr/local/lib/python3.10/dist-packages (0.12.0)\n",
      "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.33.0)\n",
      "Requirement already satisfied: bitsandbytes in /usr/local/lib/python3.10/dist-packages (0.43.3)\n",
      "Requirement already satisfied: triton in /usr/local/lib/python3.10/dist-packages (3.0.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n",
    "!pip install --upgrade pip\n",
    "!pip install \"unsloth[cu121-torch240] @ git+https://github.com/unslothai/unsloth.git\"\n",
    "!pip install --no-deps xformers trl peft accelerate bitsandbytes triton"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2afe7ab9-7634-44c5-8ef0-090018f5c86c",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'unsloth'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01munsloth\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m FastLanguageModel\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdatasets\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Dataset\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'unsloth'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from unsloth import FastLanguageModel\n",
    "import torch\n",
    "from datasets import Dataset\n",
    "from trl import SFTTrainer\n",
    "from transformers import TrainingArguments\n",
    "from unsloth import is_bfloat16_supported\n",
    "\n",
    "# 모델 설정\n",
    "max_seq_length = 2048  # 시퀀스 최대 길이 설정. RoPE Scaling을 지원합니다.\n",
    "dtype = None  # 데이터 유형 자동 감지. Tesla T4, V100의 경우 Float16, Ampere 이상의 경우 Bfloat16 사용 가능\n",
    "load_in_4bit = True  # 메모리 사용량 감소를 위해 4비트 양자화 사용. False로 설정 가능\n",
    "\n",
    "# 4bit pre quantized models we support for 4x faster downloading + no OOMs.\n",
    "fourbit_models = [\n",
    "    \"unsloth/Meta-Llama-3.1-8B-bnb-4bit\",        # Llama-3.1 15 trillion tokens model 2x faster!\n",
    "    \"unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit\",\n",
    "    \"unsloth/Meta-Llama-3.1-70B-bnb-4bit\",\n",
    "    \"unsloth/Meta-Llama-3.1-405B-bnb-4bit\",       # We also uploaded 4bit for 405b!\n",
    "    \"unsloth/Mistral-Nemo-Base-2407-bnb-4bit\", # New Mistral 12b 2x faster!\n",
    "    \"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit\",\n",
    "    \"unsloth/mistral-7b-v0.3-bnb-4bit\",           # Mistral v3 2x faster!\n",
    "    \"unsloth/mistral-7b-instruct-v0.3-bnb-4bit\",\n",
    "    \"unsloth/Phi-3.5-mini-instruct\",             # Phi-3.5 2x faster!\n",
    "    \"unsloth/Phi-3-medium-4k-instruct\",\n",
    "    \"unsloth/gemma-2-9b-bnb-4bit\",\n",
    "    \"unsloth/gemma-2-27b-bnb-4bit\",               # Gemma 2x faster!\n",
    "] # More models at https://huggingface.co/unsloth\n",
    "\n",
    "# Llama 3.1 모델 및 토크나이저 로드\n",
    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
    "    model_name = \"unsloth/Meta-Llama-3.1-8B\",\n",
    "    max_seq_length = max_seq_length,\n",
    "    dtype = dtype,\n",
    "    load_in_4bit = load_in_4bit,\n",
    "    # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-h\n",
    ")\n",
    "\n",
    "# LoRA 설정 및 적용\n",
    "model = FastLanguageModel.get_peft_model(\n",
    "    model,\n",
    "    r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128\n",
    "    target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n",
    "                      \"gate_proj\", \"up_proj\", \"down_proj\",],\n",
    "    lora_alpha =  16,\n",
    "    lora_dropout = 0, # Supports any, but = 0 is optimized\n",
    "    bias = \"none\",    # Supports any, but = \"none\" is optimized\n",
    "    # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n",
    "    use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n",
    "    random_state = 3407,\n",
    "    use_rslora = False,  # We support rank stabilized LoRA\n",
    "    loftq_config = None, # And LoftQ\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
