From eb677979d4ce2579f588b4d540a638a615758ee1 Mon Sep 17 00:00:00 2001 From: Dorothee Date: Sun, 20 Jul 2025 16:41:56 +0200 Subject: [PATCH] Add notebook for quantizing tinyllama with SpQR --- QuantizeTinyLLama.ipynb | 350 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 QuantizeTinyLLama.ipynb diff --git a/QuantizeTinyLLama.ipynb b/QuantizeTinyLLama.ipynb new file mode 100644 index 0000000..5d8352b --- /dev/null +++ b/QuantizeTinyLLama.ipynb @@ -0,0 +1,350 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "bc2d31ca-c7c0-4720-810a-1bb08feff62c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting torch\n", + " Downloading torch-2.7.1-cp311-none-macosx_11_0_arm64.whl.metadata (29 kB)\n", + "Collecting filelock (from torch)\n", + " Downloading filelock-3.18.0-py3-none-any.whl.metadata (2.9 kB)\n", + "Requirement already satisfied: typing-extensions>=4.10.0 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from torch) (4.12.2)\n", + "Collecting sympy>=1.13.3 (from torch)\n", + " Downloading sympy-1.14.0-py3-none-any.whl.metadata (12 kB)\n", + "Collecting networkx (from torch)\n", + " Downloading networkx-3.5-py3-none-any.whl.metadata (6.3 kB)\n", + "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from torch) (3.1.6)\n", + "Collecting fsspec (from torch)\n", + " Downloading fsspec-2025.7.0-py3-none-any.whl.metadata (12 kB)\n", + "Collecting mpmath<1.4,>=1.1.0 (from sympy>=1.13.3->torch)\n", + " Downloading mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from jinja2->torch) (3.0.2)\n", + "Downloading torch-2.7.1-cp311-none-macosx_11_0_arm64.whl (68.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m68.6/68.6 MB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hDownloading sympy-1.14.0-py3-none-any.whl (6.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0mm\n", + "\u001b[?25hDownloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading filelock-3.18.0-py3-none-any.whl (16 kB)\n", + "Downloading fsspec-2025.7.0-py3-none-any.whl (199 kB)\n", + "Downloading networkx-3.5-py3-none-any.whl (2.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: mpmath, sympy, networkx, fsspec, filelock, torch\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6/6\u001b[0m [torch]32m5/6\u001b[0m [torch]]x]\n", + "\u001b[1A\u001b[2KSuccessfully installed filelock-3.18.0 fsspec-2025.7.0 mpmath-1.3.0 networkx-3.5 sympy-1.14.0 torch-2.7.1\n" + ] + } + ], + "source": [ + "!pip install torch " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "f65e39b7-54d9-4577-8cc2-64d94339b362", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting transformers\n", + " Downloading transformers-4.53.2-py3-none-any.whl.metadata (40 kB)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from transformers) (3.18.0)\n", + "Collecting huggingface-hub<1.0,>=0.30.0 (from transformers)\n", + " Downloading huggingface_hub-0.33.4-py3-none-any.whl.metadata (14 kB)\n", + "Requirement already satisfied: numpy>=1.17 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from transformers) (1.26.4)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from transformers) (25.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from transformers) (6.0.2)\n", + "Collecting regex!=2019.12.17 (from transformers)\n", + " Downloading regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl.metadata (40 kB)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from transformers) (2.32.4)\n", + "Collecting tokenizers<0.22,>=0.21 (from transformers)\n", + " Downloading tokenizers-0.21.2-cp39-abi3-macosx_11_0_arm64.whl.metadata (6.8 kB)\n", + "Collecting safetensors>=0.4.3 (from transformers)\n", + " Downloading safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl.metadata (3.8 kB)\n", + "Collecting tqdm>=4.27 (from transformers)\n", + " Downloading tqdm-4.67.1-py3-none-any.whl.metadata (57 kB)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (2025.7.0)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (4.12.2)\n", + "Collecting hf-xet<2.0.0,>=1.1.2 (from huggingface-hub<1.0,>=0.30.0->transformers)\n", + " Downloading hf_xet-1.1.5-cp37-abi3-macosx_11_0_arm64.whl.metadata (879 bytes)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from requests->transformers) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from requests->transformers) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from requests->transformers) (2.5.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages (from requests->transformers) (2025.7.14)\n", + "Downloading transformers-4.53.2-py3-none-any.whl (10.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.8/10.8 MB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading huggingface_hub-0.33.4-py3-none-any.whl (515 kB)\n", + "Downloading hf_xet-1.1.5-cp37-abi3-macosx_11_0_arm64.whl (2.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading tokenizers-0.21.2-cp39-abi3-macosx_11_0_arm64.whl (2.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl (284 kB)\n", + "Downloading safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl (418 kB)\n", + "Downloading tqdm-4.67.1-py3-none-any.whl (78 kB)\n", + "Installing collected packages: tqdm, safetensors, regex, hf-xet, huggingface-hub, tokenizers, transformers\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7/7\u001b[0m [transformers][0m [transformers]ub]\n", + "\u001b[1A\u001b[2KSuccessfully installed hf-xet-1.1.5 huggingface-hub-0.33.4 regex-2024.11.6 safetensors-0.5.3 tokenizers-0.21.2 tqdm-4.67.1 transformers-4.53.2\n" + ] + } + ], + "source": [ + "!pip install transformers" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "179a4065-ff16-40b0-a2ee-fa81f056702f", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/quantizeEnv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4.53.2\n" + ] + } + ], + "source": [ + "import transformers\n", + "\n", + "print(transformers.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "51eeda48-a98a-4e19-a403-98c4a90936ff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.7.1\n" + ] + } + ], + "source": [ + "import torch \n", + "print(torch.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c711bdc4-06c7-4492-ade8-ab3497cb5dcf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting git+https://github.com/Vahe1994/SpQR.git\n", + " Cloning https://github.com/Vahe1994/SpQR.git to /private/var/folders/x9/myx7g6t11gv0dghwblns7qww0000gn/T/pip-req-build-yrjhb217\n", + " Running command git clone --filter=blob:none --quiet https://github.com/Vahe1994/SpQR.git /private/var/folders/x9/myx7g6t11gv0dghwblns7qww0000gn/T/pip-req-build-yrjhb217\n", + " Resolved https://github.com/Vahe1994/SpQR.git to commit 902abdb012b24b07d0f4376bdc46952e4189add1\n", + " Installing build dependencies ... \u001b[?25ldone\n", + "\u001b[?25h Getting requirements to build wheel ... \u001b[?25lerror\n", + " \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n", + " \n", + " \u001b[31m×\u001b[0m \u001b[32mGetting requirements to build wheel\u001b[0m did not run successfully.\n", + " \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n", + " \u001b[31m╰─>\u001b[0m \u001b[31m[14 lines of output]\u001b[0m\n", + " \u001b[31m \u001b[0m error: Multiple top-level packages discovered in a flat-layout: ['data', 'inference_lib'].\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m To avoid accidental inclusion of unwanted files or directories,\n", + " \u001b[31m \u001b[0m setuptools will not proceed with this build.\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m If you are trying to create a single distribution with multiple packages\n", + " \u001b[31m \u001b[0m on purpose, you should not rely on automatic discovery.\n", + " \u001b[31m \u001b[0m Instead, consider the following options:\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m 1. set up custom discovery (`find` directive with `include` or `exclude`)\n", + " \u001b[31m \u001b[0m 2. use a `src-layout`\n", + " \u001b[31m \u001b[0m 3. explicitly set `py_modules` or `packages` with a list of names\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m To find more information, look for \"package discovery\" on setuptools docs.\n", + " \u001b[31m \u001b[0m \u001b[31m[end of output]\u001b[0m\n", + " \n", + " \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n", + "\u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n", + "\n", + "\u001b[31m×\u001b[0m \u001b[32mGetting requirements to build wheel\u001b[0m did not run successfully.\n", + "\u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n", + "\u001b[31m╰─>\u001b[0m See above for output.\n", + "\n", + "\u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n", + "\u001b[?25h" + ] + } + ], + "source": [ + "!pip install git+https://github.com/Vahe1994/SpQR.git" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "80983f78-45f4-48a4-9749-592c635eff05", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting git+https://github.com/Vahe1994/SpQR.git\n", + " Cloning https://github.com/Vahe1994/SpQR.git to /private/var/folders/x9/myx7g6t11gv0dghwblns7qww0000gn/T/pip-req-build-8n1i0qes\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " Running command git clone --filter=blob:none --quiet https://github.com/Vahe1994/SpQR.git /private/var/folders/x9/myx7g6t11gv0dghwblns7qww0000gn/T/pip-req-build-8n1i0qes\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Resolved https://github.com/Vahe1994/SpQR.git to commit 902abdb012b24b07d0f4376bdc46952e4189add1\n", + " Installing build dependencies: started\n", + " Installing build dependencies: finished with status 'done'\n", + " Getting requirements to build wheel: started\n", + " Getting requirements to build wheel: finished with status 'error'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n", + " \n", + " \u001b[31m×\u001b[0m \u001b[32mGetting requirements to build wheel\u001b[0m did not run successfully.\n", + " \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n", + " \u001b[31m╰─>\u001b[0m \u001b[31m[14 lines of output]\u001b[0m\n", + " \u001b[31m \u001b[0m error: Multiple top-level packages discovered in a flat-layout: ['data', 'inference_lib'].\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m To avoid accidental inclusion of unwanted files or directories,\n", + " \u001b[31m \u001b[0m setuptools will not proceed with this build.\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m If you are trying to create a single distribution with multiple packages\n", + " \u001b[31m \u001b[0m on purpose, you should not rely on automatic discovery.\n", + " \u001b[31m \u001b[0m Instead, consider the following options:\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m 1. set up custom discovery (`find` directive with `include` or `exclude`)\n", + " \u001b[31m \u001b[0m 2. use a `src-layout`\n", + " \u001b[31m \u001b[0m 3. explicitly set `py_modules` or `packages` with a list of names\n", + " \u001b[31m \u001b[0m \n", + " \u001b[31m \u001b[0m To find more information, look for \"package discovery\" on setuptools docs.\n", + " \u001b[31m \u001b[0m \u001b[31m[end of output]\u001b[0m\n", + " \n", + " \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n", + "\u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n", + "\n", + "\u001b[31m×\u001b[0m \u001b[32mGetting requirements to build wheel\u001b[0m did not run successfully.\n", + "\u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n", + "\u001b[31m╰─>\u001b[0m See above for output.\n", + "\n", + "\u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n" + ] + }, + { + "ename": "CalledProcessError", + "evalue": "Command '['/opt/anaconda3/envs/quantizeEnv/bin/python', '-m', 'pip', 'install', 'git+https://github.com/Vahe1994/SpQR.git']' returned non-zero exit status 1.", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mModuleNotFoundError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m 3\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mspqr\u001b[39;00m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n", + "\u001b[31mModuleNotFoundError\u001b[39m: No module named 'spqr'", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[31mCalledProcessError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mspqr\u001b[39;00m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m subprocess.check_call([sys.executable, \u001b[33m\"\u001b[39m\u001b[33m-m\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mpip\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33minstall\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mgit+https://github.com/Vahe1994/SpQR.git\u001b[39m\u001b[33m\"\u001b[39m])\n", + "\u001b[36mFile \u001b[39m\u001b[32m/opt/anaconda3/envs/quantizeEnv/lib/python3.11/subprocess.py:413\u001b[39m, in \u001b[36mcheck_call\u001b[39m\u001b[34m(*popenargs, **kwargs)\u001b[39m\n\u001b[32m 411\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m cmd \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 412\u001b[39m cmd = popenargs[\u001b[32m0\u001b[39m]\n\u001b[32m--> \u001b[39m\u001b[32m413\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CalledProcessError(retcode, cmd)\n\u001b[32m 414\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[32m0\u001b[39m\n", + "\u001b[31mCalledProcessError\u001b[39m: Command '['/opt/anaconda3/envs/quantizeEnv/bin/python', '-m', 'pip', 'install', 'git+https://github.com/Vahe1994/SpQR.git']' returned non-zero exit status 1." + ] + } + ], + "source": [ + "import subprocess, sys\n", + "\n", + "try:\n", + " import spqr\n", + "except ImportError:\n", + " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"git+https://github.com/Vahe1994/SpQR.git\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ec5aff0e-e93f-4150-b6e4-7365aa733b99", + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'spqr'", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mModuleNotFoundError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[5]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mspqr\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m quantize_model\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtransformers\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m AutoModelForCausalLM, AutoTokenizer\n\u001b[32m 3\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mhuggingface_hub\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m snapshot_download\n", + "\u001b[31mModuleNotFoundError\u001b[39m: No module named 'spqr'" + ] + } + ], + "source": [ + "from spqr import quantize_model\n", + "from transformers import AutoModelForCausalLM, AutoTokenizer\n", + "from huggingface_hub import snapshot_download" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "543f0bf8-af85-4c97-9e85-4d248a618409", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}