LamaLLM Updated.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bf2d13a0ccc546668e628a89cf6b6863": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_81024b01759a48f1be7e16413443cd60",
              "IPY_MODEL_14ddaeec0cb14d0daf7bcb768785ac8d",
              "IPY_MODEL_e7358323d6ec4e6ebe62a1506dccc681"
            ],
            "layout": "IPY_MODEL_0f778cba57f74b2ea891fea1c43e1998"
          }
        },
        "81024b01759a48f1be7e16413443cd60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6fc320863cdd4d99b376556cfc65d911",
            "placeholder": "​",
            "style": "IPY_MODEL_e56be80e92b64982827174622e270f78",
            "value": "llama-2-13b-chat.ggmlv3.q5_1.bin: 100%"
          }
        },
        "14ddaeec0cb14d0daf7bcb768785ac8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c0ad75499da648d3a4aa77fbd2ef8f59",
            "max": 9763701888,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_68587ca62e474ef4b435eeb700716f67",
            "value": 9763701888
          }
        },
        "e7358323d6ec4e6ebe62a1506dccc681": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7ef6813601143bcbf72267a407d306d",
            "placeholder": "​",
            "style": "IPY_MODEL_c333794522d04abcb858a6de0660e6e1",
            "value": " 9.76G/9.76G [01:13&lt;00:00, 187MB/s]"
          }
        },
        "0f778cba57f74b2ea891fea1c43e1998": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6fc320863cdd4d99b376556cfc65d911": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e56be80e92b64982827174622e270f78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0ad75499da648d3a4aa77fbd2ef8f59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68587ca62e474ef4b435eeb700716f67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d7ef6813601143bcbf72267a407d306d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c333794522d04abcb858a6de0660e6e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# GPU llama-cpp-python\n",
        "!CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" FORCE_CMAKE=1 pip install llama-cpp-python==0.1.78 numpy==1.23.4 --force-reinstall --upgrade --no-cache-dir --verbose\n",
        "!pip install huggingface_hub\n",
        "!pip install llama-cpp-python==0.1.78\n",
        "!pip install numpy==1.23.4"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-mbILqAIWsx9",
        "outputId": "b54443dc-b40a-4972-95c0-7ce28b5a9443"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using pip 23.1.2 from /usr/local/lib/python3.10/dist-packages/pip (python 3.10)\n",
            "Collecting llama-cpp-python==0.1.78\n",
            "  Downloading llama_cpp_python-0.1.78.tar.gz (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Running command pip subprocess to install build dependencies\n",
            "  Using pip 23.1.2 from /usr/local/lib/python3.10/dist-packages/pip (python 3.10)\n",
            "  Collecting setuptools>=42\n",
            "    Downloading setuptools-69.1.1-py3-none-any.whl (819 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 819.3/819.3 kB 5.4 MB/s eta 0:00:00\n",
            "  Collecting scikit-build>=0.13\n",
            "    Downloading scikit_build-0.17.6-py3-none-any.whl (84 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 84.3/84.3 kB 12.1 MB/s eta 0:00:00\n",
            "  Collecting cmake>=3.18\n",
            "    Downloading cmake-3.28.3-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (26.3 MB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 26.3/26.3 MB 37.5 MB/s eta 0:00:00\n",
            "  Collecting ninja\n",
            "    Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl (307 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 307.2/307.2 kB 32.2 MB/s eta 0:00:00\n",
            "  Collecting distro (from scikit-build>=0.13)\n",
            "    Downloading distro-1.9.0-py3-none-any.whl (20 kB)\n",
            "  Collecting packaging (from scikit-build>=0.13)\n",
            "    Downloading packaging-23.2-py3-none-any.whl (53 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 53.0/53.0 kB 6.5 MB/s eta 0:00:00\n",
            "  Collecting tomli (from scikit-build>=0.13)\n",
            "    Downloading tomli-2.0.1-py3-none-any.whl (12 kB)\n",
            "  Collecting wheel>=0.32.0 (from scikit-build>=0.13)\n",
            "    Downloading wheel-0.42.0-py3-none-any.whl (65 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.4/65.4 kB 6.6 MB/s eta 0:00:00\n",
            "  Installing collected packages: ninja, cmake, wheel, tomli, setuptools, packaging, distro, scikit-build\n",
            "    Creating /tmp/pip-build-env-2eostsai/overlay/local/bin\n",
            "    changing mode of /tmp/pip-build-env-2eostsai/overlay/local/bin/ninja to 755\n",
            "    changing mode of /tmp/pip-build-env-2eostsai/overlay/local/bin/cmake to 755\n",
            "    changing mode of /tmp/pip-build-env-2eostsai/overlay/local/bin/cpack to 755\n",
            "    changing mode of /tmp/pip-build-env-2eostsai/overlay/local/bin/ctest to 755\n",
            "    changing mode of /tmp/pip-build-env-2eostsai/overlay/local/bin/wheel to 755\n",
            "    changing mode of /tmp/pip-build-env-2eostsai/overlay/local/bin/distro to 755\n",
            "  ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "  ipython 7.34.0 requires jedi>=0.16, which is not installed.\n",
            "  Successfully installed cmake-3.28.3 distro-1.9.0 ninja-1.11.1.1 packaging-23.2 scikit-build-0.17.6 setuptools-69.1.1 tomli-2.0.1 wheel-0.42.0\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Running command Getting requirements to build wheel\n",
            "  running egg_info\n",
            "  writing llama_cpp_python.egg-info/PKG-INFO\n",
            "  writing dependency_links to llama_cpp_python.egg-info/dependency_links.txt\n",
            "  writing requirements to llama_cpp_python.egg-info/requires.txt\n",
            "  writing top-level names to llama_cpp_python.egg-info/top_level.txt\n",
            "  reading manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  adding license file 'LICENSE.md'\n",
            "  writing manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Running command Preparing metadata (pyproject.toml)\n",
            "  running dist_info\n",
            "  creating /tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info\n",
            "  writing /tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info/PKG-INFO\n",
            "  writing dependency_links to /tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info/dependency_links.txt\n",
            "  writing requirements to /tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info/requires.txt\n",
            "  writing top-level names to /tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info/top_level.txt\n",
            "  writing manifest file '/tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  reading manifest file '/tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  adding license file 'LICENSE.md'\n",
            "  writing manifest file '/tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  creating '/tmp/pip-modern-metadata-_v1n4mjq/llama_cpp_python-0.1.78.dist-info'\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting numpy==1.23.4\n",
            "  Downloading numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.1/17.1 MB\u001b[0m \u001b[31m90.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions>=4.5.0 (from llama-cpp-python==0.1.78)\n",
            "  Downloading typing_extensions-4.10.0-py3-none-any.whl (33 kB)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python==0.1.78)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m161.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: llama-cpp-python\n",
            "  Running command Building wheel for llama-cpp-python (pyproject.toml)\n",
            "\n",
            "\n",
            "  --------------------------------------------------------------------------------\n",
            "  -- Trying 'Ninja' generator\n",
            "  --------------------------------\n",
            "  ---------------------------\n",
            "  ----------------------\n",
            "  -----------------\n",
            "  ------------\n",
            "  -------\n",
            "  --\n",
            "  CMake Deprecation Warning at CMakeLists.txt:1 (cmake_minimum_required):\n",
            "    Compatibility with CMake < 3.5 will be removed from a future version of\n",
            "    CMake.\n",
            "\n",
            "    Update the VERSION argument <min> value or use a ...<max> suffix to tell\n",
            "    CMake that the project does not need compatibility with older versions.\n",
            "\n",
            "  Not searching for unused variables given on the command line.\n",
            "\n",
            "  -- The C compiler identification is GNU 11.4.0\n",
            "  -- Detecting C compiler ABI info\n",
            "  -- Detecting C compiler ABI info - done\n",
            "  -- Check for working C compiler: /usr/bin/cc - skipped\n",
            "  -- Detecting C compile features\n",
            "  -- Detecting C compile features - done\n",
            "  -- The CXX compiler identification is GNU 11.4.0\n",
            "  -- Detecting CXX compiler ABI info\n",
            "  -- Detecting CXX compiler ABI info - done\n",
            "  -- Check for working CXX compiler: /usr/bin/c++ - skipped\n",
            "  -- Detecting CXX compile features\n",
            "  -- Detecting CXX compile features - done\n",
            "  -- Configuring done (0.9s)\n",
            "  -- Generating done (0.0s)\n",
            "  -- Build files have been written to: /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_cmake_test_compile/build\n",
            "  --\n",
            "  -------\n",
            "  ------------\n",
            "  -----------------\n",
            "  ----------------------\n",
            "  ---------------------------\n",
            "  --------------------------------\n",
            "  -- Trying 'Ninja' generator - success\n",
            "  --------------------------------------------------------------------------------\n",
            "\n",
            "  Configuring Project\n",
            "    Working directory:\n",
            "      /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-build\n",
            "    Command:\n",
            "      /tmp/pip-build-env-2eostsai/overlay/local/lib/python3.10/dist-packages/cmake/data/bin/cmake /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2 -G Ninja -DCMAKE_MAKE_PROGRAM:FILEPATH=/tmp/pip-build-env-2eostsai/overlay/local/lib/python3.10/dist-packages/ninja/data/bin/ninja --no-warn-unused-cli -DCMAKE_INSTALL_PREFIX:PATH=/tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install -DPYTHON_VERSION_STRING:STRING=3.10.12 -DSKBUILD:INTERNAL=TRUE -DCMAKE_MODULE_PATH:PATH=/tmp/pip-build-env-2eostsai/overlay/local/lib/python3.10/dist-packages/skbuild/resources/cmake -DPYTHON_EXECUTABLE:PATH=/usr/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/usr/include/python3.10 -DPYTHON_LIBRARY:PATH=/usr/lib/x86_64-linux-gnu/libpython3.10.so -DPython_EXECUTABLE:PATH=/usr/bin/python3 -DPython_ROOT_DIR:PATH=/usr -DPython_FIND_REGISTRY:STRING=NEVER -DPython_INCLUDE_DIR:PATH=/usr/include/python3.10 -DPython3_EXECUTABLE:PATH=/usr/bin/python3 -DPython3_ROOT_DIR:PATH=/usr -DPython3_FIND_REGISTRY:STRING=NEVER -DPython3_INCLUDE_DIR:PATH=/usr/include/python3.10 -DCMAKE_MAKE_PROGRAM:FILEPATH=/tmp/pip-build-env-2eostsai/overlay/local/lib/python3.10/dist-packages/ninja/data/bin/ninja -DLLAMA_CUBLAS=on -DCMAKE_BUILD_TYPE:STRING=Release -DLLAMA_CUBLAS=on\n",
            "\n",
            "  Not searching for unused variables given on the command line.\n",
            "  -- The C compiler identification is GNU 11.4.0\n",
            "  -- The CXX compiler identification is GNU 11.4.0\n",
            "  -- Detecting C compiler ABI info\n",
            "  -- Detecting C compiler ABI info - done\n",
            "  -- Check for working C compiler: /usr/bin/cc - skipped\n",
            "  -- Detecting C compile features\n",
            "  -- Detecting C compile features - done\n",
            "  -- Detecting CXX compiler ABI info\n",
            "  -- Detecting CXX compiler ABI info - done\n",
            "  -- Check for working CXX compiler: /usr/bin/c++ - skipped\n",
            "  -- Detecting CXX compile features\n",
            "  -- Detecting CXX compile features - done\n",
            "  -- Found Git: /usr/bin/git (found version \"2.34.1\")\n",
            "  fatal: not a git repository (or any of the parent directories): .git\n",
            "  fatal: not a git repository (or any of the parent directories): .git\n",
            "  CMake Warning at vendor/llama.cpp/CMakeLists.txt:117 (message):\n",
            "    Git repository not found; to enable automatic generation of build info,\n",
            "    make sure Git is installed and the project is a Git repository.\n",
            "\n",
            "\n",
            "  -- Performing Test CMAKE_HAVE_LIBC_PTHREAD\n",
            "  -- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success\n",
            "  -- Found Threads: TRUE\n",
            "  -- Found CUDAToolkit: /usr/local/cuda/targets/x86_64-linux/include (found version \"12.2.140\")\n",
            "  -- cuBLAS found\n",
            "  -- The CUDA compiler identification is NVIDIA 12.2.140\n",
            "  -- Detecting CUDA compiler ABI info\n",
            "  -- Detecting CUDA compiler ABI info - done\n",
            "  -- Check for working CUDA compiler: /usr/local/cuda/bin/nvcc - skipped\n",
            "  -- Detecting CUDA compile features\n",
            "  -- Detecting CUDA compile features - done\n",
            "  -- Using CUDA architectures: 52;61;70\n",
            "  -- CMAKE_SYSTEM_PROCESSOR: x86_64\n",
            "  -- x86 detected\n",
            "  -- Configuring done (3.7s)\n",
            "  -- Generating done (0.0s)\n",
            "  -- Build files have been written to: /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-build\n",
            "  [1/9] Building C object vendor/llama.cpp/CMakeFiles/ggml.dir/ggml-alloc.c.o\n",
            "  [2/9] Building C object vendor/llama.cpp/CMakeFiles/ggml.dir/k_quants.c.o\n",
            "  [3/9] Building C object vendor/llama.cpp/CMakeFiles/ggml.dir/ggml.c.o\n",
            "  [4/9] Building CXX object vendor/llama.cpp/CMakeFiles/llama.dir/llama.cpp.o\n",
            "  [5/9] Building CUDA object vendor/llama.cpp/CMakeFiles/ggml.dir/ggml-cuda.cu.o\n",
            "  [6/9] Linking CUDA shared library vendor/llama.cpp/libggml_shared.so\n",
            "  [7/9] Linking CXX shared library vendor/llama.cpp/libllama.so\n",
            "  [8/9] Linking CUDA static library vendor/llama.cpp/libggml_static.a\n",
            "  [8/9] Install the project...\n",
            "  -- Install configuration: \"Release\"\n",
            "  -- Installing: /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install/lib/libggml_shared.so\n",
            "  -- Installing: /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install/lib/libllama.so\n",
            "  -- Set non-toolchain portion of runtime path of \"/tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install/lib/libllama.so\" to \"\"\n",
            "  -- Installing: /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install/bin/convert.py\n",
            "  -- Installing: /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install/bin/convert-lora-to-ggml.py\n",
            "  -- Installing: /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/libllama.so\n",
            "  -- Set non-toolchain portion of runtime path of \"/tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/_skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/libllama.so\" to \"\"\n",
            "\n",
            "  copying llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/utils.py\n",
            "  copying llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_grammar.py\n",
            "  copying llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_cpp.py\n",
            "  copying llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_types.py\n",
            "  copying llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama.py\n",
            "  copying llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/__init__.py\n",
            "  creating directory _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server\n",
            "  copying llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/app.py\n",
            "  copying llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__main__.py\n",
            "  copying llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__init__.py\n",
            "  copying /tmp/pip-install-tpj7l7pr/llama-cpp-python_51cc007e4766430d94d7c074c6b733d2/llama_cpp/py.typed -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/py.typed\n",
            "\n",
            "  running bdist_wheel\n",
            "  running build\n",
            "  running build_py\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/py.typed -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/libllama.so -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copied 9 files\n",
            "  running build_ext\n",
            "  installing to _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel\n",
            "  running install\n",
            "  running install_lib\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/libllama.so -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/py.typed -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copied 11 files\n",
            "  running install_data\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/lib\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/lib/libggml_shared.so -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/lib\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/lib/libllama.so -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/lib\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/bin\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/bin/convert.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/bin\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/bin/convert-lora-to-ggml.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/bin\n",
            "  running install_egg_info\n",
            "  running egg_info\n",
            "  writing llama_cpp_python.egg-info/PKG-INFO\n",
            "  writing dependency_links to llama_cpp_python.egg-info/dependency_links.txt\n",
            "  writing requirements to llama_cpp_python.egg-info/requires.txt\n",
            "  writing top-level names to llama_cpp_python.egg-info/top_level.txt\n",
            "  reading manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  adding license file 'LICENSE.md'\n",
            "  writing manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  Copying llama_cpp_python.egg-info to _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78-py3.10.egg-info\n",
            "  running install_scripts\n",
            "  copied 0 files\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.dist-info/WHEEL\n",
            "  creating '/tmp/pip-wheel-8mr9k8ip/.tmp-nvjsdq7u/llama_cpp_python-0.1.78-cp310-cp310-linux_x86_64.whl' and adding '_skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel' to it\n",
            "  adding 'llama_cpp/__init__.py'\n",
            "  adding 'llama_cpp/libllama.so'\n",
            "  adding 'llama_cpp/llama.py'\n",
            "  adding 'llama_cpp/llama_cpp.py'\n",
            "  adding 'llama_cpp/llama_grammar.py'\n",
            "  adding 'llama_cpp/llama_types.py'\n",
            "  adding 'llama_cpp/py.typed'\n",
            "  adding 'llama_cpp/utils.py'\n",
            "  adding 'llama_cpp/server/__init__.py'\n",
            "  adding 'llama_cpp/server/__main__.py'\n",
            "  adding 'llama_cpp/server/app.py'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/bin/convert-lora-to-ggml.py'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/bin/convert.py'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/lib/libggml_shared.so'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/lib/libllama.so'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/LICENSE.md'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/METADATA'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/WHEEL'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/top_level.txt'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/RECORD'\n",
            "  removing _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.1.78-cp310-cp310-linux_x86_64.whl size=5811107 sha256=3dba952e2712b70fad2f1d5719672f59e23d401dd104dcf83a433b7605860ef1\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-d_u03onm/wheels/61/f9/20/9ca660a9d3f2a47e44217059409478865948b5c8a1cba70030\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: typing-extensions, numpy, diskcache, llama-cpp-python\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing_extensions 4.9.0\n",
            "    Uninstalling typing_extensions-4.9.0:\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/__pycache__/typing_extensions.cpython-310.pyc\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/typing_extensions-4.9.0.dist-info/\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/typing_extensions.py\n",
            "      Successfully uninstalled typing_extensions-4.9.0\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.25.2\n",
            "    Uninstalling numpy-1.25.2:\n",
            "      Removing file or directory /usr/local/bin/f2py\n",
            "      Removing file or directory /usr/local/bin/f2py3\n",
            "      Removing file or directory /usr/local/bin/f2py3.10\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/numpy-1.25.2.dist-info/\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/numpy.libs/\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/numpy/\n",
            "      Successfully uninstalled numpy-1.25.2\n",
            "  changing mode of /usr/local/bin/f2py to 755\n",
            "  changing mode of /usr/local/bin/f2py3 to 755\n",
            "  changing mode of /usr/local/bin/f2py3.10 to 755\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "chex 0.1.85 requires numpy>=1.24.1, but you have numpy 1.23.4 which is incompatible.\n",
            "tensorflow 2.15.0 requires numpy<2.0.0,>=1.23.5, but you have numpy 1.23.4 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed diskcache-5.6.3 llama-cpp-python-0.1.78 numpy-1.23.4 typing-extensions-4.10.0\n",
            "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.20.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.66.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.10.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (23.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2024.2.2)\n",
            "Requirement already satisfied: llama-cpp-python==0.1.78 in /usr/local/lib/python3.10/dist-packages (0.1.78)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python==0.1.78) (4.10.0)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python==0.1.78) (1.23.4)\n",
            "Requirement already satisfied: diskcache>=5.6.1 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python==0.1.78) (5.6.3)\n",
            "Requirement already satisfied: numpy==1.23.4 in /usr/local/lib/python3.10/dist-packages (1.23.4)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!nvcc --version"
      ],
      "metadata": {
        "id": "Kb5EnW5UjHkD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9880ee5d-203b-414a-c5cb-068b2276a163"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2023 NVIDIA Corporation\n",
            "Built on Tue_Aug_15_22:02:13_PDT_2023\n",
            "Cuda compilation tools, release 12.2, V12.2.140\n",
            "Build cuda_12.2.r12.2/compiler.33191640_0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K3ZcrIAoVeex"
      },
      "outputs": [],
      "source": [
        "model_name_or_path = \"TheBloke/Llama-2-13B-chat-GGML\"\n",
        "model_basename = \"llama-2-13b-chat.ggmlv3.q5_1.bin\" # the model is in bin format"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import hf_hub_download"
      ],
      "metadata": {
        "id": "y9jVwU54foOl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_cpp import Llama"
      ],
      "metadata": {
        "id": "Ez5qcEcCf5BI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_path = hf_hub_download(repo_id=model_name_or_path, filename=model_basename)"
      ],
      "metadata": {
        "id": "box7wI_ef6rX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153,
          "referenced_widgets": [
            "bf2d13a0ccc546668e628a89cf6b6863",
            "81024b01759a48f1be7e16413443cd60",
            "14ddaeec0cb14d0daf7bcb768785ac8d",
            "e7358323d6ec4e6ebe62a1506dccc681",
            "0f778cba57f74b2ea891fea1c43e1998",
            "6fc320863cdd4d99b376556cfc65d911",
            "e56be80e92b64982827174622e270f78",
            "c0ad75499da648d3a4aa77fbd2ef8f59",
            "68587ca62e474ef4b435eeb700716f67",
            "d7ef6813601143bcbf72267a407d306d",
            "c333794522d04abcb858a6de0660e6e1"
          ]
        },
        "outputId": "38db81dc-358d-4fbe-8dc7-f9b62a61f07d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "llama-2-13b-chat.ggmlv3.q5_1.bin:   0%|          | 0.00/9.76G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bf2d13a0ccc546668e628a89cf6b6863"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# GPU\n",
        "lcpp_llm = None\n",
        "lcpp_llm = Llama(\n",
        "    model_path=model_path,\n",
        "    n_threads=2, # CPU cores\n",
        "    n_batch=512, # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.\n",
        "    n_gpu_layers=32 # Change this value based on your model and your GPU VRAM pool.\n",
        "    )"
      ],
      "metadata": {
        "id": "RSp8fKrIgA_h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5b1efa73-d16d-470d-f2bd-d49d5f968a2f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | VSX = 0 | \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# See the number of layers in GPU\n",
        "lcpp_llm.params.n_gpu_layers"
      ],
      "metadata": {
        "id": "T6ICBFcqgCzh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2af65aa-baf0-4cf2-98ae-27e77188b568"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install -y poppler-utils\n"
      ],
      "metadata": {
        "id": "HXCYD9dzOo9V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4ceea1cf-1701-4efc-c5ff-09c02266dda3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  poppler-utils\n",
            "0 upgraded, 1 newly installed, 0 to remove and 35 not upgraded.\n",
            "Need to get 186 kB of archives.\n",
            "After this operation, 696 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 poppler-utils amd64 22.02.0-2ubuntu0.3 [186 kB]\n",
            "Fetched 186 kB in 1s (230 kB/s)\n",
            "Selecting previously unselected package poppler-utils.\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../poppler-utils_22.02.0-2ubuntu0.3_amd64.deb ...\n",
            "Unpacking poppler-utils (22.02.0-2ubuntu0.3) ...\n",
            "Setting up poppler-utils (22.02.0-2ubuntu0.3) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pdftotext\n"
      ],
      "metadata": {
        "id": "40DjT515SdJy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96c321ed-c8ee-4140-f374-126c55acf28c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pdftotext\n",
            "  Downloading pdftotext-2.2.2.tar.gz (113 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.9/113.9 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: pdftotext\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py bdist_wheel\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Building wheel for pdftotext (setup.py) ... \u001b[?25lerror\n",
            "\u001b[31m  ERROR: Failed building wheel for pdftotext\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[?25h  Running setup.py clean for pdftotext\n",
            "Failed to build pdftotext\n",
            "\u001b[31mERROR: Could not build wheels for pdftotext, which is required to install pyproject.toml-based projects\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install PyPDF2"
      ],
      "metadata": {
        "id": "rhFjXgt7ZZt9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f56e2e5e-9675-4dcd-8772-562f12676e34"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: PyPDF2 in /usr/local/lib/python3.10/dist-packages (3.0.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import PyPDF2\n",
        "\n",
        "def extract_text_from_pdf(pdf_file: str) -> [str]:\n",
        "  with open(pdf_file,'rb') as pdf:\n",
        "    reader = PyPDF2.PdfReader(pdf)\n",
        "    pdf_text = []\n",
        "    for page in reader.pages:\n",
        "     content = page.extract_text()\n",
        "     pdf_text.append(content)\n",
        "     return pdf_text\n"
      ],
      "metadata": {
        "id": "t01g9AZwOf4B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = extract_text_from_pdf(\"rental.pdf\")"
      ],
      "metadata": {
        "id": "YfXE7FAuCLXG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(text)\n",
        "\n",
        "str1 = \"\".join(text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "stpMY-kskkH0",
        "outputId": "6746fef5-1eef-4020-88a0-fccebd34e436"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[' \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nRENEWAL OF RENTAL AGREEMENT  \\nThis AGREEMENT of Rent is made in Bangalore and Executed today the lstth of May \\n2010 \\nBY AND BETWEEN  \\n1. Mr. Balaji.R  \\nAged about 63 years,  \\nNo 24 2nd Cross, SBM \\nColony Mathikere - 560054  \\nHereinafter referred and called as the ‘Lessor’ of the First part of one part:  \\n//AND//  \\n1 Mr.Kartheek R \\nAged about 31 years,  \\nNo.81, sri manjunatha nilaya, raju colony, \\nyamalur  \\nB angalore -560037.  \\nHereinafter referred and called as the ‘Lessees’ of the second  part of the another part:   ']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import List\n",
        "\n",
        "prompt_template= f'''*SYSTEM:* I am a document classification assistant. I analyzed the document and predict the most likely class in one word:\n",
        "\n",
        "*Predicted Class (One Word):*\n",
        "\n",
        "*USER:*'''+ str1[:50]"
      ],
      "metadata": {
        "id": "Rq50NhpzgQ3g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "response=lcpp_llm(prompt=prompt_template, max_tokens=256, temperature=0.5, top_p=0.95,\n",
        "                  repeat_penalty=1.2, top_k=150,\n",
        "                  echo=True)"
      ],
      "metadata": {
        "id": "NBr3nMyOvDKR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4d8492bd-eabd-41e2-e48e-f964a50ee520"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response[\"choices\"][0][\"text\"])"
      ],
      "metadata": {
        "id": "Wc4dJZKQvJjK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a5a0d772-ee26-4f7c-80f3-ecf1f8c66ba5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "*SYSTEM:* I am a document classification assistant. I analyzed the document and predict the most likely class in one word:\n",
            "\n",
            "*Predicted Class (One Word):*\n",
            "\n",
            "*USER:* \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            " \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response)"
      ],
      "metadata": {
        "id": "853eWkgYVP3w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3485ccbd-8887-42fd-bf75-b1f3d8531138"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id': 'cmpl-8f86ae46-033f-4f88-bacd-17f9a5117879', 'object': 'text_completion', 'created': 1708955844, 'model': '/root/.cache/huggingface/hub/models--TheBloke--Llama-2-13B-chat-GGML/snapshots/3140827b4dfcb6b562cd87ee3d7f07109b014dd0/llama-2-13b-chat.ggmlv3.q5_1.bin', 'choices': [{'text': '*SYSTEM:* I am a document classification assistant. I analyzed the document and predict the most likely class in one word:\\n\\n*Predicted Class (One Word):*\\n\\n*USER:* \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n', 'index': 0, 'logprobs': None, 'finish_reason': 'length'}], 'usage': {'prompt_tokens': 96, 'completion_tokens': 256, 'total_tokens': 352}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AT_OIbZEWIz4"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}