From 21cd5c7a56bbedd1915993264d8b12301aa75347 Mon Sep 17 00:00:00 2001 From: lygitdata Date: Fri, 30 Jan 2026 08:25:54 +0000 Subject: [PATCH 1/3] Initial commit --- Dockerfile | 34 ++ LICENSE | 373 ++++++++++++++++++ README.md | 134 +++++++ docker-compose.yml | 11 + requirements.txt | 4 + ...ion_model_fine_tuning_advanced_usage.ipynb | 178 +++++++++ ...l_training_evaluation_advanced_usage.ipynb | 278 +++++++++++++ .../classification_quick_start.ipynb | 163 ++++++++ ...asurement_instruction_advanced_usage.ipynb | 278 +++++++++++++ .../landmark_detection_quick_start.ipynb | 141 +++++++ ...efinement_and_derivation_quick_start.ipynb | 208 ++++++++++ .../segmentation_quick_start.ipynb | 121 ++++++ .../tailor_quick_start.ipynb | 223 +++++++++++ .../web_landmark_detection/main.ipynb | 163 ++++++++ working/examples/web_segmentation/main.ipynb | 167 ++++++++ working/examples/web_tailor/main.ipynb | 273 +++++++++++++ 16 files changed, 2749 insertions(+) create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb create mode 100644 working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb create mode 100644 working/examples/classification_quick_start/classification_quick_start.ipynb create mode 100644 working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb create mode 100644 working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb create mode 100644 working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb create mode 100644 working/examples/segmentation_quick_start/segmentation_quick_start.ipynb create mode 100644 working/examples/tailor_quick_start/tailor_quick_start.ipynb create mode 100644 working/examples/web_landmark_detection/main.ipynb create mode 100644 working/examples/web_segmentation/main.ipynb create mode 100644 working/examples/web_tailor/main.ipynb diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e52bb61 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# Use official Python base image +FROM python:3.11.12 + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + git \ + wget \ + libgl1 \ + libglib2.0-0 \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements and install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Install JupyterLab and ipywidgets to suppress tqdm warnings +RUN pip install --no-cache-dir jupyterlab==4.4.1 ipywidgets + +# Suppress matplotlib.font_manager INFO logs using a hidden startup script +RUN mkdir -p /etc/python_startup && \ + echo "import logging; logging.basicConfig(level=logging.WARNING)" > /etc/python_startup/startup.py + +# Run the startup script for all Python sessions +ENV PYTHONSTARTUP=/etc/python_startup/startup.py + +# Expose JupyterLab port +EXPOSE 8888 + +# Set default command to start JupyterLab +CMD ["jupyter", "lab", "--ip=0.0.0.0", "--port=8888", "--allow-root", "--NotebookApp.token=''"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a612ad9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/README.md b/README.md new file mode 100644 index 0000000..922bc13 --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +# GarmentIQ MagicBox (Version 1.7) + +*Last update: 06/20/2025* + +*Navigation*: + +1. [How to install GarmentIQ MagicBox?](#how-to-install-garmentiq-magicbox) +2. [How to stop / uninstall GarmentIQ MagicBox?](#how-to-stop--uninstall-garmentiq-magicbox) +3. [How to run code in GarmentIQ MagicBox?](#how-to-run-code-in-garmentiq-magicbox) + +--- + +![image](https://github.com/user-attachments/assets/bd40b7b6-941c-429f-a160-0dd0eae0d295) + +GarmentIQ MagicBox is a streamlined, Docker-based JupyterLab environment designed to simplify the use of the GarmentIQ Python API. By packaging the development tools and dependencies into a Docker container, MagicBox ensures a consistent and reproducible environment for all users—eliminating the need for complex local setup or manual dependency management. Whether you're analyzing garment data, prototyping models, or running production-grade scripts, MagicBox offers a plug-and-play solution that gets you up and running quickly and reliably. + +## How to install GarmentIQ MagicBox? + +Before installing GarmentIQ MagicBox, ensure that Docker is installed and running on your system. For optimal performance, it is recommended to have at least 16 GB of RAM, an NVIDIA GPU with CUDA support and a minimum of 4 GB of GPU memory, and at least 20 GB space available on disk. + +### Step 1: Download GarmentIQ MagicBox configuration files + +Choosing a preferred directory. + +- For Windows, run the following command in Windows command prompt. + + ```bash + powershell -Command "Invoke-WebRequest -Uri 'https://github.com/lygitdata/GarmentIQ/archive/refs/heads/magicbox.zip' -OutFile 'magicbox.zip'; Expand-Archive -Path 'magicbox.zip' -DestinationPath .; Rename-Item 'GarmentIQ-magicbox' 'garmentiq_magicbox'; Remove-Item 'magicbox.zip'" + ``` + +- For Linux / MacOS, run the following command in terminal. Make sure you have already installed `curl` and `unzip`. + + ```bash + curl -L -o magicbox.zip https://github.com/lygitdata/GarmentIQ/archive/refs/heads/magicbox.zip && unzip magicbox.zip && mv GarmentIQ-magicbox garmentiq_magicbox && rm magicbox.zip + ``` + +### Step 2: Switch the directory + +Run the following command. + +```bash +cd garmentiq_magicbox +``` + +### Step 3: Build the Docker image + +Run the following command to build the Docker image. Make sure you have already installed Docker. This process is time consuming. Make sure your have a stable internet connection. + +```bash +docker build --no-cache -t garmentiq_magicbox . +``` + +### Step 4: Run the Docker container + +- For Windows, run one of the following commands in Windows command prompt. + + - (Recommended) Run with GPU in addition to CPU. + + ```bash + docker run -d --name magicbox_container -p 8888:8888 -p 5000:5000 -p 5001:5001 -p 5002:5002 --gpus all -v "%cd%\working:/app/working" garmentiq_magicbox + ``` + + - Run with CPU only. + + ```bash + docker run -d --name magicbox_container -p 8888:8888 -p 5000:5000 -p 5001:5001 -p 5002:5002 -v "%cd%\working:/app/working" garmentiq_magicbox + ``` + +- For Linux / MacOS, run the following command in terminal. + + - (Recommended) Run with GPU in addition to CPU. + + ```bash + docker run -d --name magicbox_container -p 8888:8888 -p 5000:5000 -p 5001:5001 -p 5002:5002 --gpus all -v "$(pwd)/working:/app/working" garmentiq_magicbox + ``` + + - Run with CPU only. + + ```bash + docker run -d --name magicbox_container -p 8888:8888 -p 5000:5000 -p 5001:5001 -p 5002:5002 -v "$(pwd)/working:/app/working" garmentiq_magicbox + ``` + +### Step 5: Start GarmentIQ MagicBox + +Open your browser, type http://127.0.0.1:8888 to access the Jupyter Lab interface of GarmentIQ MagicBox. + +## How to stop / uninstall GarmentIQ MagicBox? + +### Stop and Remove the Docker Container + +- To stop the Docker container, run the following command. + + ```bash + docker stop magicbox_container + ``` + +- To remove the Docker container, run the following command. + + ```bash + docker rm magicbox_container + ``` + +### Uninstall the Docker Image + +To remove the Docker image, run the following command. + +```bash +docker rmi garmentiq_magicbox +``` + +## How to Run Code in GarmentIQ MagicBox + +To run code in GarmentIQ MagicBox, follow the steps below: + +### 1. Prepare Your Local Directory +- Place all your configuration files for GarmentIQ MagicBox in your local directory. +- The contents inside the `working/` folder will be synchronized with this directory, so ensure it reflects the latest state. + +### 2. Check Dependencies +- All necessary dependencies are already included in GarmentIQ MagicBox. +- You don't need to manually install them, making it easy to get started right away. +- In case if you need additional dipendencies, modify the `requirements.txt` in the configuration files in your local machine, then rebuild the Docker image and container, or simply using `!pip install`. + +### 3. Create a New Notebook or Script +- You can create a new Jupyter notebook or a Python script to write your code. +- Ensure your notebook or script is placed inside the `working/` folder to enable synchronization. + +### 4. Explore Example Notebooks +- If you're new to GarmentIQ MagicBox or want to quickly see how things work, check out the example notebooks under the `working/examples/` folder. +- These examples will help you understand how to use the system. + +### 5. Run Your Code +- Once your notebook or script is ready, simply run it. +- Any outputs or changes made within the `working/` directory will be reflected in your local directory. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8b317a0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +version: '3.8' + +services: + garmentiq: + build: . + ports: + - "8888:8888" + volumes: + - ./working:/app/working + environment: + - PYTHONPATH=/app:/app/working \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b7da4d3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +flask==3.0.2 +flask-cors==4.0.0 +gunicorn==21.2.0 +garmentiq diff --git a/working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb b/working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb new file mode 100644 index 0000000..6abe03e --- /dev/null +++ b/working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb @@ -0,0 +1,178 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "uTYHtjS_NzhK" + }, + "source": [ + "# GarmentIQ Classification Model Fine-tuning - Advanced Usage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "L7eOzqrGPvLp" + }, + "outputs": [], + "source": [ + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import tinyViT\n", + "from garmentiq.classification.utils import CachedDataset\n", + "import torch.optim as optim" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QIb-L3XU9KJn" + }, + "outputs": [], + "source": [ + "# Download fine-tuning data\n", + "# To train a model using GarmentIQ framework, your data must be in a zip file\n", + "# and the zip file should have the same structure as our data. See the link:\n", + "# https://www.kaggle.com/datasets/lygitdata/zara-clothes-image-data\n", + "!curl -L -o /app/working/examples/classification_model_fine_tuning_advanced_usage/zara-clothes-image-data.zip\\\n", + " https://www.kaggle.com/api/v1/datasets/download/lygitdata/zara-clothes-image-data\n", + "\n", + "# Download the base model - tinyViT - to be finetuned\n", + "!mkdir -p /app/working/examples/classification_model_fine_tuning_advanced_usage/models\n", + "!wget -q -O /app/working/examples/classification_model_fine_tuning_advanced_usage/models/tiny_vit.pt \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit.pt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "col7KU5vOFt6" + }, + "outputs": [], + "source": [ + "# Prepare the data for fine-tuning\n", + "# As our data size is small, we make the testing set to be 0%\n", + "# You can see from the test set summary that the size is 0\n", + "data = giq.classification.train_test_split(\n", + " output_dir=\"/app/working/examples/classification_model_fine_tuning_advanced_usage/data\",\n", + " train_zip_dir=\"/app/working/examples/classification_model_fine_tuning_advanced_usage/zara-clothes-image-data.zip\",\n", + " metadata_csv=\"metadata.csv\",\n", + " label_column=\"garment\",\n", + " test_size=0,\n", + " verbose=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CcS54pI1QrJx" + }, + "outputs": [], + "source": [ + "# Load the training set into memory for faster I/O during training\n", + "train_images, train_labels, _ = giq.classification.load_data(\n", + " df=data[\"train_metadata\"],\n", + " img_dir=data[\"train_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(120, 184),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zTfBwce_IH1r" + }, + "outputs": [], + "source": [ + "# Fine-tune the pretrained tinyViT model\n", + "# For demonstration purpose, we only use 5 folds and 5 epochs\n", + "# Models are saved at the folder `finetuned_models`\n", + "# It automatically selects the model with the lowest cross entropy\n", + "# as the best model\n", + "giq.classification.fine_tune_pytorch_nn(\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"metadata_df\": data[\"train_metadata\"],\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\n", + " \"pretrained_path\": \"/app/working/examples/classification_model_fine_tuning_advanced_usage/models/tiny_vit.pt\",\n", + " \"freeze_layers\": True,\n", + " \"unfreeze_patterns\": [\"classifier\", \"fc\"],\n", + " \"optimizer_class\": optim.AdamW,\n", + " \"optimizer_args\": {\"lr\": 0.00002, \"weight_decay\": 1e-4},\n", + " \"n_fold\": 5,\n", + " \"n_epoch\": 5,\n", + " \"patience\": 2,\n", + " \"batch_size\": 128,\n", + " \"model_save_dir\": \"/app/working/examples/classification_model_fine_tuning_advanced_usage/finetuned_models\",\n", + " \"best_model_name\": \"best_finetuned.pt\"\n", + " },\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E9y2RIbsJenJ" + }, + "outputs": [], + "source": [ + "# See the performance of the finetuned model on the fine-tuning dataset\n", + "giq.classification.test_pytorch_nn(\n", + " model_path=\"/app/working/examples/classification_model_fine_tuning_advanced_usage/finetuned_models/best_finetuned.pt\",\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\"batch_size\": 64},\n", + ")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "authorship_tag": "ABX9TyPBgI9OQorOESyY4wReaUl+", + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb b/working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb new file mode 100644 index 0000000..e338767 --- /dev/null +++ b/working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb @@ -0,0 +1,278 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "uTYHtjS_NzhK" + }, + "source": [ + "# GarmentIQ Classification Model Training & Evaluation - Advanced Usage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "L7eOzqrGPvLp" + }, + "outputs": [], + "source": [ + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import CNN3\n", + "from garmentiq.classification.utils import CachedDataset\n", + "import torch.optim as optim\n", + "import torch.nn as nn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QIb-L3XU9KJn" + }, + "outputs": [], + "source": [ + "# Download training data\n", + "# To train a model using GarmentIQ framework, your data must be in a zip file\n", + "# and the zip file should have the same structure as our data. See the link:\n", + "# https://www.kaggle.com/datasets/lygitdata/garmentiq-classification-set-nordstrom-and-myntra\n", + "!curl -L -o /app/working/examples/classification_model_training_evaluation_advanced_usage/garmentiq-classification-set-nordstrom-and-myntra.zip \\\n", + " https://www.kaggle.com/api/v1/datasets/download/lygitdata/garmentiq-classification-set-nordstrom-and-myntra" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "col7KU5vOFt6" + }, + "outputs": [], + "source": [ + "# Split the data into training set and testing set\n", + "data = giq.classification.train_test_split(\n", + " output_dir=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/data\",\n", + " train_zip_dir=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/garmentiq-classification-set-nordstrom-and-myntra.zip\",\n", + " metadata_csv=\"metadata.csv\",\n", + " label_column=\"garment\",\n", + " test_size=0.15,\n", + " verbose=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CcS54pI1QrJx" + }, + "outputs": [], + "source": [ + "# Load the training set into memory for faster I/O during training\n", + "train_images, train_labels, _ = giq.classification.load_data(\n", + " df=data[\"train_metadata\"],\n", + " img_dir=data[\"train_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(60, 92),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZWzfw235-nG8" + }, + "outputs": [], + "source": [ + "# Train with GarmentIQ predefined model - CNN3\n", + "# Note that we use `model_class=CNN3` here\n", + "# For demonstration purpose, we only use 2 folds and 5 epochs\n", + "# Models are saved at the folder `cnn3_models`\n", + "# It automatically selects the model with the lowest cross entropy\n", + "# as the best model\n", + "giq.classification.train_pytorch_nn(\n", + " model_class=CNN3,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"metadata_df\": data[\"train_metadata\"],\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\n", + " \"optimizer_class\": optim.AdamW,\n", + " \"optimizer_args\": {\"lr\": 0.001, \"weight_decay\": 1e-4},\n", + " \"n_fold\": 2,\n", + " \"n_epoch\": 5,\n", + " \"patience\": 2,\n", + " \"batch_size\": 256,\n", + " \"model_save_dir\": \"/app/working/examples/classification_model_training_evaluation_advanced_usage/cnn3_models\",\n", + " \"best_model_name\": \"best_cnn3_model.pt\",\n", + " },\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jxoTUMM5ADAR" + }, + "outputs": [], + "source": [ + "# Train with a user-defined model\n", + "class UserDefinedCNN(nn.Module):\n", + " def __init__(self, num_classes):\n", + " super(UserDefinedCNN, self).__init__()\n", + " self.features = nn.Sequential(\n", + " nn.Conv2d(3, 32, kernel_size=3, padding=1),\n", + " nn.BatchNorm2d(32),\n", + " nn.ReLU(inplace=True),\n", + " nn.MaxPool2d(2, 2),\n", + " nn.Dropout(0.2),\n", + " nn.Conv2d(32, 64, kernel_size=3, padding=1),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(inplace=True),\n", + " nn.MaxPool2d(2, 2),\n", + " nn.Dropout(0.25),\n", + " nn.AdaptiveAvgPool2d((4, 4)),\n", + " )\n", + " self.classifier = nn.Sequential(\n", + " nn.Linear(64 * 4 * 4, 128),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(0.3),\n", + " nn.Linear(128, num_classes),\n", + " )\n", + " def forward(self, x):\n", + " x = self.features(x)\n", + " x = x.view(x.size(0), -1)\n", + " x = self.classifier(x)\n", + " return x\n", + "\n", + "# For demonstration purpose, we only use 2 folds and 5 epochs\n", + "# Models are saved at the folder `cnn1_models`\n", + "# It automatically selects the model with the lowest cross entropy\n", + "# as the best model\n", + "giq.classification.train_pytorch_nn(\n", + " model_class=UserDefinedCNN,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"metadata_df\": data[\"train_metadata\"],\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\n", + " \"optimizer_class\": optim.AdamW,\n", + " \"optimizer_args\": {\"lr\": 0.001, \"weight_decay\": 1e-4},\n", + " \"n_fold\": 2,\n", + " \"n_epoch\": 5,\n", + " \"patience\": 2,\n", + " \"batch_size\": 256,\n", + " \"model_save_dir\": \"/app/working/examples/classification_model_training_evaluation_advanced_usage/userdefined_cnn_models\",\n", + " \"best_model_name\": \"best_userdefined_cnn_model.pt\",\n", + " },\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4SJBuwwj_csd" + }, + "outputs": [], + "source": [ + "# Load the testing set for model evaluation\n", + "test_images, test_labels, _ = giq.classification.load_data(\n", + " df=data[\"test_metadata\"],\n", + " img_dir=data[\"test_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(60, 92),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MsJVxrcgCkjx" + }, + "outputs": [], + "source": [ + "# Evaluate CNN3 model on the testing set\n", + "giq.classification.test_pytorch_nn(\n", + " model_path=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/cnn3_models/best_cnn3_model.pt\",\n", + " model_class=CNN3,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": data[\"test_metadata\"][\"garment\"],\n", + " \"cached_images\": test_images,\n", + " \"cached_labels\": test_labels,\n", + " },\n", + " param={\"batch_size\": 64},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_zeZcebZCsCp" + }, + "outputs": [], + "source": [ + "# Evaluate user-defined model on the testing set\n", + "# We can see that the CNN3 model performs better on the test set\n", + "# than this user-defined CNN model\n", + "giq.classification.test_pytorch_nn(\n", + " model_path=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/userdefined_cnn_models/best_userdefined_cnn_model.pt\",\n", + " model_class=UserDefinedCNN,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": data[\"test_metadata\"][\"garment\"],\n", + " \"cached_images\": test_images,\n", + " \"cached_labels\": test_labels,\n", + " },\n", + " param={\"batch_size\": 64},\n", + ")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "authorship_tag": "ABX9TyM86uuEAMztFXJELwGY11Hc", + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/classification_quick_start/classification_quick_start.ipynb b/working/examples/classification_quick_start/classification_quick_start.ipynb new file mode 100644 index 0000000..cb72f80 --- /dev/null +++ b/working/examples/classification_quick_start/classification_quick_start.ipynb @@ -0,0 +1,163 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "jrn-fD7chMUe" + }, + "source": [ + "# GarmentIQ Classification Quick Start" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E8RxkKlsh5IY" + }, + "outputs": [], + "source": [ + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import tinyViT\n", + "from garmentiq.classification.utils import CachedDataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3-b9AZr9hJyK" + }, + "outputs": [], + "source": [ + "# Download test data and a pretrained model\n", + "!mkdir -p /app/working/examples/classification_quick_start/models\n", + "\n", + "!curl -L -o /app/working/examples/classification_quick_start/garmentiq-classification-set-nordstrom-and-myntra.zip \\\n", + " https://www.kaggle.com/api/v1/datasets/download/lygitdata/garmentiq-classification-set-nordstrom-and-myntra\n", + "\n", + "!wget -q -O /app/working/examples/classification_quick_start/models/tiny_vit.pt \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit.pt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aW3E9wDbivLQ" + }, + "outputs": [], + "source": [ + "# Prepare test data garmentiq-classification-set-nordstrom-and-myntra.zip\n", + "DATA = giq.classification.train_test_split(\n", + " output_dir=\"/app/working/examples/classification_quick_start/data\",\n", + " metadata_csv=\"metadata.csv\",\n", + " label_column=\"garment\",\n", + " train_zip_dir=\"/app/working/examples/classification_quick_start/garmentiq-classification-set-nordstrom-and-myntra.zip\",\n", + " test_size=0.15,\n", + " verbose=True\n", + ")\n", + "\n", + "test_images, test_labels, _ = giq.classification.load_data(\n", + " df=DATA[\"test_metadata\"],\n", + " img_dir=DATA[\"test_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(120, 184),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the pretrained model\n", + "classifier = giq.classification.load_model(\n", + " model_path=\"/app/working/examples/classification_quick_start/models/tiny_vit.pt\",\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6}\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hfXjYs-mkE4s" + }, + "outputs": [], + "source": [ + "# Fit the model on the whole test data\n", + "giq.classification.test_pytorch_nn(\n", + " model_path=\"/app/working/examples/classification_quick_start/models/tiny_vit.pt\",\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": DATA[\"test_metadata\"][\"garment\"],\n", + " \"cached_images\": test_images,\n", + " \"cached_labels\": test_labels,\n", + " },\n", + " param={\"batch_size\": 64},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vJ6kl9rTkerA" + }, + "outputs": [], + "source": [ + "# Fit the model on a single image\n", + "img_to_test = DATA['test_metadata']['filename'][88]\n", + "\n", + "pred_label, pred_prob = giq.classification.predict(\n", + " model=classifier,\n", + " image_path=f\"/app/working/examples/classification_quick_start/data/test/images/{img_to_test}\",\n", + " classes=DATA['test_metadata']['garment'].unique().tolist(),\n", + " resize_dim=(120, 184),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081]\n", + ")\n", + "\n", + "print(\n", + " \"True label: \", img_to_test,\n", + " \"\\nPredicted label: \", pred_label,\n", + " \"\\nPredicted Probabilities: \", pred_prob\n", + ")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb b/working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb new file mode 100644 index 0000000..7b43cee --- /dev/null +++ b/working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb @@ -0,0 +1,278 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "uTYHtjS_NzhK" + }, + "source": [ + "# GarmentIQ Custom Measurement Instruction - Advanced Usage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "L7eOzqrGPvLp" + }, + "outputs": [], + "source": [ + "import json\n", + "import copy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "col7KU5vOFt6" + }, + "outputs": [], + "source": [ + "# See our predefined garments and measurement instructions\n", + "#\n", + "# - `num_predefeined_points` and `index_range` are determined by training data -\n", + "# DeepFashion2, change them if it is indeed necessary.\n", + "#\n", + "# - `instruction` is the path to a .json file, GarmentQI has predefined\n", + "# instructions.\n", + "#\n", + "# Generate or see predefined instructions at:\n", + "# https://garmentiq.ly.gd.edu.kg/application/demo/instruction-generation/\n", + "from garmentiq.garment_classes import garment_classes\n", + "print(json.dumps(garment_classes, indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CcS54pI1QrJx" + }, + "outputs": [], + "source": [ + "# Let us change the measurement instruction of skirt\n", + "# See the predefined instruction\n", + "# There are `waist`, `full length`, and `hips` as measurements in GarmentIQ's\n", + "# default measurement instruction of a skirt. What if we only want `waist` and\n", + "# `hips`? Simply create one! Recommend to use GarmentIQ instruction generation\n", + "# tool to generate the instruciton:\n", + "# https://garmentiq.ly.gd.edu.kg/application/demo/instruction-generation/\n", + "with open(garment_classes['skirt']['instruction'], 'r') as file:\n", + " data = json.load(file)\n", + "print(json.dumps(data, indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gB1AxdinRqvR" + }, + "outputs": [], + "source": [ + "# Let us create a simple instruction in python here and convert it to json\n", + "skirt_new = {\n", + " \"skirt\": {\n", + " \"landmarks\": {\n", + " \"1\": {\n", + " \"predefined\": True,\n", + " \"description\": \"waist_left\",\n", + " \"x\": 60,\n", + " \"y\": 40\n", + " },\n", + " \"3\": {\n", + " \"predefined\": True,\n", + " \"description\": \"waist_right\",\n", + " \"x\": 140,\n", + " \"y\": 40\n", + " },\n", + " \"4\": {\n", + " \"predefined\": True,\n", + " \"description\": \"side_seam_left\",\n", + " \"x\": 50,\n", + " \"y\": 80\n", + " },\n", + " \"8\": {\n", + " \"predefined\": True,\n", + " \"description\": \"side_seam_right\",\n", + " \"x\": 150,\n", + " \"y\": 80\n", + " }\n", + " },\n", + " \"measurements\": {\n", + " \"waist\": {\n", + " \"landmarks\": {\n", + " \"start\": \"1\",\n", + " \"end\": \"3\"\n", + " },\n", + " \"description\": \"/\"\n", + " },\n", + " \"hips\": {\n", + " \"landmarks\": {\n", + " \"start\": \"4\",\n", + " \"end\": \"8\"\n", + " },\n", + " \"description\": \"/\"\n", + " }\n", + " }\n", + " }\n", + "}\n", + "\n", + "# Save it to a JSON file with indentation\n", + "with open('skirt_new.json', 'w') as file:\n", + " json.dump(skirt_new, file, indent=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lb1wL06USdzn" + }, + "outputs": [], + "source": [ + "# Create a new garment class metadata dictionary\n", + "new_garment_classes = copy.deepcopy(garment_classes)\n", + "new_garment_classes['skirt']['instruction'] = 'skirt_new.json'\n", + "print(new_garment_classes['skirt'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XiAh-4I6TNXl" + }, + "outputs": [], + "source": [ + "# Okay now we have the new instruction, let us try the detection model to see\n", + "# the difference\n", + "# Import necessary modules\n", + "import garmentiq as giq\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "\n", + "# Download a skirt image and a pretrained model\n", + "!mkdir -p /app/working/examples/custom_measurement_instruction_advanced_usage/test_image\n", + "!wget -q -O /app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_4.jpg\n", + "\n", + "!mkdir -p /app/working/examples/custom_measurement_instruction_advanced_usage/models\n", + "!wget -q -O /app/working/examples/custom_measurement_instruction_advanced_usage/models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aN_ikAdETn5J" + }, + "outputs": [], + "source": [ + "# Plot the image\n", + "giq.landmark.plot(image_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg\", figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sMKutxcVTrLw" + }, + "outputs": [], + "source": [ + "# Load the pretrained model from Hugging Face\n", + "HRNet = giq.landmark.detection.load_model(\n", + " model_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/models/hrnet.pth\",\n", + " model_class=PoseHighResolutionNet()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Frx1SAxVTu1J" + }, + "outputs": [], + "source": [ + "# Detect landmarks (DEFAULT instruction)\n", + "_, _, detection_dict = giq.landmark.detect(\n", + " class_name=\"skirt\",\n", + " class_dict=garment_classes,\n", + " image_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225]\n", + ")\n", + "\n", + "# Clean the detection dictionary\n", + "# See that we have all the three measurements\n", + "detection_dict_cleaned = giq.utils.clean_detection_dict(\n", + " class_name='skirt',\n", + " image_name='cloth_4.jpg',\n", + " detection_dict=detection_dict\n", + ")\n", + "detection_dict_cleaned" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "R34QuDmYT_X0" + }, + "outputs": [], + "source": [ + "# Detect landmarks (NEW instruction)\n", + "# Note that we use `class_dict=new_garment_classes` here\n", + "_, _, detection_dict_new = giq.landmark.detect(\n", + " class_name=\"skirt\",\n", + " class_dict=new_garment_classes,\n", + " image_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225]\n", + ")\n", + "\n", + "# Clean the detection dictionary\n", + "# See that we only have two measurements we want\n", + "detection_dict_new_cleaned = giq.utils.clean_detection_dict(\n", + " class_name='skirt',\n", + " image_name='cloth_4.jpg',\n", + " detection_dict=detection_dict_new\n", + ")\n", + "detection_dict_new_cleaned" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb b/working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb new file mode 100644 index 0000000..14b3419 --- /dev/null +++ b/working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb @@ -0,0 +1,141 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "jrn-fD7chMUe" + }, + "source": [ + "# GarmentIQ Landmark Detection Quick Start" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E8RxkKlsh5IY" + }, + "outputs": [], + "source": [ + "import garmentiq as giq\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3-b9AZr9hJyK" + }, + "outputs": [], + "source": [ + "# Download a vest dress image and a pretrained model\n", + "!mkdir -p /app/working/examples/landmark_detection_quick_start/test_image\n", + "!wget -q -O /app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", + "\n", + "!mkdir -p /app/working/examples/landmark_detection_quick_start/models\n", + "!wget -q -O /app/working/examples/landmark_detection_quick_start/models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 337 + }, + "id": "NYFrZUe48hLi", + "outputId": "6d881cf8-75c3-46f1-acf5-c0b691b934c6" + }, + "outputs": [], + "source": [ + "# Plot the image\n", + "giq.landmark.plot(image_path=\"/app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg\", figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pMGZwL-UjPPN" + }, + "outputs": [], + "source": [ + "# Load the pretrained model from Hugging Face\n", + "HRNet = giq.landmark.detection.load_model(\n", + " model_path=\"/app/working/examples/landmark_detection_quick_start/models/hrnet.pth\",\n", + " model_class=PoseHighResolutionNet()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "deJChHYZ7rpT" + }, + "outputs": [], + "source": [ + "# Detect predefined landmarks\n", + "coords, maxvals, detection_dict = giq.landmark.detect(\n", + " class_name=\"vest dress\",\n", + " class_dict=garment_classes,\n", + " image_path=\"/app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 337 + }, + "id": "rpKL-_2685Rd", + "outputId": "634798cc-ab48-4d4f-d7bf-2459890d9efb" + }, + "outputs": [], + "source": [ + "# Plot the detected coordinates\n", + "giq.landmark.plot(image_path=\"/app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg\", coordinate=coords, figsize=(3, 3), color=\"green\")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb b/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb new file mode 100644 index 0000000..3813cc8 --- /dev/null +++ b/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb @@ -0,0 +1,208 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "jrn-fD7chMUe" + }, + "source": [ + "# GarmentIQ Landmark Detection Quick Start" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E8RxkKlsh5IY" + }, + "outputs": [], + "source": [ + "import garmentiq as giq\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3-b9AZr9hJyK" + }, + "outputs": [], + "source": [ + "# Download a vest dress image and a pretrained model\n", + "!mkdir -p test_image\n", + "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", + "\n", + "!mkdir -p models\n", + "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NYFrZUe48hLi" + }, + "outputs": [], + "source": [ + "# Plot the image\n", + "giq.landmark.plot(\n", + " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\", \n", + " figsize=(3, 3)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pMGZwL-UjPPN" + }, + "outputs": [], + "source": [ + "# Load the pretrained model from Hugging Face\n", + "HRNet = giq.landmark.detection.load_model(\n", + " model_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/models/hrnet.pth\",\n", + " model_class=PoseHighResolutionNet()\n", + ")\n", + "\n", + "# Detect predefined landmarks\n", + "coords, maxvals, detection_dict = giq.landmark.detect(\n", + " class_name=\"vest dress\",\n", + " class_dict=garment_classes,\n", + " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225]\n", + ")\n", + "\n", + "# Plot the detected coordinates\n", + "giq.landmark.plot(\n", + " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\", \n", + " coordinate=coords, \n", + " figsize=(3, 3), \n", + " color=\"green\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "y4-zd5CusfED" + }, + "outputs": [], + "source": [ + "# Segmentation mask is required for refinement and derivation\n", + "# So we need to do segmentation first\n", + "BiRefNet = giq.segmentation.load_model(\n", + " pretrained_model='lygitdata/BiRefNet_garmentiq_backup',\n", + " pretrained_model_args={'trust_remote_code': True},\n", + " high_precision=True\n", + ")\n", + "original_img, mask = giq.segmentation.extract(\n", + " model=BiRefNet,\n", + " image_path='/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg',\n", + " resize_dim=(1024, 1024),\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " high_precision=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "deJChHYZ7rpT" + }, + "outputs": [], + "source": [ + "# Refine the landmarks\n", + "refined_coords, refined_detection_dict = giq.landmark.refine(\n", + " class_name=\"vest dress\",\n", + " detection_np=coords,\n", + " detection_conf=maxvals,\n", + " detection_dict=detection_dict,\n", + " mask=mask,\n", + " window_size=5,\n", + " ksize=(11, 11),\n", + " sigmaX=0.0\n", + ")\n", + "\n", + "# Print the original coordinates and the refined coordinates\n", + "print(\"Original coordinates:\\n\", coords)\n", + "print(\"Refined coordinates:\\n\", refined_coords)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rpKL-_2685Rd" + }, + "outputs": [], + "source": [ + "# Derive custom landmarks\n", + "derived_coords, derived_detection_dict = giq.landmark.derive(\n", + " class_name=\"vest dress\",\n", + " detection_dict=refined_detection_dict,\n", + " derivation_dict=giq.landmark.derivation_dict.derivation_dict,\n", + " landmark_coords=refined_coords,\n", + " np_mask=mask\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LTskt_1KwmBG" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "# Plot the derived point\n", + "giq.landmark.plot(\n", + " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\",\n", + " coordinate=np.concatenate((refined_coords, np.array([[derived_coords['20']]])), axis=1),\n", + " figsize=(3, 3),\n", + " color=\"green\"\n", + ")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb b/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb new file mode 100644 index 0000000..f3ce798 --- /dev/null +++ b/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb @@ -0,0 +1,121 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "jrn-fD7chMUe" + }, + "source": [ + "# GarmentIQ Segmentation Quick Start" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E8RxkKlsh5IY" + }, + "outputs": [], + "source": [ + "import garmentiq as giq" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3-b9AZr9hJyK" + }, + "outputs": [], + "source": [ + "# Download a test image\n", + "!mkdir -p /app/working/examples/segmentation_quick_start/test_image\n", + "!wget -q -O /app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pMGZwL-UjPPN" + }, + "outputs": [], + "source": [ + "# Load the pretrained model from Hugging Face\n", + "BiRefNet = giq.segmentation.load_model(\n", + " pretrained_model='lygitdata/BiRefNet_garmentiq_backup',\n", + " pretrained_model_args={'trust_remote_code': True},\n", + " high_precision=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hfXjYs-mkE4s" + }, + "outputs": [], + "source": [ + "# Extract the mask\n", + "original_img, mask = giq.segmentation.extract(\n", + " model=BiRefNet,\n", + " image_path='/app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg',\n", + " resize_dim=(1024, 1024),\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " high_precision=True\n", + ")\n", + "\n", + "# Change background color\n", + "bg_modified_img = giq.segmentation.change_background_color(\n", + " image_np=original_img,\n", + " mask_np=mask,\n", + " background_color=[102, 255, 102]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dPkVas24rw2H" + }, + "outputs": [], + "source": [ + "# Plot the original image, mask, and background modified image\n", + "giq.segmentation.plot(image_np=original_img, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=mask, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=bg_modified_img, figsize=(3, 3))" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/tailor_quick_start/tailor_quick_start.ipynb b/working/examples/tailor_quick_start/tailor_quick_start.ipynb new file mode 100644 index 0000000..5e0e2dc --- /dev/null +++ b/working/examples/tailor_quick_start/tailor_quick_start.ipynb @@ -0,0 +1,223 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "jrn-fD7chMUe" + }, + "source": [ + "# GarmentIQ Tailor Quick Start" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E8RxkKlsh5IY" + }, + "outputs": [], + "source": [ + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import tinyViT\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes\n", + "from garmentiq.landmark.derivation.derivation_dict import derivation_dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3-b9AZr9hJyK" + }, + "outputs": [], + "source": [ + "# Download 4 test images\n", + "# cloth_1 and cloth_2 are short sleeve tops, cloth_3 is vest dress, cloth_4 is skirt\n", + "!mkdir -p /app/working/examples/tailor_quick_start/test_image\n", + "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_1.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg\n", + "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_2.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_2.jpg\n", + "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_3.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", + "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_4.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_4.jpg\n", + "\n", + "# Download the classification model\n", + "!mkdir -p /app/working/examples/tailor_quick_start/models\n", + "!wget -q -O /app/working/examples/tailor_quick_start/models/tiny_vit_inditex_finetuned.pt \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit_inditex_finetuned.pt\n", + "\n", + "# Download the landmark detection model\n", + "!wget -q -O /app/working/examples/tailor_quick_start/models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NYFrZUe48hLi" + }, + "outputs": [], + "source": [ + "# Setup the tailor agent\n", + "tailor = giq.tailor(\n", + " input_dir=\"/app/working/examples/tailor_quick_start/test_image\",\n", + " model_dir=\"/app/working/examples/tailor_quick_start/models\",\n", + " output_dir=\"/app/working/examples/tailor_quick_start/output\",\n", + " class_dict=garment_classes,\n", + " do_derive=True,\n", + " derivation_dict=derivation_dict,\n", + " do_refine=True,\n", + " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", + " classification_model_class=tinyViT,\n", + " classification_model_args={\n", + " \"num_classes\": len(list(garment_classes.keys())),\n", + " \"img_size\": (120, 184),\n", + " \"patch_size\": 6,\n", + " \"resize_dim\": (120, 184),\n", + " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", + " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", + " },\n", + " segmentation_model_name=\"lygitdata/BiRefNet_garmentiq_backup\",\n", + " segmentation_model_args={\n", + " \"trust_remote_code\": True,\n", + " \"resize_dim\": (1024, 1024),\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + " \"high_precision\": True,\n", + " \"background_color\": [102, 255, 102],\n", + " },\n", + " landmark_detection_model_path=\"hrnet.pth\",\n", + " landmark_detection_model_class=PoseHighResolutionNet(),\n", + " landmark_detection_model_args={\n", + " \"scale_std\": 200.0,\n", + " \"resize_dim\": [288, 384],\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + " },\n", + ")\n", + "\n", + "# See the tailor agent's basic information\n", + "tailor.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "F3_WQnWmCLY2" + }, + "outputs": [], + "source": [ + "# Start the measurement with refinement and derivation\n", + "metadata, outputs = tailor.measure(save_segmentation_image=True, save_measurement_image=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-spavypXE5xb" + }, + "outputs": [], + "source": [ + "# See the metadata\n", + "# It makes file access much easier\n", + "print(metadata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XoK9vs-tERj-", + "scrolled": true + }, + "outputs": [], + "source": [ + "# Plot the masks\n", + "# Go to /app/working/examples/tailor_quick_start/output/mask_image/ to see the high resolution images\n", + "for image in metadata['mask_image']:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LartOXSkEruQ" + }, + "outputs": [], + "source": [ + "# Plot the background modified images\n", + "# Go to /app/working/examples/tailor_quick_start/output/bg_modified_image to see the high resolution images\n", + "for image in metadata['bg_modified_image']:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HF7z8tZtDAz8" + }, + "outputs": [], + "source": [ + "# Plot the images with desired landmarks\n", + "# Go to /app/working/examples/tailor_quick_start/output/measurement_image/ to see the high resolution images\n", + "for image in metadata['measurement_image']:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jbLD40U5ELe9" + }, + "outputs": [], + "source": [ + "# See the measurement results in JSON format\n", + "# Go to /app/working/examples/tailor_quick_start/output/measurement_json/ to see the JSON files\n", + "import json\n", + "\n", + "for json_path in metadata['measurement_json']:\n", + " with open(json_path, 'r') as file:\n", + " data = json.load(file)\n", + " print(f\"{json_path}:\\n\")\n", + " print(json.dumps(data, indent=4, sort_keys=True))\n", + " print(\"\\n\\n\")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/working/examples/web_landmark_detection/main.ipynb b/working/examples/web_landmark_detection/main.ipynb new file mode 100644 index 0000000..0540c08 --- /dev/null +++ b/working/examples/web_landmark_detection/main.ipynb @@ -0,0 +1,163 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "835e74a1-b705-4742-9199-f45e48336cf6", + "metadata": {}, + "outputs": [], + "source": [ + "import garmentiq as giq\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes\n", + "from flask_cors import CORS\n", + "from flask import Flask, request, jsonify\n", + "from PIL import Image\n", + "import numpy as np\n", + "import io\n", + "import cv2\n", + "import base64" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd59761d-ea54-4486-a960-95d1fb020350", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "mkdir -p /app/working/examples/web_landmark_detection/models\n", + "if [ ! -f /app/working/examples/web_landmark_detection/models/hrnet.pth ]; then\n", + " wget -q -O /app/working/examples/web_landmark_detection/models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", + "else\n", + " echo \"Model file already exists, skipping download.\"\n", + "fi" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6cec29b-d894-4207-ab07-fe8fcc0a659c", + "metadata": {}, + "outputs": [], + "source": [ + "app = Flask(__name__)\n", + "CORS(app)\n", + "\n", + "# load model once\n", + "print(\"Loading HRNet model…\")\n", + "HRNet = giq.landmark.detection.load_model(\n", + " model_path=\"/app/working/examples/web_landmark_detection/models/hrnet.pth\",\n", + " model_class=PoseHighResolutionNet()\n", + ")\n", + "print(\"Model loaded.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9392663-bd9b-4071-a500-ad4ac046d54f", + "metadata": {}, + "outputs": [], + "source": [ + "@app.route('/health')\n", + "def health_check():\n", + " return jsonify({\n", + " \"status\": \"active\",\n", + " \"token\": \"a3f7d2c14e65bb2e8f01a9dc4f6c9823d279f1e05b3a6d74c0987b1c2fae3c65\",\n", + " \"model\": \"web_landmark_detection\",\n", + " \"version\": \"1.0.0\"\n", + " })\n", + "\n", + "@app.route('/landmark_detection', methods=['POST'])\n", + "def landmark_detection():\n", + " files = request.files.getlist('images')\n", + " garment_class = request.form.get('garment_class')\n", + "\n", + " images_np = []\n", + " images_coords = []\n", + "\n", + " # Convert uploaded images to numpy arrays\n", + " for file in files:\n", + " image = Image.open(file.stream).convert('RGB')\n", + " image_np = np.array(image)\n", + " images_np.append(image_np)\n", + "\n", + " # Run detection and collect coordinates\n", + " for img in images_np:\n", + " coords, _, _ = giq.landmark.detect(\n", + " class_name=garment_class,\n", + " class_dict=garment_classes,\n", + " image_path=img,\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225]\n", + " )\n", + " images_coords.append(coords)\n", + "\n", + " # Annotate images and encode to base64\n", + " base64_images = []\n", + " for img, coords in zip(images_np, images_coords):\n", + " # Convert RGB numpy array to BGR for OpenCV drawing\n", + " img_annotated = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n", + " height, width = img_annotated.shape[:2]\n", + " \n", + " # Circle radius relative to image size\n", + " radius = max(2, int(round(min(height, width) * 0.0125))) # at least radius 2\n", + " \n", + " for point in coords[0]: # coords is (1, N, 2)\n", + " x, y = int(round(point[0])), int(round(point[1]))\n", + " cv2.circle(img_annotated, (x, y), radius, (0, 255, 0), -1) # green filled circle in BGR\n", + " \n", + " # Convert back to RGB for PIL\n", + " img_annotated_rgb = cv2.cvtColor(img_annotated, cv2.COLOR_BGR2RGB)\n", + " pil_img = Image.fromarray(img_annotated_rgb)\n", + " \n", + " buffered = io.BytesIO()\n", + " pil_img.save(buffered, format=\"PNG\")\n", + " img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')\n", + " base64_images.append(img_str)\n", + "\n", + " return jsonify({\n", + " \"results\": base64_images\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "460f8ce6-82dd-4393-a588-bfbf7466cbe3", + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == '__main__':\n", + " app.run(host='0.0.0.0', port=5001)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/working/examples/web_segmentation/main.ipynb b/working/examples/web_segmentation/main.ipynb new file mode 100644 index 0000000..1a04a95 --- /dev/null +++ b/working/examples/web_segmentation/main.ipynb @@ -0,0 +1,167 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "835e74a1-b705-4742-9199-f45e48336cf6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import base64\n", + "from flask import Flask, request, jsonify\n", + "from flask_cors import CORS\n", + "import garmentiq as giq\n", + "import shutil" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6cec29b-d894-4207-ab07-fe8fcc0a659c", + "metadata": {}, + "outputs": [], + "source": [ + "app = Flask(__name__)\n", + "CORS(app)\n", + "\n", + "# where we put uploads & outputs\n", + "BASE_DIR = 'images'\n", + "INPUT_DIR = os.path.join(BASE_DIR, 'input')\n", + "OUTPUT_DIR = os.path.join(BASE_DIR, 'output')\n", + "os.makedirs(INPUT_DIR, exist_ok=True)\n", + "os.makedirs(OUTPUT_DIR, exist_ok=True)\n", + "\n", + "# load model once\n", + "print(\"Loading BiRefNet model…\")\n", + "BiRefNet = giq.segmentation.load_model(\n", + " pretrained_model='lygitdata/BiRefNet_garmentiq_backup',\n", + " pretrained_model_args={'trust_remote_code': True},\n", + " high_precision=True\n", + ")\n", + "print(\"Model loaded.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9392663-bd9b-4071-a500-ad4ac046d54f", + "metadata": {}, + "outputs": [], + "source": [ + "@app.route('/health')\n", + "def health_check():\n", + " return jsonify({\n", + " \"status\": \"active\",\n", + " \"token\": \"fe9def684914b305540615b3bba461fbf8ea58f460c72b1fe128d3cef93fe4f8\",\n", + " \"model\": \"web_segmentation\",\n", + " \"version\": \"1.0.0\"\n", + " })\n", + "\n", + "@app.route('/segment', methods=['POST'])\n", + "def segment():\n", + " files = request.files.getlist('images')\n", + " if not files:\n", + " return jsonify(error='No images uploaded'), 400\n", + "\n", + " # Ensure input/output directories are fresh\n", + " for d in (INPUT_DIR, OUTPUT_DIR):\n", + " if os.path.exists(d):\n", + " shutil.rmtree(d)\n", + " os.makedirs(d, exist_ok=True)\n", + "\n", + " # Save uploaded images\n", + " for f in files:\n", + " f.save(os.path.join(INPUT_DIR, f.filename))\n", + "\n", + " # Parse optional RGB values\n", + " r = request.form.get('red', '')\n", + " g = request.form.get('green', '')\n", + " b = request.form.get('blue', '')\n", + " background_color = None\n", + " if r != '' and g != '' and b != '':\n", + " try:\n", + " rgb = [int(r), int(g), int(b)]\n", + " if any(c < 0 or c > 255 for c in rgb):\n", + " raise ValueError\n", + " background_color = rgb\n", + " except ValueError:\n", + " return jsonify(error='Invalid RGB values; must be integers 0–255'), 400\n", + "\n", + " # Prepare processing arguments\n", + " kwargs = {\n", + " 'image_dir': INPUT_DIR,\n", + " 'output_dir': OUTPUT_DIR,\n", + " 'model': BiRefNet,\n", + " 'resize_dim': (1024, 1024),\n", + " 'normalize_mean': [0.485, 0.456, 0.406],\n", + " 'normalize_std': [0.229, 0.224, 0.225],\n", + " 'high_precision': True\n", + " }\n", + " if background_color:\n", + " kwargs['background_color'] = background_color\n", + "\n", + " # Run segmentation\n", + " giq.segmentation.process_and_save_images(**kwargs)\n", + "\n", + " # Collect masks\n", + " masks = []\n", + " masks_folder = os.path.join(OUTPUT_DIR, 'masks')\n", + " if os.path.isdir(masks_folder):\n", + " for fn in sorted(os.listdir(masks_folder)):\n", + " path = os.path.join(masks_folder, fn)\n", + " if os.path.isfile(path):\n", + " with open(path, 'rb') as imgf:\n", + " b64 = base64.b64encode(imgf.read()).decode('utf-8')\n", + " masks.append({'filename': fn, 'base64': b64})\n", + "\n", + " # Collect background-modified images if any\n", + " modified = []\n", + " if background_color:\n", + " mod_folder = os.path.join(OUTPUT_DIR, 'bg_modified')\n", + " if os.path.isdir(mod_folder):\n", + " for fn in sorted(os.listdir(mod_folder)):\n", + " path = os.path.join(mod_folder, fn)\n", + " if os.path.isfile(path):\n", + " with open(path, 'rb') as imgf:\n", + " b64 = base64.b64encode(imgf.read()).decode('utf-8')\n", + " modified.append({'filename': fn, 'base64': b64})\n", + "\n", + " # Return results\n", + " return jsonify(masks=masks, bg_modified=modified)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "460f8ce6-82dd-4393-a588-bfbf7466cbe3", + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == '__main__':\n", + " app.run(host='0.0.0.0', port=5000)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/working/examples/web_tailor/main.ipynb b/working/examples/web_tailor/main.ipynb new file mode 100644 index 0000000..e0acced --- /dev/null +++ b/working/examples/web_tailor/main.ipynb @@ -0,0 +1,273 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "6d1331b2-f8b8-4ed0-a7c3-1d28785922a9", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import base64\n", + "from flask import Flask, request, jsonify\n", + "from flask_cors import CORS\n", + "import shutil\n", + "import urllib.request\n", + "import pandas as pd\n", + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import tinyViT\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes\n", + "from garmentiq.landmark.derivation.derivation_dict import derivation_dict\n", + "\n", + "# Initialize the empty tailer object\n", + "tailor = None\n", + "do_refine = None\n", + "do_derive = None\n", + "background_color = None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "286a359e-1014-4113-9d71-24208724573d", + "metadata": {}, + "outputs": [], + "source": [ + "app = Flask(__name__)\n", + "CORS(app)\n", + "\n", + "# where we put uploads & outputs\n", + "BASE_DIR = 'tailor_files'\n", + "INPUT_DIR = os.path.join(BASE_DIR, 'input')\n", + "OUTPUT_DIR = os.path.join(BASE_DIR, 'output')\n", + "MODELS_DIR = os.path.join(BASE_DIR, 'models')\n", + "os.makedirs(INPUT_DIR, exist_ok=True)\n", + "os.makedirs(OUTPUT_DIR, exist_ok=True)\n", + "os.makedirs(MODELS_DIR, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59917ee5-62f3-47f0-a953-d4bec64db55e", + "metadata": {}, + "outputs": [], + "source": [ + "# Download the classification model\n", + "# Define model URLs\n", + "classification_model_url = \"https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit_inditex_finetuned.pt\"\n", + "landmark_model_url = \"https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\"\n", + "\n", + "# Define local file paths using MODELS_DIR\n", + "classification_model_path = os.path.join(MODELS_DIR, \"tiny_vit_inditex_finetuned.pt\")\n", + "landmark_model_path = os.path.join(MODELS_DIR, \"hrnet.pth\")\n", + "\n", + "# Function to download only if file doesn't exist\n", + "def download_if_missing(url, destination_path):\n", + " if not os.path.exists(destination_path):\n", + " print(f\"Downloading to {destination_path}...\")\n", + " urllib.request.urlretrieve(url, destination_path)\n", + " print(\"Download complete.\")\n", + " else:\n", + " print(f\"File already exists: {destination_path}. Skipping download.\")\n", + "\n", + "# Perform conditional downloads\n", + "download_if_missing(classification_model_url, classification_model_path)\n", + "download_if_missing(landmark_model_url, landmark_model_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8877cadd-1ffd-4ce4-9017-c2f77a8dc750", + "metadata": {}, + "outputs": [], + "source": [ + "def encode_image_to_base64(path):\n", + " with open(path, \"rb\") as img_file:\n", + " return base64.b64encode(img_file.read()).decode('utf-8')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4232f4e-3b49-4ee2-b2d9-ad754c292e24", + "metadata": {}, + "outputs": [], + "source": [ + "@app.route('/health')\n", + "def health_check():\n", + " return jsonify({\n", + " \"status\": \"active\",\n", + " \"token\": \"31f21a4323cf148339669c736f522cf89fa570a2bffc80ba874078477b76f81b\",\n", + " \"model\": \"web_tailor\",\n", + " \"version\": \"1.0.0\"\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd94aa86-b570-4809-9078-0499750c40bb", + "metadata": {}, + "outputs": [], + "source": [ + "@app.route('/setup', methods=['POST'])\n", + "def setup():\n", + " global tailor\n", + " global do_refine\n", + " global do_derive\n", + " global background_color\n", + "\n", + " do_refine = request.form.get('do_refine') == 'true'\n", + " do_derive = request.form.get('do_derive') == 'true'\n", + " r = request.form.get('red', '')\n", + " g = request.form.get('green', '')\n", + " b = request.form.get('blue', '')\n", + " background_color = None\n", + "\n", + " if r and g and b:\n", + " try:\n", + " rgb = [int(r), int(g), int(b)]\n", + " if any(c < 0 or c > 255 for c in rgb):\n", + " raise ValueError\n", + " background_color = rgb\n", + " except ValueError:\n", + " return jsonify(error='Invalid RGB values; must be integers 0–255'), 400\n", + "\n", + " # Setup the tailor agent\n", + " tailor = giq.tailor(\n", + " input_dir=INPUT_DIR,\n", + " model_dir=MODELS_DIR,\n", + " output_dir=OUTPUT_DIR,\n", + " class_dict=garment_classes,\n", + " do_derive=do_derive,\n", + " derivation_dict=derivation_dict,\n", + " do_refine=do_refine,\n", + " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", + " classification_model_class=tinyViT,\n", + " classification_model_args={\n", + " \"num_classes\": len(garment_classes),\n", + " \"img_size\": (120, 184),\n", + " \"patch_size\": 6,\n", + " \"resize_dim\": (120, 184),\n", + " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", + " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", + " },\n", + " segmentation_model_name=\"lygitdata/BiRefNet_garmentiq_backup\",\n", + " segmentation_model_args={\n", + " \"trust_remote_code\": True,\n", + " \"resize_dim\": (1024, 1024),\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + " \"high_precision\": True,\n", + " \"background_color\": background_color,\n", + " },\n", + " landmark_detection_model_path=\"hrnet.pth\",\n", + " landmark_detection_model_class=PoseHighResolutionNet(),\n", + " landmark_detection_model_args={\n", + " \"scale_std\": 200.0,\n", + " \"resize_dim\": [288, 384],\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + " },\n", + " )\n", + "\n", + " return jsonify(message='Tailor setup complete'), 200" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "233bec33-c6a0-4542-819c-72fe4f873f0c", + "metadata": {}, + "outputs": [], + "source": [ + "@app.route('/measure', methods=['POST'])\n", + "def measure():\n", + " global tailor, do_refine, do_derive, background_color\n", + " \n", + " files = request.files.getlist('images')\n", + " if not files:\n", + " return jsonify(error='No images uploaded'), 400\n", + "\n", + " # Ensure input/output directories are fresh\n", + " for d in (INPUT_DIR, OUTPUT_DIR):\n", + " if os.path.exists(d):\n", + " shutil.rmtree(d)\n", + " os.makedirs(d, exist_ok=True)\n", + "\n", + " # Save uploaded images\n", + " for f in files:\n", + " f.save(os.path.join(INPUT_DIR, f.filename))\n", + "\n", + " # Run segmentation & measurement\n", + " metadata, _ = tailor.measure(save_segmentation_image=True, save_measurement_image=True)\n", + " metadata = metadata.sort_values(by='filename', ascending=True)\n", + "\n", + " # Structure results\n", + " results = []\n", + " for idx, row in metadata.iterrows():\n", + " filename = row['filename']\n", + " \n", + " # Read & encode the JSON contents\n", + " json_b64 = None\n", + " json_path = row.get(\"measurement_json\")\n", + " if json_path:\n", + " with open(json_path, 'r') as jf:\n", + " text = jf.read()\n", + " json_b64 = base64.b64encode(text.encode('utf-8')).decode('utf-8')\n", + " \n", + " entry = {\n", + " \"Image name\": filename,\n", + " \"Class\": row.get(\"class\", None),\n", + " \"Measurement image\": encode_image_to_base64(row[\"measurement_image\"]),\n", + " \"Measurement JSON (base64)\": json_b64,\n", + " }\n", + " \n", + " if row.get(\"mask_image\"):\n", + " entry[\"Mask\"] = encode_image_to_base64(row[\"mask_image\"])\n", + " \n", + " if row.get(\"bg_modified_image\"):\n", + " entry[\"Background modified\"] = encode_image_to_base64(row[\"bg_modified_image\"])\n", + " \n", + " results.append(entry)\n", + " \n", + " return jsonify(results), 200" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "677539e0-33bb-4dcd-b976-5937730e4b9b", + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == '__main__':\n", + " app.run(host='0.0.0.0', port=5002)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 99601c3e9d7c00e1133ce8292536fcb103eb972c Mon Sep 17 00:00:00 2001 From: Lee <120439693+lygitdata@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:55:56 +0800 Subject: [PATCH 2/3] Add files via upload --- ...efinement_and_derivation_quick_start.ipynb | 17 +- .../segmentation_quick_start.ipynb | 106 +++++++--- .../tailor_quick_start.ipynb | 188 ++++++++++++++++-- working/examples/web_segmentation/main.ipynb | 28 ++- working/examples/web_tailor/main.ipynb | 14 +- 5 files changed, 281 insertions(+), 72 deletions(-) diff --git a/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb b/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb index 3813cc8..d9402bb 100644 --- a/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb +++ b/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb @@ -19,25 +19,30 @@ "source": [ "import garmentiq as giq\n", "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", - "from garmentiq.garment_classes import garment_classes" + "from garmentiq.garment_classes import garment_classes\n", + "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "id": "3-b9AZr9hJyK" - }, + "metadata": {}, "outputs": [], "source": [ - "# Download a vest dress image and a pretrained model\n", + "# Download a vest dress image\n", "!mkdir -p test_image\n", "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg \\\n", " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", "\n", + "# Download HRNet model\n", "!mkdir -p models\n", "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/models/hrnet.pth \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", + "\n", + "# Download BiRefNet model\n", + "!mkdir -p /app/working/examples/landmark_refinement_and_derivation_quick_start/models/birefnet\n", + "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors" ] }, { diff --git a/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb b/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb index f3ce798..7628c9f 100644 --- a/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb +++ b/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb @@ -17,77 +17,119 @@ }, "outputs": [], "source": [ - "import garmentiq as giq" + "import garmentiq as giq\n", + "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config\n", + "from garmentiq.segmentation.model_definition.sam import SamModel, load_sam_config, load_sam_processor" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "id": "3-b9AZr9hJyK" - }, + "metadata": {}, "outputs": [], "source": [ "# Download a test image\n", "!mkdir -p /app/working/examples/segmentation_quick_start/test_image\n", "!wget -q -O /app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg" + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg\n", + "\n", + "# Download BiRefNet model\n", + "!mkdir -p /app/working/examples/segmentation_quick_start/models/birefnet\n", + "\n", + "!wget -q -O /app/working/examples/segmentation_quick_start/models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", + "\n", + "# Download SAM base model\n", + "!mkdir -p /app/working/examples/segmentation_quick_start/models/sam_b\n", + "\n", + "!wget -q -O /app/working/examples/segmentation_quick_start/models/sam_b/model.safetensors \\\n", + " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors" ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pMGZwL-UjPPN" - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "# Load the pretrained model from Hugging Face\n", - "BiRefNet = giq.segmentation.load_model(\n", - " pretrained_model='lygitdata/BiRefNet_garmentiq_backup',\n", - " pretrained_model_args={'trust_remote_code': True},\n", - " high_precision=True\n", - ")" + "## Example with BiRefNet model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "hfXjYs-mkE4s" + "id": "pMGZwL-UjPPN" }, "outputs": [], "source": [ + "# Load the BiRefNet model\n", + "birefnet = giq.segmentation.load_model(\n", + " model_class=BiRefNet,\n", + " model_path=\"/app/working/examples/segmentation_quick_start/models/birefnet/model.safetensors\",\n", + " model_args=load_birefnet_config()\n", + ")\n", + "\n", "# Extract the mask\n", - "original_img, mask = giq.segmentation.extract(\n", - " model=BiRefNet,\n", + "original_img_biref, mask_biref = giq.segmentation.extract(\n", + " model=birefnet,\n", " image_path='/app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg',\n", + " # BiRefNet specific kwargs:\n", " resize_dim=(1024, 1024),\n", " normalize_mean=[0.485, 0.456, 0.406],\n", - " normalize_std=[0.229, 0.224, 0.225],\n", - " high_precision=True\n", + " normalize_std=[0.229, 0.224, 0.225]\n", ")\n", "\n", "# Change background color\n", - "bg_modified_img = giq.segmentation.change_background_color(\n", - " image_np=original_img,\n", - " mask_np=mask,\n", + "bg_modified_img_biref = giq.segmentation.change_background_color(\n", + " image_np=original_img_biref,\n", + " mask_np=mask_biref,\n", " background_color=[102, 255, 102]\n", - ")" + ")\n", + "\n", + "# Plot the original image, mask, and background modified image\n", + "giq.segmentation.plot(image_np=original_img_biref, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=mask_biref, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=bg_modified_img_biref, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example with SAM base model" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "id": "dPkVas24rw2H" - }, + "metadata": {}, "outputs": [], "source": [ + "# Load the BiRefNet model\n", + "sam = giq.segmentation.load_model(\n", + " model_class=SamModel,\n", + " model_path=\"/app/working/examples/segmentation_quick_start/models/sam_b/model.safetensors\",\n", + " model_args={\"config\": load_sam_config(\"sam-vit-b\")}\n", + ")\n", + "\n", + "# Extract the mask\n", + "original_img_sam, mask_sam = giq.segmentation.extract(\n", + " model=sam,\n", + " image_path='/app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg',\n", + " processor=load_sam_processor(\"sam-vit-b\"),\n", + " input_points=[[[512, 512]]]\n", + ")\n", + "\n", + "# Change background color\n", + "bg_modified_img_sam = giq.segmentation.change_background_color(\n", + " image_np=original_img_sam,\n", + " mask_np=mask_sam,\n", + " background_color=[102, 255, 102]\n", + ")\n", + "\n", "# Plot the original image, mask, and background modified image\n", - "giq.segmentation.plot(image_np=original_img, figsize=(3, 3))\n", - "giq.segmentation.plot(image_np=mask, figsize=(3, 3))\n", - "giq.segmentation.plot(image_np=bg_modified_img, figsize=(3, 3))" + "giq.segmentation.plot(image_np=original_img_sam, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=mask_sam, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=bg_modified_img_sam, figsize=(3, 3))" ] } ], diff --git a/working/examples/tailor_quick_start/tailor_quick_start.ipynb b/working/examples/tailor_quick_start/tailor_quick_start.ipynb index 5e0e2dc..3c80a62 100644 --- a/working/examples/tailor_quick_start/tailor_quick_start.ipynb +++ b/working/examples/tailor_quick_start/tailor_quick_start.ipynb @@ -21,22 +21,20 @@ "from garmentiq.classification.model_definition import tinyViT\n", "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", "from garmentiq.garment_classes import garment_classes\n", - "from garmentiq.landmark.derivation.derivation_dict import derivation_dict" + "from garmentiq.landmark.derivation.derivation_dict import derivation_dict\n", + "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config\n", + "from garmentiq.segmentation.model_definition.sam import SamModel, load_sam_config, load_sam_processor" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "id": "3-b9AZr9hJyK" - }, + "metadata": {}, "outputs": [], "source": [ - "# Download 4 test images\n", - "# cloth_1 and cloth_2 are short sleeve tops, cloth_3 is vest dress, cloth_4 is skirt\n", + "# Download 3 test images\n", + "# cloth_2 is short sleeve tops, cloth_3 is vest dress, cloth_4 is skirt\n", "!mkdir -p /app/working/examples/tailor_quick_start/test_image\n", - "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_1.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg\n", "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_2.jpg \\\n", " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_2.jpg\n", "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_3.jpg \\\n", @@ -45,13 +43,30 @@ " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_4.jpg\n", "\n", "# Download the classification model\n", - "!mkdir -p /app/working/examples/tailor_quick_start/models\n", + "!mkdir -p models\n", "!wget -q -O /app/working/examples/tailor_quick_start/models/tiny_vit_inditex_finetuned.pt \\\n", " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit_inditex_finetuned.pt\n", "\n", "# Download the landmark detection model\n", "!wget -q -O /app/working/examples/tailor_quick_start/models/hrnet.pth \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", + "\n", + "# Download BiRefNet model\n", + "!mkdir -p models/birefnet\n", + "!wget -q -O /app/working/examples/tailor_quick_start/models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", + "\n", + "# Download SAM base model\n", + "!mkdir -p models/sam_b\n", + "!wget -q -O /app/working/examples/tailor_quick_start/models/sam_b/model.safetensors \\\n", + " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example with BiRefNet model" ] }, { @@ -63,10 +78,10 @@ "outputs": [], "source": [ "# Setup the tailor agent\n", - "tailor = giq.tailor(\n", + "tailor_biref = giq.tailor(\n", " input_dir=\"/app/working/examples/tailor_quick_start/test_image\",\n", " model_dir=\"/app/working/examples/tailor_quick_start/models\",\n", - " output_dir=\"/app/working/examples/tailor_quick_start/output\",\n", + " output_dir=\"/app/working/examples/tailor_quick_start/output_biref\",\n", " class_dict=garment_classes,\n", " do_derive=True,\n", " derivation_dict=derivation_dict,\n", @@ -81,14 +96,14 @@ " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", " },\n", - " segmentation_model_name=\"lygitdata/BiRefNet_garmentiq_backup\",\n", + " segmentation_model_path=\"birefnet/model.safetensors\",\n", + " segmentation_model_class=BiRefNet,\n", " segmentation_model_args={\n", - " \"trust_remote_code\": True,\n", + " \"model_config\": load_birefnet_config(),\n", " \"resize_dim\": (1024, 1024),\n", " \"normalize_mean\": [0.485, 0.456, 0.406],\n", " \"normalize_std\": [0.229, 0.224, 0.225],\n", - " \"high_precision\": True,\n", - " \"background_color\": [102, 255, 102],\n", + " \"background_color\": [102, 255, 102]\n", " },\n", " landmark_detection_model_path=\"hrnet.pth\",\n", " landmark_detection_model_class=PoseHighResolutionNet(),\n", @@ -101,7 +116,7 @@ ")\n", "\n", "# See the tailor agent's basic information\n", - "tailor.summary()" + "tailor_biref.summary()" ] }, { @@ -113,7 +128,7 @@ "outputs": [], "source": [ "# Start the measurement with refinement and derivation\n", - "metadata, outputs = tailor.measure(save_segmentation_image=True, save_measurement_image=True)" + "metadata, outputs = tailor_biref.measure(save_segmentation_image=True, save_measurement_image=True)" ] }, { @@ -139,7 +154,7 @@ "outputs": [], "source": [ "# Plot the masks\n", - "# Go to /app/working/examples/tailor_quick_start/output/mask_image/ to see the high resolution images\n", + "# Go to /app/working/examples/tailor_quick_start/output_biref/mask_image/ to see the high resolution images\n", "for image in metadata['mask_image']:\n", " giq.landmark.plot(image_path=image, figsize=(3, 3))" ] @@ -153,7 +168,7 @@ "outputs": [], "source": [ "# Plot the background modified images\n", - "# Go to /app/working/examples/tailor_quick_start/output/bg_modified_image to see the high resolution images\n", + "# Go to /app/working/examples/tailor_quick_start/output_biref/bg_modified_image to see the high resolution images\n", "for image in metadata['bg_modified_image']:\n", " giq.landmark.plot(image_path=image, figsize=(3, 3))" ] @@ -167,7 +182,7 @@ "outputs": [], "source": [ "# Plot the images with desired landmarks\n", - "# Go to /app/working/examples/tailor_quick_start/output/measurement_image/ to see the high resolution images\n", + "# Go to /app/working/examples/tailor_quick_start/output_biref/measurement_image/ to see the high resolution images\n", "for image in metadata['measurement_image']:\n", " giq.landmark.plot(image_path=image, figsize=(3, 3))" ] @@ -181,7 +196,136 @@ "outputs": [], "source": [ "# See the measurement results in JSON format\n", - "# Go to /app/working/examples/tailor_quick_start/output/measurement_json/ to see the JSON files\n", + "# Go to /app/working/examples/tailor_quick_start/output_biref/measurement_json/ to see the JSON files\n", + "import json\n", + "\n", + "for json_path in metadata['measurement_json']:\n", + " with open(json_path, 'r') as file:\n", + " data = json.load(file)\n", + " print(f\"{json_path}:\\n\")\n", + " print(json.dumps(data, indent=4, sort_keys=True))\n", + " print(\"\\n\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example with SAM base model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Setup the tailor agent\n", + "tailor_sam = giq.tailor(\n", + " input_dir=\"/app/working/examples/tailor_quick_start/test_image\",\n", + " model_dir=\"/app/working/examples/tailor_quick_start/models\",\n", + " output_dir=\"/app/working/examples/tailor_quick_start/output_sam\",\n", + " class_dict=garment_classes,\n", + " do_derive=True,\n", + " derivation_dict=derivation_dict,\n", + " do_refine=False,\n", + " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", + " classification_model_class=tinyViT,\n", + " classification_model_args={\n", + " \"num_classes\": len(list(garment_classes.keys())),\n", + " \"img_size\": (120, 184),\n", + " \"patch_size\": 6,\n", + " \"resize_dim\": (120, 184),\n", + " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", + " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", + " },\n", + " segmentation_model_path=\"sam_b/model.safetensors\",\n", + " segmentation_model_class=SamModel,\n", + " segmentation_model_args={\n", + " \"model_config\": {\"config\": load_sam_config(\"sam-vit-b\")},\n", + " \"processor\": load_sam_processor(\"sam-vit-b\", use_fast=False),\n", + " \"input_points\": [[[1000, 900]]],\n", + " \"background_color\": [102, 255, 102]\n", + " },\n", + " landmark_detection_model_path=\"hrnet.pth\",\n", + " landmark_detection_model_class=PoseHighResolutionNet(),\n", + " landmark_detection_model_args={\n", + " \"scale_std\": 200.0,\n", + " \"resize_dim\": [288, 384],\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + " },\n", + ")\n", + "\n", + "# See the tailor agent's basic information\n", + "tailor_sam.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Start the measurement with refinement and derivation\n", + "metadata, outputs = tailor_sam.measure(save_segmentation_image=True, save_measurement_image=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# See the metadata\n", + "# It makes file access much easier\n", + "print(metadata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Plot the masks\n", + "# Go to /content/output_sam/mask_image/ to see the high resolution images\n", + "for image in metadata['mask_image']:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Plot the background modified images\n", + "# Go to /content/output_sam/bg_modified_image to see the high resolution images\n", + "for image in metadata['bg_modified_image']:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3)) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Plot the images with desired landmarks\n", + "# Go to /content/output_sam/measurement_image/ to see the high resolution images\n", + "for image in metadata['measurement_image']:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# See the measurement results in JSON format\n", + "# Go to /content/output_sam/measurement_json/ to see the JSON files\n", "import json\n", "\n", "for json_path in metadata['measurement_json']:\n", diff --git a/working/examples/web_segmentation/main.ipynb b/working/examples/web_segmentation/main.ipynb index 1a04a95..275a839 100644 --- a/working/examples/web_segmentation/main.ipynb +++ b/working/examples/web_segmentation/main.ipynb @@ -12,9 +12,24 @@ "from flask import Flask, request, jsonify\n", "from flask_cors import CORS\n", "import garmentiq as giq\n", + "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config\n", "import shutil" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "a549833b", + "metadata": {}, + "outputs": [], + "source": [ + "# Download BiRefNet model\n", + "!mkdir -p /app/working/examples/web_segmentation/models/birefnet\n", + "\n", + "!wget -q -O /app/working/examples/web_segmentation/models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors" + ] + }, { "cell_type": "code", "execution_count": null, @@ -34,10 +49,10 @@ "\n", "# load model once\n", "print(\"Loading BiRefNet model…\")\n", - "BiRefNet = giq.segmentation.load_model(\n", - " pretrained_model='lygitdata/BiRefNet_garmentiq_backup',\n", - " pretrained_model_args={'trust_remote_code': True},\n", - " high_precision=True\n", + "birefnet = giq.segmentation.load_model(\n", + " model_class=BiRefNet,\n", + " model_path=\"/app/working/examples/web_segmentation/models/birefnet/model.safetensors\",\n", + " model_args=load_birefnet_config()\n", ")\n", "print(\"Model loaded.\")" ] @@ -92,11 +107,10 @@ " kwargs = {\n", " 'image_dir': INPUT_DIR,\n", " 'output_dir': OUTPUT_DIR,\n", - " 'model': BiRefNet,\n", + " 'model': birefnet,\n", " 'resize_dim': (1024, 1024),\n", " 'normalize_mean': [0.485, 0.456, 0.406],\n", - " 'normalize_std': [0.229, 0.224, 0.225],\n", - " 'high_precision': True\n", + " 'normalize_std': [0.229, 0.224, 0.225]\n", " }\n", " if background_color:\n", " kwargs['background_color'] = background_color\n", diff --git a/working/examples/web_tailor/main.ipynb b/working/examples/web_tailor/main.ipynb index e0acced..e39e738 100644 --- a/working/examples/web_tailor/main.ipynb +++ b/working/examples/web_tailor/main.ipynb @@ -19,6 +19,7 @@ "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", "from garmentiq.garment_classes import garment_classes\n", "from garmentiq.landmark.derivation.derivation_dict import derivation_dict\n", + "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config\n", "\n", "# Initialize the empty tailer object\n", "tailor = None\n", @@ -58,10 +59,12 @@ "# Define model URLs\n", "classification_model_url = \"https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit_inditex_finetuned.pt\"\n", "landmark_model_url = \"https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\"\n", + "segmentation_model_url = \"https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\"\n", "\n", "# Define local file paths using MODELS_DIR\n", "classification_model_path = os.path.join(MODELS_DIR, \"tiny_vit_inditex_finetuned.pt\")\n", "landmark_model_path = os.path.join(MODELS_DIR, \"hrnet.pth\")\n", + "segmentation_model_path = os.path.join(MODELS_DIR, \"model.safetensors\")\n", "\n", "# Function to download only if file doesn't exist\n", "def download_if_missing(url, destination_path):\n", @@ -74,7 +77,8 @@ "\n", "# Perform conditional downloads\n", "download_if_missing(classification_model_url, classification_model_path)\n", - "download_if_missing(landmark_model_url, landmark_model_path)" + "download_if_missing(landmark_model_url, landmark_model_path)\n", + "download_if_missing(segmentation_model_url, segmentation_model_path)" ] }, { @@ -155,14 +159,14 @@ " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", " },\n", - " segmentation_model_name=\"lygitdata/BiRefNet_garmentiq_backup\",\n", + " segmentation_model_path=\"model.safetensors\",\n", + " segmentation_model_class=BiRefNet,\n", " segmentation_model_args={\n", - " \"trust_remote_code\": True,\n", + " \"model_config\": load_birefnet_config(),\n", " \"resize_dim\": (1024, 1024),\n", " \"normalize_mean\": [0.485, 0.456, 0.406],\n", " \"normalize_std\": [0.229, 0.224, 0.225],\n", - " \"high_precision\": True,\n", - " \"background_color\": background_color,\n", + " \"background_color\": [102, 255, 102]\n", " },\n", " landmark_detection_model_path=\"hrnet.pth\",\n", " landmark_detection_model_class=PoseHighResolutionNet(),\n", From aa88c3033eb8665708feccb2d0dd43815dd0e328 Mon Sep 17 00:00:00 2001 From: lygitdata Date: Sat, 12 Sep 2026 12:19:57 +0000 Subject: [PATCH 3/3] Update to magicbox.v1.8 --- README.md | 4 +- ...age_classification_model_fine_tuning.ipynb | 284 +++++++++ ...sification_model_training_evaluation.ipynb | 388 ++++++++++++ ...usage_custom_measurement_instruction.ipynb | 370 +++++++++++ ...ion_model_fine_tuning_advanced_usage.ipynb | 178 ------ ...l_training_evaluation_advanced_usage.ipynb | 278 -------- .../classification_quick_start.ipynb | 163 ----- ...asurement_instruction_advanced_usage.ipynb | 278 -------- .../landmark_detection_quick_start.ipynb | 141 ----- ...efinement_and_derivation_quick_start.ipynb | 213 ------- .../segmentation_quick_start.ipynb | 163 ----- .../tailor_quick_start.ipynb | 367 ----------- .../tutorial_classification.ipynb | 246 ++++++++ .../tutorial_grounding.ipynb | 297 +++++++++ .../tutorial_landmark_detection.ipynb | 228 +++++++ ...l_landmark_refinement_and_derivation.ipynb | 320 ++++++++++ .../tutorial_matting/tutorial_matting.ipynb | 490 ++++++++++++++ .../tutorial_segmentation.ipynb | 596 ++++++++++++++++++ .../tutorial_tailor/tutorial_tailor.ipynb | 527 ++++++++++++++++ .../web_landmark_detection/main.ipynb | 8 +- working/examples/web_segmentation/main.ipynb | 6 +- 21 files changed, 3755 insertions(+), 1790 deletions(-) create mode 100644 working/examples/adv_usage_classification_model_fine_tuning/adv_usage_classification_model_fine_tuning.ipynb create mode 100644 working/examples/adv_usage_classification_model_training_evaluation/adv_usage_classification_model_training_evaluation.ipynb create mode 100644 working/examples/adv_usage_custom_measurement_instruction/adv_usage_custom_measurement_instruction.ipynb delete mode 100644 working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb delete mode 100644 working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb delete mode 100644 working/examples/classification_quick_start/classification_quick_start.ipynb delete mode 100644 working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb delete mode 100644 working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb delete mode 100644 working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb delete mode 100644 working/examples/segmentation_quick_start/segmentation_quick_start.ipynb delete mode 100644 working/examples/tailor_quick_start/tailor_quick_start.ipynb create mode 100644 working/examples/tutorial_classification/tutorial_classification.ipynb create mode 100644 working/examples/tutorial_grounding/tutorial_grounding.ipynb create mode 100644 working/examples/tutorial_landmark_detection/tutorial_landmark_detection.ipynb create mode 100644 working/examples/tutorial_landmark_refinement_and_derivation/tutorial_landmark_refinement_and_derivation.ipynb create mode 100644 working/examples/tutorial_matting/tutorial_matting.ipynb create mode 100644 working/examples/tutorial_segmentation/tutorial_segmentation.ipynb create mode 100644 working/examples/tutorial_tailor/tutorial_tailor.ipynb diff --git a/README.md b/README.md index 922bc13..66a81fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# GarmentIQ MagicBox (Version 1.7) +# GarmentIQ MagicBox (Version 1.8) -*Last update: 06/20/2025* +*Last update: 09/12/2025* *Navigation*: diff --git a/working/examples/adv_usage_classification_model_fine_tuning/adv_usage_classification_model_fine_tuning.ipynb b/working/examples/adv_usage_classification_model_fine_tuning/adv_usage_classification_model_fine_tuning.ipynb new file mode 100644 index 0000000..63aa90a --- /dev/null +++ b/working/examples/adv_usage_classification_model_fine_tuning/adv_usage_classification_model_fine_tuning.ipynb @@ -0,0 +1,284 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced Usage - GarmentIQ Classification Model Fine-tuning\n", + "\n", + "Fine-tuning adapts the pretrained GarmentIQ classifier to your own catalog without\n", + "training from scratch. It starts from the shipped tinyViT weights, freezes the backbone,\n", + "and retrains only the classifier head, so it needs far less data and time than full\n", + "training.\n", + "\n", + "This tutorial shows how to prepare a fine-tuning dataset, freeze and unfreeze the right\n", + "layers, run cross-validated fine-tuning, and evaluate the resulting model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Prepare the fine-tuning data](#data)\n", + "3. [Fine-tune the model](#finetune)\n", + "4. [Evaluate the fine-tuned model](#evaluate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package, then download the fine-tuning dataset and the base model. On Colab\n", + "you can keep this section collapsed.\n", + "\n", + "> **Your data must be a zip file with the same structure as ours**, that is an image\n", + "> folder plus a `metadata.csv` naming each file and its label. See\n", + "> [the example dataset](https://www.kaggle.com/datasets/lygitdata/zara-clothes-image-data)\n", + "> for the exact layout." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import torch\n", + "import torch.optim as optim\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import tinyViT\n", + "from garmentiq.classification.utils import CachedDataset\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own. For training and fine-tuning the\n", + "# device is passed inside `param`, and it defaults to \"cpu\". Training on CPU is slow,\n", + "# so use a GPU runtime (\"cuda\") or Apple Silicon (\"mps\") where available.\n", + "if torch.cuda.is_available():\n", + " device = \"cuda\"\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + "else:\n", + " device = \"cpu\"\n", + "\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the fine-tuning data and the base model\n", + "\n", + "# About 2.2 GB\n", + "!curl -sL -o zara-clothes-image-data.zip \\\n", + " https://www.kaggle.com/api/v1/datasets/download/lygitdata/zara-clothes-image-data\n", + "\n", + "# The pretrained tinyViT that will be fine-tuned\n", + "!mkdir -p ./models\n", + "!wget -q -O ./models/tiny_vit.pt \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit.pt\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prepare the fine-tuning data\n", + "\n", + "`train_test_split` unpacks the archive and splits it. This dataset is small and every\n", + "image is wanted for fine-tuning, so `test_size=0` keeps the test set empty. The summary\n", + "below confirms its size is 0." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = giq.classification.train_test_split(\n", + " output_dir=\"data\",\n", + " train_zip_dir=\"zara-clothes-image-data.zip\",\n", + " metadata_csv=\"metadata.csv\",\n", + " label_column=\"garment\",\n", + " test_size=0,\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Caching the images in memory avoids re-reading them from disk on every\n", + "epoch, which is usually the bottleneck. The normalization statistics must match the ones\n", + "the base model was trained with." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_images, train_labels, _ = giq.classification.load_data(\n", + " df=data[\"train_metadata\"],\n", + " img_dir=data[\"train_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(120, 184),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Fine-tune the model\n", + "\n", + "The fine-tuning specific keys live in `param`:\n", + "\n", + "| Key | Purpose |\n", + "|---|---|\n", + "| `pretrained_path` | the weights to start from |\n", + "| `freeze_layers` | freeze the backbone so its features are preserved |\n", + "| `unfreeze_patterns` | substrings of layer names to keep trainable |\n", + "| `device` | where to run, defaults to `\"cpu\"` |\n", + "\n", + "Freezing everything except the classifier head is what makes fine-tuning cheap: the\n", + "backbone already knows how to read garment images, and only the final mapping onto your\n", + "labels is relearned. A small learning rate keeps the pretrained features intact.\n", + "\n", + "Training is cross-validated, and the model with the lowest cross-entropy is saved as the\n", + "best one. Five folds and five epochs are used here for demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giq.classification.fine_tune_pytorch_nn(\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"metadata_df\": data[\"train_metadata\"],\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\n", + " \"pretrained_path\": \"./models/tiny_vit.pt\",\n", + " \"freeze_layers\": True,\n", + " \"unfreeze_patterns\": [\"classifier\", \"fc\"],\n", + " \"optimizer_class\": optim.AdamW,\n", + " \"optimizer_args\": {\"lr\": 0.00002, \"weight_decay\": 1e-4},\n", + " \"n_fold\": 5,\n", + " \"n_epoch\": 5,\n", + " \"patience\": 2,\n", + " \"batch_size\": 128,\n", + " \"model_save_dir\": \"finetuned_models\",\n", + " \"best_model_name\": \"best_finetuned.pt\",\n", + " \"device\": device,\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Evaluate the fine-tuned model\n", + "\n", + "`test_pytorch_nn` loads the weights itself, so it takes a model path rather than a loaded\n", + "model. Its `device` also goes inside `param`.\n", + "\n", + "> Scores here are measured on the fine-tuning data itself, so they show how well the model\n", + "> fitted it, not how well it generalizes. Hold out a test set to measure that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giq.classification.test_pytorch_nn(\n", + " model_path=\"finetuned_models/best_finetuned.pt\",\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\"batch_size\": 64, \"device\": device},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use the fine-tuned model for prediction, load it exactly like the\n", + "shipped one. See the\n", + "[classification tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_classification.ipynb)." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/adv_usage_classification_model_training_evaluation/adv_usage_classification_model_training_evaluation.ipynb b/working/examples/adv_usage_classification_model_training_evaluation/adv_usage_classification_model_training_evaluation.ipynb new file mode 100644 index 0000000..ea3f3b8 --- /dev/null +++ b/working/examples/adv_usage_classification_model_training_evaluation/adv_usage_classification_model_training_evaluation.ipynb @@ -0,0 +1,388 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced Usage - GarmentIQ Classification Model Training and Evaluation\n", + "\n", + "Fine-tuning adapts an existing model, but sometimes you need to train one from scratch.\n", + "GarmentIQ can train any PyTorch model through the same interface, whether it is one of the\n", + "predefined architectures or your own.\n", + "\n", + "This tutorial shows how to train the built-in CNN3, how to plug in a model you wrote\n", + "yourself, and how to evaluate both on a held-out test set so the results can be compared\n", + "fairly." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Prepare the data](#data)\n", + "3. [Train a predefined model](#cnn3)\n", + "4. [Train your own model](#custom)\n", + "5. [Evaluate and compare](#evaluate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download the training data. On Colab you can keep this section\n", + "collapsed.\n", + "\n", + "> **Your data must be a zip file with the same structure as ours**, that is an image\n", + "> folder plus a `metadata.csv` naming each file and its label. See\n", + "> [the example dataset](https://www.kaggle.com/datasets/lygitdata/garmentiq-classification-set-nordstrom-and-myntra)\n", + "> for the exact layout." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import CNN3\n", + "from garmentiq.classification.utils import CachedDataset\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own. For training the device is passed\n", + "# inside `param`, and it defaults to \"cpu\".\n", + "# Both models below end in an adaptive pooling layer whose input size is not divisible\n", + "# by its output size, which PyTorch does not implement on Apple Silicon (\"mps\"), so\n", + "# this notebook uses CUDA or CPU. tinyViT has no such layer and does run on \"mps\";\n", + "# see the fine-tuning notebook.\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the training data\n", + "\n", + "# About 1.5 GB\n", + "!curl -sL -o garmentiq-classification-set-nordstrom-and-myntra.zip \\\n", + " https://www.kaggle.com/api/v1/datasets/download/lygitdata/garmentiq-classification-set-nordstrom-and-myntra\n", + "\n", + "print(\"Download finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prepare the data\n", + "\n", + "Unlike fine-tuning, training from scratch needs a held-out test set, so 15% is reserved\n", + "here. Both models below are trained and evaluated on exactly the same split, which is what\n", + "makes the comparison meaningful." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = giq.classification.train_test_split(\n", + " output_dir=\"data\",\n", + " train_zip_dir=\"garmentiq-classification-set-nordstrom-and-myntra.zip\",\n", + " metadata_csv=\"metadata.csv\",\n", + " label_column=\"garment\",\n", + " test_size=0.15,\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "These models are trained at a smaller `resize_dim` than the shipped\n", + "tinyViT, which trains faster and suits the simpler architectures. Whatever you choose here\n", + "must be reused at evaluation and prediction time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_images, train_labels, _ = giq.classification.load_data(\n", + " df=data[\"train_metadata\"],\n", + " img_dir=data[\"train_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(60, 92),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Train a predefined model\n", + "\n", + "`train_pytorch_nn` takes the architecture as `model_class` and its constructor arguments\n", + "as `model_args`, so switching models is a one-line change. Training is cross-validated and\n", + "the checkpoint with the lowest cross-entropy is kept as the best one.\n", + "\n", + "Two folds and five epochs are used here for demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giq.classification.train_pytorch_nn(\n", + " model_class=CNN3,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"metadata_df\": data[\"train_metadata\"],\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\n", + " \"optimizer_class\": optim.AdamW,\n", + " \"optimizer_args\": {\"lr\": 0.001, \"weight_decay\": 1e-4},\n", + " \"n_fold\": 2,\n", + " \"n_epoch\": 5,\n", + " \"patience\": 2,\n", + " \"batch_size\": 256,\n", + " \"model_save_dir\": \"cnn3_models\",\n", + " \"best_model_name\": \"best_cnn3_model.pt\",\n", + " \"device\": device,\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Train your own model\n", + "\n", + "Any `torch.nn.Module` works. GarmentIQ only requires that the constructor accepts the\n", + "arguments you list in `model_args`, and that `forward` returns one logit per class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class UserDefinedCNN(nn.Module):\n", + " def __init__(self, num_classes):\n", + " super(UserDefinedCNN, self).__init__()\n", + " self.features = nn.Sequential(\n", + " nn.Conv2d(3, 32, kernel_size=3, padding=1),\n", + " nn.BatchNorm2d(32),\n", + " nn.ReLU(inplace=True),\n", + " nn.MaxPool2d(2, 2),\n", + " nn.Dropout(0.2),\n", + " nn.Conv2d(32, 64, kernel_size=3, padding=1),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(inplace=True),\n", + " nn.MaxPool2d(2, 2),\n", + " nn.Dropout(0.25),\n", + " nn.AdaptiveAvgPool2d((4, 4)),\n", + " )\n", + " self.classifier = nn.Sequential(\n", + " nn.Linear(64 * 4 * 4, 128),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(0.3),\n", + " nn.Linear(128, num_classes),\n", + " )\n", + "\n", + " def forward(self, x):\n", + " x = self.features(x)\n", + " x = x.view(x.size(0), -1)\n", + " x = self.classifier(x)\n", + " return x" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The training call is identical apart from `model_class` and the output\n", + "directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giq.classification.train_pytorch_nn(\n", + " model_class=UserDefinedCNN,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"metadata_df\": data[\"train_metadata\"],\n", + " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", + " \"cached_images\": train_images,\n", + " \"cached_labels\": train_labels,\n", + " },\n", + " param={\n", + " \"optimizer_class\": optim.AdamW,\n", + " \"optimizer_args\": {\"lr\": 0.001, \"weight_decay\": 1e-4},\n", + " \"n_fold\": 2,\n", + " \"n_epoch\": 5,\n", + " \"patience\": 2,\n", + " \"batch_size\": 256,\n", + " \"model_save_dir\": \"userdefined_cnn_models\",\n", + " \"best_model_name\": \"best_userdefined_cnn_model.pt\",\n", + " \"device\": device,\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Evaluate and compare\n", + "\n", + "The test set was held out from both training runs, so neither model has seen it. Load it\n", + "with the same `resize_dim` and normalization used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_images, test_labels, _ = giq.classification.load_data(\n", + " df=data[\"test_metadata\"],\n", + " img_dir=data[\"test_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(60, 92),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The predefined CNN3\n", + "giq.classification.test_pytorch_nn(\n", + " model_path=\"cnn3_models/best_cnn3_model.pt\",\n", + " model_class=CNN3,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": data[\"test_metadata\"][\"garment\"],\n", + " \"cached_images\": test_images,\n", + " \"cached_labels\": test_labels,\n", + " },\n", + " param={\"batch_size\": 64, \"device\": device},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The user-defined model, scored the same way\n", + "giq.classification.test_pytorch_nn(\n", + " model_path=\"userdefined_cnn_models/best_userdefined_cnn_model.pt\",\n", + " model_class=UserDefinedCNN,\n", + " model_args={\"num_classes\": 9},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": data[\"test_metadata\"][\"garment\"],\n", + " \"cached_images\": test_images,\n", + " \"cached_labels\": test_labels,\n", + " },\n", + " param={\"batch_size\": 64, \"device\": device},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Compare the accuracy and F1 scores above to decide which architecture\n", + "suits your data. With these settings CNN3 typically wins, but the point is that both were\n", + "trained and scored through the same interface.\n", + "\n", + "If you would rather adapt the shipped model than train one from scratch, see the\n", + "[fine-tuning notebook](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/adv_usage_classification_model_fine_tuning.ipynb)." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/adv_usage_custom_measurement_instruction/adv_usage_custom_measurement_instruction.ipynb b/working/examples/adv_usage_custom_measurement_instruction/adv_usage_custom_measurement_instruction.ipynb new file mode 100644 index 0000000..c3454a8 --- /dev/null +++ b/working/examples/adv_usage_custom_measurement_instruction/adv_usage_custom_measurement_instruction.ipynb @@ -0,0 +1,370 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced Usage - GarmentIQ Custom Measurement Instruction\n", + "\n", + "A measurement instruction tells GarmentIQ which landmarks a garment has and which pairs of\n", + "them form a measurement. The defaults cover common garments, but you may want different\n", + "measurements, or a garment of your own.\n", + "\n", + "This tutorial shows how to read the predefined instructions, write a custom one, register\n", + "it in a garment class dictionary, and confirm that landmark detection returns exactly the\n", + "measurements you asked for." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Inspect the predefined instructions](#inspect)\n", + "3. [Write a custom instruction](#write)\n", + "4. [Register the instruction](#register)\n", + "5. [Compare the results](#compare)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download a skirt image and the landmark detection model. On Colab\n", + "you can keep this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import copy\n", + "import json\n", + "\n", + "import torch\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.garment_classes import garment_classes\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own: every model loader and every\n", + "# inference function takes a `device` argument that defaults to \"cpu\". Pass it\n", + "# explicitly to use a GPU (\"cuda\") or Apple Silicon (\"mps\").\n", + "if torch.cuda.is_available():\n", + " device = \"cuda\"\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + "else:\n", + " device = \"cpu\"\n", + "\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the test image and the landmark detection model\n", + "\n", + "!mkdir -p ./test_image\n", + "!wget -q -O ./test_image/cloth_4.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_4.jpg\n", + "\n", + "!mkdir -p ./models\n", + "!wget -q -O ./models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Inspect the predefined instructions\n", + "\n", + "`garment_classes` maps each garment to its metadata. Three fields matter here:\n", + "\n", + "| Field | Meaning |\n", + "|---|---|\n", + "| `num_predefined_points` | how many landmarks the detection model predicts |\n", + "| `index_range` | which slice of the model's output belongs to this class |\n", + "| `instruction` | path to the JSON file defining landmarks and measurements |\n", + "\n", + "`num_predefined_points` and `index_range` are fixed by the training data (DeepFashion2),\n", + "so change them only if you retrain the detection model. The `instruction` file is yours to\n", + "replace, and that is what this tutorial does." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(json.dumps(garment_classes, indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default a skirt has three measurements: `waist`, `full length`, and\n", + "`hips`. Suppose you only want `waist` and `hips`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(garment_classes[\"skirt\"][\"instruction\"]) as fh:\n", + " default_skirt = json.load(fh)\n", + "\n", + "print(json.dumps(default_skirt, indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Write a custom instruction\n", + "\n", + "An instruction has two parts. `landmarks` lists the points, each keyed by its landmark ID,\n", + "and `measurements` names each measurement by the pair of landmark IDs it spans.\n", + "\n", + "The IDs must match the ones the detection model predicts for that class, which is why the\n", + "`x` and `y` values below are only placeholders: detection overwrites them at run time.\n", + "Points marked `predefined: True` come from the model, while `predefined: False` points are\n", + "computed by the derivation step.\n", + "\n", + "> You can generate instructions visually with the\n", + "> [GarmentIQ instruction generation tool](https://garmentiq.ly.gd.edu.kg/application/demo/instruction-generation/)\n", + "> instead of writing the JSON by hand." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "skirt_new = {\n", + " \"skirt\": {\n", + " \"landmarks\": {\n", + " \"1\": {\n", + " \"predefined\": True,\n", + " \"description\": \"waist_left\",\n", + " \"x\": 60,\n", + " \"y\": 40,\n", + " },\n", + " \"3\": {\n", + " \"predefined\": True,\n", + " \"description\": \"waist_right\",\n", + " \"x\": 140,\n", + " \"y\": 40,\n", + " },\n", + " \"4\": {\n", + " \"predefined\": True,\n", + " \"description\": \"side_seam_left\",\n", + " \"x\": 50,\n", + " \"y\": 80,\n", + " },\n", + " \"8\": {\n", + " \"predefined\": True,\n", + " \"description\": \"side_seam_right\",\n", + " \"x\": 150,\n", + " \"y\": 80,\n", + " },\n", + " },\n", + " \"measurements\": {\n", + " \"waist\": {\n", + " \"landmarks\": {\"start\": \"1\", \"end\": \"3\"},\n", + " \"description\": \"/\",\n", + " },\n", + " \"hips\": {\n", + " \"landmarks\": {\"start\": \"4\", \"end\": \"8\"},\n", + " \"description\": \"/\",\n", + " },\n", + " },\n", + " }\n", + "}\n", + "\n", + "with open(\"skirt_new.json\", \"w\") as fh:\n", + " json.dump(skirt_new, fh, indent=4)\n", + "\n", + "print(\"Wrote skirt_new.json\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Register the instruction\n", + "\n", + "Point a garment class at the new file. Copy the dictionary rather than editing\n", + "`garment_classes` in place, so the defaults stay available for comparison." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_garment_classes = copy.deepcopy(garment_classes)\n", + "new_garment_classes[\"skirt\"][\"instruction\"] = \"skirt_new.json\"\n", + "\n", + "print(new_garment_classes[\"skirt\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Compare the results\n", + "\n", + "Run detection twice on the same image, changing only `class_dict`. Everything else, the\n", + "model and all its arguments, stays identical." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giq.landmark.plot(image_path=\"./test_image/cloth_4.jpg\", figsize=(3, 3))\n", + "\n", + "HRNet = giq.landmark.detection.load_model(\n", + " model_path=\"./models/hrnet.pth\",\n", + " model_class=PoseHighResolutionNet(),\n", + " device=device,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The default instruction\n", + "_, _, detection_dict = giq.landmark.detect(\n", + " class_name=\"skirt\",\n", + " class_dict=garment_classes,\n", + " image_path=\"./test_image/cloth_4.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " device=device,\n", + ")\n", + "\n", + "# `clean_detection_dict` trims the record down to the measurements themselves\n", + "default_result = giq.utils.clean_detection_dict(\n", + " class_name=\"skirt\",\n", + " image_name=\"cloth_4.jpg\",\n", + " detection_dict=detection_dict,\n", + ")\n", + "\n", + "print(\"Default measurements:\", list(default_result[\"cloth_4.jpg\"][\"measurements\"]))\n", + "default_result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The custom instruction. Only `class_dict` changes.\n", + "_, _, detection_dict_new = giq.landmark.detect(\n", + " class_name=\"skirt\",\n", + " class_dict=new_garment_classes,\n", + " image_path=\"./test_image/cloth_4.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " device=device,\n", + ")\n", + "\n", + "custom_result = giq.utils.clean_detection_dict(\n", + " class_name=\"skirt\",\n", + " image_name=\"cloth_4.jpg\",\n", + " detection_dict=detection_dict_new,\n", + ")\n", + "\n", + "print(\"Custom measurements:\", list(custom_result[\"cloth_4.jpg\"][\"measurements\"]))\n", + "custom_result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The custom instruction returns only `waist` and `hips`, as intended.\n", + "\n", + "`new_garment_classes` can be passed anywhere a class dictionary is accepted, including the\n", + "`class_dict` argument of the `tailor` pipeline, so a custom instruction flows through to\n", + "the final measurements. See the\n", + "[tailor tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_tailor.ipynb)." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb b/working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb deleted file mode 100644 index 6abe03e..0000000 --- a/working/examples/classification_model_fine_tuning_advanced_usage/classification_model_fine_tuning_advanced_usage.ipynb +++ /dev/null @@ -1,178 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "uTYHtjS_NzhK" - }, - "source": [ - "# GarmentIQ Classification Model Fine-tuning - Advanced Usage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "L7eOzqrGPvLp" - }, - "outputs": [], - "source": [ - "import garmentiq as giq\n", - "from garmentiq.classification.model_definition import tinyViT\n", - "from garmentiq.classification.utils import CachedDataset\n", - "import torch.optim as optim" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QIb-L3XU9KJn" - }, - "outputs": [], - "source": [ - "# Download fine-tuning data\n", - "# To train a model using GarmentIQ framework, your data must be in a zip file\n", - "# and the zip file should have the same structure as our data. See the link:\n", - "# https://www.kaggle.com/datasets/lygitdata/zara-clothes-image-data\n", - "!curl -L -o /app/working/examples/classification_model_fine_tuning_advanced_usage/zara-clothes-image-data.zip\\\n", - " https://www.kaggle.com/api/v1/datasets/download/lygitdata/zara-clothes-image-data\n", - "\n", - "# Download the base model - tinyViT - to be finetuned\n", - "!mkdir -p /app/working/examples/classification_model_fine_tuning_advanced_usage/models\n", - "!wget -q -O /app/working/examples/classification_model_fine_tuning_advanced_usage/models/tiny_vit.pt \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit.pt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "col7KU5vOFt6" - }, - "outputs": [], - "source": [ - "# Prepare the data for fine-tuning\n", - "# As our data size is small, we make the testing set to be 0%\n", - "# You can see from the test set summary that the size is 0\n", - "data = giq.classification.train_test_split(\n", - " output_dir=\"/app/working/examples/classification_model_fine_tuning_advanced_usage/data\",\n", - " train_zip_dir=\"/app/working/examples/classification_model_fine_tuning_advanced_usage/zara-clothes-image-data.zip\",\n", - " metadata_csv=\"metadata.csv\",\n", - " label_column=\"garment\",\n", - " test_size=0,\n", - " verbose=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "CcS54pI1QrJx" - }, - "outputs": [], - "source": [ - "# Load the training set into memory for faster I/O during training\n", - "train_images, train_labels, _ = giq.classification.load_data(\n", - " df=data[\"train_metadata\"],\n", - " img_dir=data[\"train_images\"],\n", - " label_column=\"garment\",\n", - " resize_dim=(120, 184),\n", - " normalize_mean=[0.8047, 0.7808, 0.7769],\n", - " normalize_std=[0.2957, 0.3077, 0.3081]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "zTfBwce_IH1r" - }, - "outputs": [], - "source": [ - "# Fine-tune the pretrained tinyViT model\n", - "# For demonstration purpose, we only use 5 folds and 5 epochs\n", - "# Models are saved at the folder `finetuned_models`\n", - "# It automatically selects the model with the lowest cross entropy\n", - "# as the best model\n", - "giq.classification.fine_tune_pytorch_nn(\n", - " model_class=tinyViT,\n", - " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", - " dataset_class=CachedDataset,\n", - " dataset_args={\n", - " \"metadata_df\": data[\"train_metadata\"],\n", - " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", - " \"cached_images\": train_images,\n", - " \"cached_labels\": train_labels,\n", - " },\n", - " param={\n", - " \"pretrained_path\": \"/app/working/examples/classification_model_fine_tuning_advanced_usage/models/tiny_vit.pt\",\n", - " \"freeze_layers\": True,\n", - " \"unfreeze_patterns\": [\"classifier\", \"fc\"],\n", - " \"optimizer_class\": optim.AdamW,\n", - " \"optimizer_args\": {\"lr\": 0.00002, \"weight_decay\": 1e-4},\n", - " \"n_fold\": 5,\n", - " \"n_epoch\": 5,\n", - " \"patience\": 2,\n", - " \"batch_size\": 128,\n", - " \"model_save_dir\": \"/app/working/examples/classification_model_fine_tuning_advanced_usage/finetuned_models\",\n", - " \"best_model_name\": \"best_finetuned.pt\"\n", - " },\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "E9y2RIbsJenJ" - }, - "outputs": [], - "source": [ - "# See the performance of the finetuned model on the fine-tuning dataset\n", - "giq.classification.test_pytorch_nn(\n", - " model_path=\"/app/working/examples/classification_model_fine_tuning_advanced_usage/finetuned_models/best_finetuned.pt\",\n", - " model_class=tinyViT,\n", - " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", - " dataset_class=CachedDataset,\n", - " dataset_args={\n", - " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", - " \"cached_images\": train_images,\n", - " \"cached_labels\": train_labels,\n", - " },\n", - " param={\"batch_size\": 64},\n", - ")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "authorship_tag": "ABX9TyPBgI9OQorOESyY4wReaUl+", - "gpuType": "T4", - "include_colab_link": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb b/working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb deleted file mode 100644 index e338767..0000000 --- a/working/examples/classification_model_training_evaluation_advanced_usage/classification_model_training_evaluation_advanced_usage.ipynb +++ /dev/null @@ -1,278 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "uTYHtjS_NzhK" - }, - "source": [ - "# GarmentIQ Classification Model Training & Evaluation - Advanced Usage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "L7eOzqrGPvLp" - }, - "outputs": [], - "source": [ - "import garmentiq as giq\n", - "from garmentiq.classification.model_definition import CNN3\n", - "from garmentiq.classification.utils import CachedDataset\n", - "import torch.optim as optim\n", - "import torch.nn as nn" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QIb-L3XU9KJn" - }, - "outputs": [], - "source": [ - "# Download training data\n", - "# To train a model using GarmentIQ framework, your data must be in a zip file\n", - "# and the zip file should have the same structure as our data. See the link:\n", - "# https://www.kaggle.com/datasets/lygitdata/garmentiq-classification-set-nordstrom-and-myntra\n", - "!curl -L -o /app/working/examples/classification_model_training_evaluation_advanced_usage/garmentiq-classification-set-nordstrom-and-myntra.zip \\\n", - " https://www.kaggle.com/api/v1/datasets/download/lygitdata/garmentiq-classification-set-nordstrom-and-myntra" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "col7KU5vOFt6" - }, - "outputs": [], - "source": [ - "# Split the data into training set and testing set\n", - "data = giq.classification.train_test_split(\n", - " output_dir=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/data\",\n", - " train_zip_dir=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/garmentiq-classification-set-nordstrom-and-myntra.zip\",\n", - " metadata_csv=\"metadata.csv\",\n", - " label_column=\"garment\",\n", - " test_size=0.15,\n", - " verbose=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "CcS54pI1QrJx" - }, - "outputs": [], - "source": [ - "# Load the training set into memory for faster I/O during training\n", - "train_images, train_labels, _ = giq.classification.load_data(\n", - " df=data[\"train_metadata\"],\n", - " img_dir=data[\"train_images\"],\n", - " label_column=\"garment\",\n", - " resize_dim=(60, 92),\n", - " normalize_mean=[0.8047, 0.7808, 0.7769],\n", - " normalize_std=[0.2957, 0.3077, 0.3081]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZWzfw235-nG8" - }, - "outputs": [], - "source": [ - "# Train with GarmentIQ predefined model - CNN3\n", - "# Note that we use `model_class=CNN3` here\n", - "# For demonstration purpose, we only use 2 folds and 5 epochs\n", - "# Models are saved at the folder `cnn3_models`\n", - "# It automatically selects the model with the lowest cross entropy\n", - "# as the best model\n", - "giq.classification.train_pytorch_nn(\n", - " model_class=CNN3,\n", - " model_args={\"num_classes\": 9},\n", - " dataset_class=CachedDataset,\n", - " dataset_args={\n", - " \"metadata_df\": data[\"train_metadata\"],\n", - " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", - " \"cached_images\": train_images,\n", - " \"cached_labels\": train_labels,\n", - " },\n", - " param={\n", - " \"optimizer_class\": optim.AdamW,\n", - " \"optimizer_args\": {\"lr\": 0.001, \"weight_decay\": 1e-4},\n", - " \"n_fold\": 2,\n", - " \"n_epoch\": 5,\n", - " \"patience\": 2,\n", - " \"batch_size\": 256,\n", - " \"model_save_dir\": \"/app/working/examples/classification_model_training_evaluation_advanced_usage/cnn3_models\",\n", - " \"best_model_name\": \"best_cnn3_model.pt\",\n", - " },\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jxoTUMM5ADAR" - }, - "outputs": [], - "source": [ - "# Train with a user-defined model\n", - "class UserDefinedCNN(nn.Module):\n", - " def __init__(self, num_classes):\n", - " super(UserDefinedCNN, self).__init__()\n", - " self.features = nn.Sequential(\n", - " nn.Conv2d(3, 32, kernel_size=3, padding=1),\n", - " nn.BatchNorm2d(32),\n", - " nn.ReLU(inplace=True),\n", - " nn.MaxPool2d(2, 2),\n", - " nn.Dropout(0.2),\n", - " nn.Conv2d(32, 64, kernel_size=3, padding=1),\n", - " nn.BatchNorm2d(64),\n", - " nn.ReLU(inplace=True),\n", - " nn.MaxPool2d(2, 2),\n", - " nn.Dropout(0.25),\n", - " nn.AdaptiveAvgPool2d((4, 4)),\n", - " )\n", - " self.classifier = nn.Sequential(\n", - " nn.Linear(64 * 4 * 4, 128),\n", - " nn.ReLU(inplace=True),\n", - " nn.Dropout(0.3),\n", - " nn.Linear(128, num_classes),\n", - " )\n", - " def forward(self, x):\n", - " x = self.features(x)\n", - " x = x.view(x.size(0), -1)\n", - " x = self.classifier(x)\n", - " return x\n", - "\n", - "# For demonstration purpose, we only use 2 folds and 5 epochs\n", - "# Models are saved at the folder `cnn1_models`\n", - "# It automatically selects the model with the lowest cross entropy\n", - "# as the best model\n", - "giq.classification.train_pytorch_nn(\n", - " model_class=UserDefinedCNN,\n", - " model_args={\"num_classes\": 9},\n", - " dataset_class=CachedDataset,\n", - " dataset_args={\n", - " \"metadata_df\": data[\"train_metadata\"],\n", - " \"raw_labels\": data[\"train_metadata\"][\"garment\"],\n", - " \"cached_images\": train_images,\n", - " \"cached_labels\": train_labels,\n", - " },\n", - " param={\n", - " \"optimizer_class\": optim.AdamW,\n", - " \"optimizer_args\": {\"lr\": 0.001, \"weight_decay\": 1e-4},\n", - " \"n_fold\": 2,\n", - " \"n_epoch\": 5,\n", - " \"patience\": 2,\n", - " \"batch_size\": 256,\n", - " \"model_save_dir\": \"/app/working/examples/classification_model_training_evaluation_advanced_usage/userdefined_cnn_models\",\n", - " \"best_model_name\": \"best_userdefined_cnn_model.pt\",\n", - " },\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "4SJBuwwj_csd" - }, - "outputs": [], - "source": [ - "# Load the testing set for model evaluation\n", - "test_images, test_labels, _ = giq.classification.load_data(\n", - " df=data[\"test_metadata\"],\n", - " img_dir=data[\"test_images\"],\n", - " label_column=\"garment\",\n", - " resize_dim=(60, 92),\n", - " normalize_mean=[0.8047, 0.7808, 0.7769],\n", - " normalize_std=[0.2957, 0.3077, 0.3081]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "MsJVxrcgCkjx" - }, - "outputs": [], - "source": [ - "# Evaluate CNN3 model on the testing set\n", - "giq.classification.test_pytorch_nn(\n", - " model_path=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/cnn3_models/best_cnn3_model.pt\",\n", - " model_class=CNN3,\n", - " model_args={\"num_classes\": 9},\n", - " dataset_class=CachedDataset,\n", - " dataset_args={\n", - " \"raw_labels\": data[\"test_metadata\"][\"garment\"],\n", - " \"cached_images\": test_images,\n", - " \"cached_labels\": test_labels,\n", - " },\n", - " param={\"batch_size\": 64},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_zeZcebZCsCp" - }, - "outputs": [], - "source": [ - "# Evaluate user-defined model on the testing set\n", - "# We can see that the CNN3 model performs better on the test set\n", - "# than this user-defined CNN model\n", - "giq.classification.test_pytorch_nn(\n", - " model_path=\"/app/working/examples/classification_model_training_evaluation_advanced_usage/userdefined_cnn_models/best_userdefined_cnn_model.pt\",\n", - " model_class=UserDefinedCNN,\n", - " model_args={\"num_classes\": 9},\n", - " dataset_class=CachedDataset,\n", - " dataset_args={\n", - " \"raw_labels\": data[\"test_metadata\"][\"garment\"],\n", - " \"cached_images\": test_images,\n", - " \"cached_labels\": test_labels,\n", - " },\n", - " param={\"batch_size\": 64},\n", - ")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "authorship_tag": "ABX9TyM86uuEAMztFXJELwGY11Hc", - "gpuType": "T4", - "include_colab_link": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/classification_quick_start/classification_quick_start.ipynb b/working/examples/classification_quick_start/classification_quick_start.ipynb deleted file mode 100644 index cb72f80..0000000 --- a/working/examples/classification_quick_start/classification_quick_start.ipynb +++ /dev/null @@ -1,163 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "jrn-fD7chMUe" - }, - "source": [ - "# GarmentIQ Classification Quick Start" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "E8RxkKlsh5IY" - }, - "outputs": [], - "source": [ - "import garmentiq as giq\n", - "from garmentiq.classification.model_definition import tinyViT\n", - "from garmentiq.classification.utils import CachedDataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3-b9AZr9hJyK" - }, - "outputs": [], - "source": [ - "# Download test data and a pretrained model\n", - "!mkdir -p /app/working/examples/classification_quick_start/models\n", - "\n", - "!curl -L -o /app/working/examples/classification_quick_start/garmentiq-classification-set-nordstrom-and-myntra.zip \\\n", - " https://www.kaggle.com/api/v1/datasets/download/lygitdata/garmentiq-classification-set-nordstrom-and-myntra\n", - "\n", - "!wget -q -O /app/working/examples/classification_quick_start/models/tiny_vit.pt \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit.pt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "aW3E9wDbivLQ" - }, - "outputs": [], - "source": [ - "# Prepare test data garmentiq-classification-set-nordstrom-and-myntra.zip\n", - "DATA = giq.classification.train_test_split(\n", - " output_dir=\"/app/working/examples/classification_quick_start/data\",\n", - " metadata_csv=\"metadata.csv\",\n", - " label_column=\"garment\",\n", - " train_zip_dir=\"/app/working/examples/classification_quick_start/garmentiq-classification-set-nordstrom-and-myntra.zip\",\n", - " test_size=0.15,\n", - " verbose=True\n", - ")\n", - "\n", - "test_images, test_labels, _ = giq.classification.load_data(\n", - " df=DATA[\"test_metadata\"],\n", - " img_dir=DATA[\"test_images\"],\n", - " label_column=\"garment\",\n", - " resize_dim=(120, 184),\n", - " normalize_mean=[0.8047, 0.7808, 0.7769],\n", - " normalize_std=[0.2957, 0.3077, 0.3081]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load the pretrained model\n", - "classifier = giq.classification.load_model(\n", - " model_path=\"/app/working/examples/classification_quick_start/models/tiny_vit.pt\",\n", - " model_class=tinyViT,\n", - " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6}\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "hfXjYs-mkE4s" - }, - "outputs": [], - "source": [ - "# Fit the model on the whole test data\n", - "giq.classification.test_pytorch_nn(\n", - " model_path=\"/app/working/examples/classification_quick_start/models/tiny_vit.pt\",\n", - " model_class=tinyViT,\n", - " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", - " dataset_class=CachedDataset,\n", - " dataset_args={\n", - " \"raw_labels\": DATA[\"test_metadata\"][\"garment\"],\n", - " \"cached_images\": test_images,\n", - " \"cached_labels\": test_labels,\n", - " },\n", - " param={\"batch_size\": 64},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "vJ6kl9rTkerA" - }, - "outputs": [], - "source": [ - "# Fit the model on a single image\n", - "img_to_test = DATA['test_metadata']['filename'][88]\n", - "\n", - "pred_label, pred_prob = giq.classification.predict(\n", - " model=classifier,\n", - " image_path=f\"/app/working/examples/classification_quick_start/data/test/images/{img_to_test}\",\n", - " classes=DATA['test_metadata']['garment'].unique().tolist(),\n", - " resize_dim=(120, 184),\n", - " normalize_mean=[0.8047, 0.7808, 0.7769],\n", - " normalize_std=[0.2957, 0.3077, 0.3081]\n", - ")\n", - "\n", - "print(\n", - " \"True label: \", img_to_test,\n", - " \"\\nPredicted label: \", pred_label,\n", - " \"\\nPredicted Probabilities: \", pred_prob\n", - ")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "include_colab_link": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb b/working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb deleted file mode 100644 index 7b43cee..0000000 --- a/working/examples/custom_measurement_instruction_advanced_usage/custom_measurement_instruction_advanced_usage.ipynb +++ /dev/null @@ -1,278 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "uTYHtjS_NzhK" - }, - "source": [ - "# GarmentIQ Custom Measurement Instruction - Advanced Usage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "L7eOzqrGPvLp" - }, - "outputs": [], - "source": [ - "import json\n", - "import copy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "col7KU5vOFt6" - }, - "outputs": [], - "source": [ - "# See our predefined garments and measurement instructions\n", - "#\n", - "# - `num_predefeined_points` and `index_range` are determined by training data -\n", - "# DeepFashion2, change them if it is indeed necessary.\n", - "#\n", - "# - `instruction` is the path to a .json file, GarmentQI has predefined\n", - "# instructions.\n", - "#\n", - "# Generate or see predefined instructions at:\n", - "# https://garmentiq.ly.gd.edu.kg/application/demo/instruction-generation/\n", - "from garmentiq.garment_classes import garment_classes\n", - "print(json.dumps(garment_classes, indent=4))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "CcS54pI1QrJx" - }, - "outputs": [], - "source": [ - "# Let us change the measurement instruction of skirt\n", - "# See the predefined instruction\n", - "# There are `waist`, `full length`, and `hips` as measurements in GarmentIQ's\n", - "# default measurement instruction of a skirt. What if we only want `waist` and\n", - "# `hips`? Simply create one! Recommend to use GarmentIQ instruction generation\n", - "# tool to generate the instruciton:\n", - "# https://garmentiq.ly.gd.edu.kg/application/demo/instruction-generation/\n", - "with open(garment_classes['skirt']['instruction'], 'r') as file:\n", - " data = json.load(file)\n", - "print(json.dumps(data, indent=4))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "gB1AxdinRqvR" - }, - "outputs": [], - "source": [ - "# Let us create a simple instruction in python here and convert it to json\n", - "skirt_new = {\n", - " \"skirt\": {\n", - " \"landmarks\": {\n", - " \"1\": {\n", - " \"predefined\": True,\n", - " \"description\": \"waist_left\",\n", - " \"x\": 60,\n", - " \"y\": 40\n", - " },\n", - " \"3\": {\n", - " \"predefined\": True,\n", - " \"description\": \"waist_right\",\n", - " \"x\": 140,\n", - " \"y\": 40\n", - " },\n", - " \"4\": {\n", - " \"predefined\": True,\n", - " \"description\": \"side_seam_left\",\n", - " \"x\": 50,\n", - " \"y\": 80\n", - " },\n", - " \"8\": {\n", - " \"predefined\": True,\n", - " \"description\": \"side_seam_right\",\n", - " \"x\": 150,\n", - " \"y\": 80\n", - " }\n", - " },\n", - " \"measurements\": {\n", - " \"waist\": {\n", - " \"landmarks\": {\n", - " \"start\": \"1\",\n", - " \"end\": \"3\"\n", - " },\n", - " \"description\": \"/\"\n", - " },\n", - " \"hips\": {\n", - " \"landmarks\": {\n", - " \"start\": \"4\",\n", - " \"end\": \"8\"\n", - " },\n", - " \"description\": \"/\"\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "# Save it to a JSON file with indentation\n", - "with open('skirt_new.json', 'w') as file:\n", - " json.dump(skirt_new, file, indent=4)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lb1wL06USdzn" - }, - "outputs": [], - "source": [ - "# Create a new garment class metadata dictionary\n", - "new_garment_classes = copy.deepcopy(garment_classes)\n", - "new_garment_classes['skirt']['instruction'] = 'skirt_new.json'\n", - "print(new_garment_classes['skirt'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XiAh-4I6TNXl" - }, - "outputs": [], - "source": [ - "# Okay now we have the new instruction, let us try the detection model to see\n", - "# the difference\n", - "# Import necessary modules\n", - "import garmentiq as giq\n", - "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", - "\n", - "# Download a skirt image and a pretrained model\n", - "!mkdir -p /app/working/examples/custom_measurement_instruction_advanced_usage/test_image\n", - "!wget -q -O /app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_4.jpg\n", - "\n", - "!mkdir -p /app/working/examples/custom_measurement_instruction_advanced_usage/models\n", - "!wget -q -O /app/working/examples/custom_measurement_instruction_advanced_usage/models/hrnet.pth \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "aN_ikAdETn5J" - }, - "outputs": [], - "source": [ - "# Plot the image\n", - "giq.landmark.plot(image_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg\", figsize=(3, 3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sMKutxcVTrLw" - }, - "outputs": [], - "source": [ - "# Load the pretrained model from Hugging Face\n", - "HRNet = giq.landmark.detection.load_model(\n", - " model_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/models/hrnet.pth\",\n", - " model_class=PoseHighResolutionNet()\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Frx1SAxVTu1J" - }, - "outputs": [], - "source": [ - "# Detect landmarks (DEFAULT instruction)\n", - "_, _, detection_dict = giq.landmark.detect(\n", - " class_name=\"skirt\",\n", - " class_dict=garment_classes,\n", - " image_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg\",\n", - " model=HRNet,\n", - " scale_std=200.0,\n", - " resize_dim=[288, 384],\n", - " normalize_mean=[0.485, 0.456, 0.406],\n", - " normalize_std=[0.229, 0.224, 0.225]\n", - ")\n", - "\n", - "# Clean the detection dictionary\n", - "# See that we have all the three measurements\n", - "detection_dict_cleaned = giq.utils.clean_detection_dict(\n", - " class_name='skirt',\n", - " image_name='cloth_4.jpg',\n", - " detection_dict=detection_dict\n", - ")\n", - "detection_dict_cleaned" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "R34QuDmYT_X0" - }, - "outputs": [], - "source": [ - "# Detect landmarks (NEW instruction)\n", - "# Note that we use `class_dict=new_garment_classes` here\n", - "_, _, detection_dict_new = giq.landmark.detect(\n", - " class_name=\"skirt\",\n", - " class_dict=new_garment_classes,\n", - " image_path=\"/app/working/examples/custom_measurement_instruction_advanced_usage/test_image/cloth_4.jpg\",\n", - " model=HRNet,\n", - " scale_std=200.0,\n", - " resize_dim=[288, 384],\n", - " normalize_mean=[0.485, 0.456, 0.406],\n", - " normalize_std=[0.229, 0.224, 0.225]\n", - ")\n", - "\n", - "# Clean the detection dictionary\n", - "# See that we only have two measurements we want\n", - "detection_dict_new_cleaned = giq.utils.clean_detection_dict(\n", - " class_name='skirt',\n", - " image_name='cloth_4.jpg',\n", - " detection_dict=detection_dict_new\n", - ")\n", - "detection_dict_new_cleaned" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb b/working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb deleted file mode 100644 index 14b3419..0000000 --- a/working/examples/landmark_detection_quick_start/landmark_detection_quick_start.ipynb +++ /dev/null @@ -1,141 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "jrn-fD7chMUe" - }, - "source": [ - "# GarmentIQ Landmark Detection Quick Start" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "E8RxkKlsh5IY" - }, - "outputs": [], - "source": [ - "import garmentiq as giq\n", - "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", - "from garmentiq.garment_classes import garment_classes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3-b9AZr9hJyK" - }, - "outputs": [], - "source": [ - "# Download a vest dress image and a pretrained model\n", - "!mkdir -p /app/working/examples/landmark_detection_quick_start/test_image\n", - "!wget -q -O /app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", - "\n", - "!mkdir -p /app/working/examples/landmark_detection_quick_start/models\n", - "!wget -q -O /app/working/examples/landmark_detection_quick_start/models/hrnet.pth \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 337 - }, - "id": "NYFrZUe48hLi", - "outputId": "6d881cf8-75c3-46f1-acf5-c0b691b934c6" - }, - "outputs": [], - "source": [ - "# Plot the image\n", - "giq.landmark.plot(image_path=\"/app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg\", figsize=(3, 3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pMGZwL-UjPPN" - }, - "outputs": [], - "source": [ - "# Load the pretrained model from Hugging Face\n", - "HRNet = giq.landmark.detection.load_model(\n", - " model_path=\"/app/working/examples/landmark_detection_quick_start/models/hrnet.pth\",\n", - " model_class=PoseHighResolutionNet()\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "deJChHYZ7rpT" - }, - "outputs": [], - "source": [ - "# Detect predefined landmarks\n", - "coords, maxvals, detection_dict = giq.landmark.detect(\n", - " class_name=\"vest dress\",\n", - " class_dict=garment_classes,\n", - " image_path=\"/app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg\",\n", - " model=HRNet,\n", - " scale_std=200.0,\n", - " resize_dim=[288, 384],\n", - " normalize_mean=[0.485, 0.456, 0.406],\n", - " normalize_std=[0.229, 0.224, 0.225]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 337 - }, - "id": "rpKL-_2685Rd", - "outputId": "634798cc-ab48-4d4f-d7bf-2459890d9efb" - }, - "outputs": [], - "source": [ - "# Plot the detected coordinates\n", - "giq.landmark.plot(image_path=\"/app/working/examples/landmark_detection_quick_start/test_image/cloth_3.jpg\", coordinate=coords, figsize=(3, 3), color=\"green\")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "include_colab_link": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb b/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb deleted file mode 100644 index d9402bb..0000000 --- a/working/examples/landmark_refinement_and_derivation_quick_start/landmark_refinement_and_derivation_quick_start.ipynb +++ /dev/null @@ -1,213 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "jrn-fD7chMUe" - }, - "source": [ - "# GarmentIQ Landmark Detection Quick Start" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "E8RxkKlsh5IY" - }, - "outputs": [], - "source": [ - "import garmentiq as giq\n", - "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", - "from garmentiq.garment_classes import garment_classes\n", - "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Download a vest dress image\n", - "!mkdir -p test_image\n", - "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", - "\n", - "# Download HRNet model\n", - "!mkdir -p models\n", - "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/models/hrnet.pth \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", - "\n", - "# Download BiRefNet model\n", - "!mkdir -p /app/working/examples/landmark_refinement_and_derivation_quick_start/models/birefnet\n", - "!wget -q -O /app/working/examples/landmark_refinement_and_derivation_quick_start/models/birefnet/model.safetensors \\\n", - " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NYFrZUe48hLi" - }, - "outputs": [], - "source": [ - "# Plot the image\n", - "giq.landmark.plot(\n", - " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\", \n", - " figsize=(3, 3)\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pMGZwL-UjPPN" - }, - "outputs": [], - "source": [ - "# Load the pretrained model from Hugging Face\n", - "HRNet = giq.landmark.detection.load_model(\n", - " model_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/models/hrnet.pth\",\n", - " model_class=PoseHighResolutionNet()\n", - ")\n", - "\n", - "# Detect predefined landmarks\n", - "coords, maxvals, detection_dict = giq.landmark.detect(\n", - " class_name=\"vest dress\",\n", - " class_dict=garment_classes,\n", - " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\",\n", - " model=HRNet,\n", - " scale_std=200.0,\n", - " resize_dim=[288, 384],\n", - " normalize_mean=[0.485, 0.456, 0.406],\n", - " normalize_std=[0.229, 0.224, 0.225]\n", - ")\n", - "\n", - "# Plot the detected coordinates\n", - "giq.landmark.plot(\n", - " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\", \n", - " coordinate=coords, \n", - " figsize=(3, 3), \n", - " color=\"green\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "y4-zd5CusfED" - }, - "outputs": [], - "source": [ - "# Segmentation mask is required for refinement and derivation\n", - "# So we need to do segmentation first\n", - "BiRefNet = giq.segmentation.load_model(\n", - " pretrained_model='lygitdata/BiRefNet_garmentiq_backup',\n", - " pretrained_model_args={'trust_remote_code': True},\n", - " high_precision=True\n", - ")\n", - "original_img, mask = giq.segmentation.extract(\n", - " model=BiRefNet,\n", - " image_path='/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg',\n", - " resize_dim=(1024, 1024),\n", - " normalize_mean=[0.485, 0.456, 0.406],\n", - " normalize_std=[0.229, 0.224, 0.225],\n", - " high_precision=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "deJChHYZ7rpT" - }, - "outputs": [], - "source": [ - "# Refine the landmarks\n", - "refined_coords, refined_detection_dict = giq.landmark.refine(\n", - " class_name=\"vest dress\",\n", - " detection_np=coords,\n", - " detection_conf=maxvals,\n", - " detection_dict=detection_dict,\n", - " mask=mask,\n", - " window_size=5,\n", - " ksize=(11, 11),\n", - " sigmaX=0.0\n", - ")\n", - "\n", - "# Print the original coordinates and the refined coordinates\n", - "print(\"Original coordinates:\\n\", coords)\n", - "print(\"Refined coordinates:\\n\", refined_coords)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rpKL-_2685Rd" - }, - "outputs": [], - "source": [ - "# Derive custom landmarks\n", - "derived_coords, derived_detection_dict = giq.landmark.derive(\n", - " class_name=\"vest dress\",\n", - " detection_dict=refined_detection_dict,\n", - " derivation_dict=giq.landmark.derivation_dict.derivation_dict,\n", - " landmark_coords=refined_coords,\n", - " np_mask=mask\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LTskt_1KwmBG" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "# Plot the derived point\n", - "giq.landmark.plot(\n", - " image_path=\"/app/working/examples/landmark_refinement_and_derivation_quick_start/test_image/cloth_3.jpg\",\n", - " coordinate=np.concatenate((refined_coords, np.array([[derived_coords['20']]])), axis=1),\n", - " figsize=(3, 3),\n", - " color=\"green\"\n", - ")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "include_colab_link": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb b/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb deleted file mode 100644 index 7628c9f..0000000 --- a/working/examples/segmentation_quick_start/segmentation_quick_start.ipynb +++ /dev/null @@ -1,163 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "jrn-fD7chMUe" - }, - "source": [ - "# GarmentIQ Segmentation Quick Start" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "E8RxkKlsh5IY" - }, - "outputs": [], - "source": [ - "import garmentiq as giq\n", - "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config\n", - "from garmentiq.segmentation.model_definition.sam import SamModel, load_sam_config, load_sam_processor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Download a test image\n", - "!mkdir -p /app/working/examples/segmentation_quick_start/test_image\n", - "!wget -q -O /app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg\n", - "\n", - "# Download BiRefNet model\n", - "!mkdir -p /app/working/examples/segmentation_quick_start/models/birefnet\n", - "\n", - "!wget -q -O /app/working/examples/segmentation_quick_start/models/birefnet/model.safetensors \\\n", - " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", - "\n", - "# Download SAM base model\n", - "!mkdir -p /app/working/examples/segmentation_quick_start/models/sam_b\n", - "\n", - "!wget -q -O /app/working/examples/segmentation_quick_start/models/sam_b/model.safetensors \\\n", - " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example with BiRefNet model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pMGZwL-UjPPN" - }, - "outputs": [], - "source": [ - "# Load the BiRefNet model\n", - "birefnet = giq.segmentation.load_model(\n", - " model_class=BiRefNet,\n", - " model_path=\"/app/working/examples/segmentation_quick_start/models/birefnet/model.safetensors\",\n", - " model_args=load_birefnet_config()\n", - ")\n", - "\n", - "# Extract the mask\n", - "original_img_biref, mask_biref = giq.segmentation.extract(\n", - " model=birefnet,\n", - " image_path='/app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg',\n", - " # BiRefNet specific kwargs:\n", - " resize_dim=(1024, 1024),\n", - " normalize_mean=[0.485, 0.456, 0.406],\n", - " normalize_std=[0.229, 0.224, 0.225]\n", - ")\n", - "\n", - "# Change background color\n", - "bg_modified_img_biref = giq.segmentation.change_background_color(\n", - " image_np=original_img_biref,\n", - " mask_np=mask_biref,\n", - " background_color=[102, 255, 102]\n", - ")\n", - "\n", - "# Plot the original image, mask, and background modified image\n", - "giq.segmentation.plot(image_np=original_img_biref, figsize=(3, 3))\n", - "giq.segmentation.plot(image_np=mask_biref, figsize=(3, 3))\n", - "giq.segmentation.plot(image_np=bg_modified_img_biref, figsize=(3, 3))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example with SAM base model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load the BiRefNet model\n", - "sam = giq.segmentation.load_model(\n", - " model_class=SamModel,\n", - " model_path=\"/app/working/examples/segmentation_quick_start/models/sam_b/model.safetensors\",\n", - " model_args={\"config\": load_sam_config(\"sam-vit-b\")}\n", - ")\n", - "\n", - "# Extract the mask\n", - "original_img_sam, mask_sam = giq.segmentation.extract(\n", - " model=sam,\n", - " image_path='/app/working/examples/segmentation_quick_start/test_image/cloth_1.jpg',\n", - " processor=load_sam_processor(\"sam-vit-b\"),\n", - " input_points=[[[512, 512]]]\n", - ")\n", - "\n", - "# Change background color\n", - "bg_modified_img_sam = giq.segmentation.change_background_color(\n", - " image_np=original_img_sam,\n", - " mask_np=mask_sam,\n", - " background_color=[102, 255, 102]\n", - ")\n", - "\n", - "# Plot the original image, mask, and background modified image\n", - "giq.segmentation.plot(image_np=original_img_sam, figsize=(3, 3))\n", - "giq.segmentation.plot(image_np=mask_sam, figsize=(3, 3))\n", - "giq.segmentation.plot(image_np=bg_modified_img_sam, figsize=(3, 3))" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "include_colab_link": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/tailor_quick_start/tailor_quick_start.ipynb b/working/examples/tailor_quick_start/tailor_quick_start.ipynb deleted file mode 100644 index 3c80a62..0000000 --- a/working/examples/tailor_quick_start/tailor_quick_start.ipynb +++ /dev/null @@ -1,367 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "jrn-fD7chMUe" - }, - "source": [ - "# GarmentIQ Tailor Quick Start" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "E8RxkKlsh5IY" - }, - "outputs": [], - "source": [ - "import garmentiq as giq\n", - "from garmentiq.classification.model_definition import tinyViT\n", - "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", - "from garmentiq.garment_classes import garment_classes\n", - "from garmentiq.landmark.derivation.derivation_dict import derivation_dict\n", - "from garmentiq.segmentation.model_definition.birefnet import BiRefNet, load_birefnet_config\n", - "from garmentiq.segmentation.model_definition.sam import SamModel, load_sam_config, load_sam_processor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Download 3 test images\n", - "# cloth_2 is short sleeve tops, cloth_3 is vest dress, cloth_4 is skirt\n", - "!mkdir -p /app/working/examples/tailor_quick_start/test_image\n", - "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_2.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_2.jpg\n", - "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_3.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", - "!wget -q -O /app/working/examples/tailor_quick_start/test_image/cloth_4.jpg \\\n", - " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_4.jpg\n", - "\n", - "# Download the classification model\n", - "!mkdir -p models\n", - "!wget -q -O /app/working/examples/tailor_quick_start/models/tiny_vit_inditex_finetuned.pt \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit_inditex_finetuned.pt\n", - "\n", - "# Download the landmark detection model\n", - "!wget -q -O /app/working/examples/tailor_quick_start/models/hrnet.pth \\\n", - " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", - "\n", - "# Download BiRefNet model\n", - "!mkdir -p models/birefnet\n", - "!wget -q -O /app/working/examples/tailor_quick_start/models/birefnet/model.safetensors \\\n", - " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", - "\n", - "# Download SAM base model\n", - "!mkdir -p models/sam_b\n", - "!wget -q -O /app/working/examples/tailor_quick_start/models/sam_b/model.safetensors \\\n", - " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example with BiRefNet model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NYFrZUe48hLi" - }, - "outputs": [], - "source": [ - "# Setup the tailor agent\n", - "tailor_biref = giq.tailor(\n", - " input_dir=\"/app/working/examples/tailor_quick_start/test_image\",\n", - " model_dir=\"/app/working/examples/tailor_quick_start/models\",\n", - " output_dir=\"/app/working/examples/tailor_quick_start/output_biref\",\n", - " class_dict=garment_classes,\n", - " do_derive=True,\n", - " derivation_dict=derivation_dict,\n", - " do_refine=True,\n", - " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", - " classification_model_class=tinyViT,\n", - " classification_model_args={\n", - " \"num_classes\": len(list(garment_classes.keys())),\n", - " \"img_size\": (120, 184),\n", - " \"patch_size\": 6,\n", - " \"resize_dim\": (120, 184),\n", - " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", - " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", - " },\n", - " segmentation_model_path=\"birefnet/model.safetensors\",\n", - " segmentation_model_class=BiRefNet,\n", - " segmentation_model_args={\n", - " \"model_config\": load_birefnet_config(),\n", - " \"resize_dim\": (1024, 1024),\n", - " \"normalize_mean\": [0.485, 0.456, 0.406],\n", - " \"normalize_std\": [0.229, 0.224, 0.225],\n", - " \"background_color\": [102, 255, 102]\n", - " },\n", - " landmark_detection_model_path=\"hrnet.pth\",\n", - " landmark_detection_model_class=PoseHighResolutionNet(),\n", - " landmark_detection_model_args={\n", - " \"scale_std\": 200.0,\n", - " \"resize_dim\": [288, 384],\n", - " \"normalize_mean\": [0.485, 0.456, 0.406],\n", - " \"normalize_std\": [0.229, 0.224, 0.225],\n", - " },\n", - ")\n", - "\n", - "# See the tailor agent's basic information\n", - "tailor_biref.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "F3_WQnWmCLY2" - }, - "outputs": [], - "source": [ - "# Start the measurement with refinement and derivation\n", - "metadata, outputs = tailor_biref.measure(save_segmentation_image=True, save_measurement_image=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "-spavypXE5xb" - }, - "outputs": [], - "source": [ - "# See the metadata\n", - "# It makes file access much easier\n", - "print(metadata)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XoK9vs-tERj-", - "scrolled": true - }, - "outputs": [], - "source": [ - "# Plot the masks\n", - "# Go to /app/working/examples/tailor_quick_start/output_biref/mask_image/ to see the high resolution images\n", - "for image in metadata['mask_image']:\n", - " giq.landmark.plot(image_path=image, figsize=(3, 3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LartOXSkEruQ" - }, - "outputs": [], - "source": [ - "# Plot the background modified images\n", - "# Go to /app/working/examples/tailor_quick_start/output_biref/bg_modified_image to see the high resolution images\n", - "for image in metadata['bg_modified_image']:\n", - " giq.landmark.plot(image_path=image, figsize=(3, 3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "HF7z8tZtDAz8" - }, - "outputs": [], - "source": [ - "# Plot the images with desired landmarks\n", - "# Go to /app/working/examples/tailor_quick_start/output_biref/measurement_image/ to see the high resolution images\n", - "for image in metadata['measurement_image']:\n", - " giq.landmark.plot(image_path=image, figsize=(3, 3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jbLD40U5ELe9" - }, - "outputs": [], - "source": [ - "# See the measurement results in JSON format\n", - "# Go to /app/working/examples/tailor_quick_start/output_biref/measurement_json/ to see the JSON files\n", - "import json\n", - "\n", - "for json_path in metadata['measurement_json']:\n", - " with open(json_path, 'r') as file:\n", - " data = json.load(file)\n", - " print(f\"{json_path}:\\n\")\n", - " print(json.dumps(data, indent=4, sort_keys=True))\n", - " print(\"\\n\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example with SAM base model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Setup the tailor agent\n", - "tailor_sam = giq.tailor(\n", - " input_dir=\"/app/working/examples/tailor_quick_start/test_image\",\n", - " model_dir=\"/app/working/examples/tailor_quick_start/models\",\n", - " output_dir=\"/app/working/examples/tailor_quick_start/output_sam\",\n", - " class_dict=garment_classes,\n", - " do_derive=True,\n", - " derivation_dict=derivation_dict,\n", - " do_refine=False,\n", - " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", - " classification_model_class=tinyViT,\n", - " classification_model_args={\n", - " \"num_classes\": len(list(garment_classes.keys())),\n", - " \"img_size\": (120, 184),\n", - " \"patch_size\": 6,\n", - " \"resize_dim\": (120, 184),\n", - " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", - " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", - " },\n", - " segmentation_model_path=\"sam_b/model.safetensors\",\n", - " segmentation_model_class=SamModel,\n", - " segmentation_model_args={\n", - " \"model_config\": {\"config\": load_sam_config(\"sam-vit-b\")},\n", - " \"processor\": load_sam_processor(\"sam-vit-b\", use_fast=False),\n", - " \"input_points\": [[[1000, 900]]],\n", - " \"background_color\": [102, 255, 102]\n", - " },\n", - " landmark_detection_model_path=\"hrnet.pth\",\n", - " landmark_detection_model_class=PoseHighResolutionNet(),\n", - " landmark_detection_model_args={\n", - " \"scale_std\": 200.0,\n", - " \"resize_dim\": [288, 384],\n", - " \"normalize_mean\": [0.485, 0.456, 0.406],\n", - " \"normalize_std\": [0.229, 0.224, 0.225],\n", - " },\n", - ")\n", - "\n", - "# See the tailor agent's basic information\n", - "tailor_sam.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Start the measurement with refinement and derivation\n", - "metadata, outputs = tailor_sam.measure(save_segmentation_image=True, save_measurement_image=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# See the metadata\n", - "# It makes file access much easier\n", - "print(metadata)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot the masks\n", - "# Go to /content/output_sam/mask_image/ to see the high resolution images\n", - "for image in metadata['mask_image']:\n", - " giq.landmark.plot(image_path=image, figsize=(3, 3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot the background modified images\n", - "# Go to /content/output_sam/bg_modified_image to see the high resolution images\n", - "for image in metadata['bg_modified_image']:\n", - " giq.landmark.plot(image_path=image, figsize=(3, 3)) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot the images with desired landmarks\n", - "# Go to /content/output_sam/measurement_image/ to see the high resolution images\n", - "for image in metadata['measurement_image']:\n", - " giq.landmark.plot(image_path=image, figsize=(3, 3))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# See the measurement results in JSON format\n", - "# Go to /content/output_sam/measurement_json/ to see the JSON files\n", - "import json\n", - "\n", - "for json_path in metadata['measurement_json']:\n", - " with open(json_path, 'r') as file:\n", - " data = json.load(file)\n", - " print(f\"{json_path}:\\n\")\n", - " print(json.dumps(data, indent=4, sort_keys=True))\n", - " print(\"\\n\\n\")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "include_colab_link": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/working/examples/tutorial_classification/tutorial_classification.ipynb b/working/examples/tutorial_classification/tutorial_classification.ipynb new file mode 100644 index 0000000..afb92cf --- /dev/null +++ b/working/examples/tutorial_classification/tutorial_classification.ipynb @@ -0,0 +1,246 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - GarmentIQ Classification\n", + "\n", + "GarmentIQ classification identifies which category a garment image belongs to, such as a\n", + "short sleeve top, a vest dress, or a skirt. It is the first step of the measurement\n", + "pipeline, because every later stage needs to know the garment type before it can choose\n", + "the right landmarks and measurement instructions.\n", + "\n", + "This tutorial shows how to load the pretrained tinyViT classifier, predict the category of\n", + "a single image, and evaluate the model across a whole dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Classify a single image](#single)\n", + "3. [Evaluate on a dataset](#dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package, then download the test dataset and the pretrained weights. On Colab\n", + "you can keep this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import torch\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import tinyViT\n", + "from garmentiq.classification.utils import CachedDataset\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own: every model loader and every\n", + "# inference function takes a `device` argument that defaults to \"cpu\". Pass it\n", + "# explicitly to use a GPU (\"cuda\") or Apple Silicon (\"mps\").\n", + "if torch.cuda.is_available():\n", + " device = \"cuda\"\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + "else:\n", + " device = \"cpu\"\n", + "\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the dataset and the pretrained model\n", + "\n", + "!mkdir -p models\n", + "\n", + "!curl -sL -o garmentiq-classification-set-nordstrom-and-myntra.zip \\\n", + " https://www.kaggle.com/api/v1/datasets/download/lygitdata/garmentiq-classification-set-nordstrom-and-myntra\n", + "\n", + "!wget -q -O ./models/tiny_vit.pt \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit.pt\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Classify a single image\n", + "\n", + "Every GarmentIQ model follows the same two-step shape: load the model once, then call the\n", + "inference function with it. Classification is no exception." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Split the archive into a train and test set, then cache the test images\n", + "DATA = giq.classification.train_test_split(\n", + " output_dir=\"data\",\n", + " metadata_csv=\"metadata.csv\",\n", + " label_column=\"garment\",\n", + " train_zip_dir=\"garmentiq-classification-set-nordstrom-and-myntra.zip\",\n", + " test_size=0.15,\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 1: load the model\n", + "classifier = giq.classification.load_model(\n", + " model_path=\"./models/tiny_vit.pt\",\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", + " device=device,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 2: predict. `classes` maps the model's output index onto a readable label.\n", + "img_to_test = DATA[\"test_metadata\"][\"filename\"][88]\n", + "\n", + "pred_label, pred_prob = giq.classification.predict(\n", + " model=classifier,\n", + " image_path=f\"data/test/images/{img_to_test}\",\n", + " classes=DATA[\"test_metadata\"][\"garment\"].unique().tolist(),\n", + " resize_dim=(120, 184),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081],\n", + " device=device,\n", + ")\n", + "\n", + "print(\"Image:\", img_to_test)\n", + "print(\"Predicted label:\", pred_label)\n", + "print(\"Predicted probabilities:\", pred_prob)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Evaluate on a dataset\n", + "\n", + "`test_pytorch_nn` runs the model over a whole dataset and reports accuracy. It loads the\n", + "weights itself, so it takes the model path rather than a loaded model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_images, test_labels, _ = giq.classification.load_data(\n", + " df=DATA[\"test_metadata\"],\n", + " img_dir=DATA[\"test_images\"],\n", + " label_column=\"garment\",\n", + " resize_dim=(120, 184),\n", + " normalize_mean=[0.8047, 0.7808, 0.7769],\n", + " normalize_std=[0.2957, 0.3077, 0.3081],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giq.classification.test_pytorch_nn(\n", + " model_path=\"./models/tiny_vit.pt\",\n", + " model_class=tinyViT,\n", + " model_args={\"num_classes\": 9, \"img_size\": (120, 184), \"patch_size\": 6},\n", + " dataset_class=CachedDataset,\n", + " dataset_args={\n", + " \"raw_labels\": DATA[\"test_metadata\"][\"garment\"],\n", + " \"cached_images\": test_images,\n", + " \"cached_labels\": test_labels,\n", + " },\n", + " # This function takes its device inside `param`, alongside the batch size\n", + " param={\"batch_size\": 64, \"device\": device},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Training and fine-tuning your own classifier are covered in the\n", + "advanced usage notebooks." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/tutorial_grounding/tutorial_grounding.ipynb b/working/examples/tutorial_grounding/tutorial_grounding.ipynb new file mode 100644 index 0000000..ad62880 --- /dev/null +++ b/working/examples/tutorial_grounding/tutorial_grounding.ipynb @@ -0,0 +1,297 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - GarmentIQ Grounding\n", + "\n", + "Grounding turns a natural-language phrase into bounding boxes. GarmentIQ uses it to give\n", + "SAM 1 and SAM 2 text-prompted segmentation, since neither model contains a text encoder.\n", + "\n", + "This tutorial shows how to load Grounding DINO, convert a phrase such as \"a shirt\" into\n", + "boxes, tune the detection thresholds, and pass the result into segmentation. Because the\n", + "boxes are model agnostic, grounding lives in its own module and works with any\n", + "prompt-driven model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Ground a phrase into boxes](#ground)\n", + "3. [Tune the thresholds](#thresholds)\n", + "4. [Use grounding with segmentation](#segmentation)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download the test image, Grounding DINO, and a SAM model. On Colab\n", + "you can keep this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import torch\n", + "from PIL import Image\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.grounding import (\n", + " load_grounding_model,\n", + " load_grounding_processor,\n", + " ground_text_to_boxes,\n", + ")\n", + "from garmentiq.segmentation.model_definition.sam import (\n", + " SamModel,\n", + " load_sam_config,\n", + " load_sam_processor,\n", + ")\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own: every model loader and every\n", + "# inference function takes a `device` argument that defaults to \"cpu\". Pass it\n", + "# explicitly to use a GPU (\"cuda\") or Apple Silicon (\"mps\").\n", + "if torch.cuda.is_available():\n", + " device = \"cuda\"\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + "else:\n", + " device = \"cpu\"\n", + "\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the test image and the model weights\n", + "\n", + "!mkdir -p ./test_image\n", + "!wget -q -O ./test_image/cloth_1.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg\n", + "\n", + "# Grounding DINO. The whole directory is needed because the processor bundles a tokenizer.\n", + "!mkdir -p ./models/gdino\n", + "for _f in [\n", + " \"config.json\", \"preprocessor_config.json\", \"tokenizer.json\",\n", + " \"tokenizer_config.json\", \"special_tokens_map.json\", \"vocab.txt\",\n", + " \"model.safetensors\",\n", + "]:\n", + " !wget -q -O ./models/gdino/{_f} https://huggingface.co/IDEA-Research/grounding-dino-tiny/resolve/main/{_f}\n", + "\n", + "# SAM 1, base, to consume the boxes at the end of the tutorial\n", + "!mkdir -p ./models/sam_b\n", + "!wget -q -O ./models/sam_b/model.safetensors \\\n", + " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Ground a phrase into boxes\n", + "\n", + "Grounding DINO has a dedicated loader. It ties several decoder heads to one shared set of\n", + "weights, and only `from_pretrained` performs that tying correctly, so loading it through\n", + "the generic model loader would leave those tensors randomly initialized.\n", + "\n", + "`ground_text_to_boxes` returns boxes as `[x_min, y_min, x_max, y_max]` in pixel\n", + "coordinates, ordered by descending confidence." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "grounder = load_grounding_model(\"./models/gdino\", device=device)\n", + "grounding_processor = load_grounding_processor(\"./models/gdino\")\n", + "\n", + "image = Image.open(\"./test_image/cloth_1.jpg\").convert(\"RGB\")\n", + "print(\"image size (w, h):\", image.size)\n", + "\n", + "boxes = ground_text_to_boxes(\n", + " model=grounder,\n", + " processor=grounding_processor,\n", + " image=image,\n", + " text=\"a shirt\",\n", + " box_threshold=0.3,\n", + " text_threshold=0.3,\n", + " device=device,\n", + ")\n", + "\n", + "for i, b in enumerate(boxes):\n", + " print(f\"box {i}: {[round(v) for v in b]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The phrase is free text. Grounding DINO expects lowercase phrases ending\n", + "in a period, and GarmentIQ normalizes your input to that convention automatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for phrase in [\"a shirt\", \"sleeve\", \"the neckline\"]:\n", + " try:\n", + " found = ground_text_to_boxes(\n", + " model=grounder,\n", + " processor=grounding_processor,\n", + " image=image,\n", + " text=phrase,\n", + " max_boxes=1,\n", + " device=device,\n", + " )\n", + " print(f\"{phrase:<15} -> {[round(v) for v in found[0]]}\")\n", + " except ValueError as e:\n", + " print(f\"{phrase:<15} -> no match ({e})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Tune the thresholds\n", + "\n", + "`box_threshold` is the minimum detection confidence and `text_threshold` the minimum\n", + "text-matching score. Raising them returns fewer but more certain boxes. `max_boxes` keeps\n", + "only the highest-scoring few, which is usually what you want before handing a box to SAM.\n", + "\n", + "If nothing clears the thresholds, a `ValueError` is raised rather than an empty list, so a\n", + "failed grounding cannot quietly become an empty prompt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for t in [0.1, 0.3, 0.5]:\n", + " try:\n", + " found = ground_text_to_boxes(\n", + " model=grounder,\n", + " processor=grounding_processor,\n", + " image=image,\n", + " text=\"a shirt\",\n", + " box_threshold=t,\n", + " text_threshold=t,\n", + " device=device,\n", + " )\n", + " print(f\"threshold={t} -> {len(found)} box(es)\")\n", + " except ValueError as e:\n", + " print(f\"threshold={t} -> {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Use grounding with segmentation\n", + "\n", + "You rarely need to call `ground_text_to_boxes` yourself. Pass a `text` prompt plus a\n", + "grounding model to `segmentation.extract` and GarmentIQ runs both steps for you." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sam = giq.segmentation.load_model(\n", + " model_class=SamModel,\n", + " model_path=\"./models/sam_b/model.safetensors\",\n", + " model_args={\"config\": load_sam_config(\"sam-vit-b\")},\n", + " device=device,\n", + ")\n", + "\n", + "original_img, mask = giq.segmentation.extract(\n", + " model=sam,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=load_sam_processor(\"sam-vit-b\"),\n", + " prompt={\"text\": \"a shirt\"},\n", + " grounding_model=grounder,\n", + " grounding_processor=grounding_processor,\n", + " grounding_args={\"box_threshold\": 0.3, \"text_threshold\": 0.3, \"max_boxes\": 1},\n", + " device=device,\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=mask, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Grounding is only needed for SAM 1 and SAM 2. SAM 3 understands text\n", + "natively, so it takes a `text` prompt with no grounding model at all. See the\n", + "[segmentation tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_segmentation.ipynb)." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/tutorial_landmark_detection/tutorial_landmark_detection.ipynb b/working/examples/tutorial_landmark_detection/tutorial_landmark_detection.ipynb new file mode 100644 index 0000000..89d4c14 --- /dev/null +++ b/working/examples/tutorial_landmark_detection/tutorial_landmark_detection.ipynb @@ -0,0 +1,228 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - GarmentIQ Landmark Detection\n", + "\n", + "Landmark detection locates the key points of a garment, such as shoulders, sleeve ends,\n", + "and hems. These points are what GarmentIQ measures between, so accurate landmarks lead\n", + "directly to accurate measurements.\n", + "\n", + "This tutorial shows how to load the pretrained HRNet model, detect the predefined\n", + "landmarks for a known garment class, read the confidence scores, and plot the results on\n", + "the original image." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Detect landmarks](#detect)\n", + "3. [Understand the output](#output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download the test image and the pretrained weights. On Colab you\n", + "can keep this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import torch\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own: every model loader and every\n", + "# inference function takes a `device` argument that defaults to \"cpu\". Pass it\n", + "# explicitly to use a GPU (\"cuda\") or Apple Silicon (\"mps\").\n", + "if torch.cuda.is_available():\n", + " device = \"cuda\"\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + "else:\n", + " device = \"cpu\"\n", + "\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the test image and the pretrained model\n", + "\n", + "!mkdir -p ./test_image\n", + "!wget -q -O ./test_image/cloth_3.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", + "\n", + "!mkdir -p ./models\n", + "!wget -q -O ./models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Detect landmarks\n", + "\n", + "Landmark detection needs to know the garment class, because each class has its own set of\n", + "landmarks. `garment_classes` holds the predefined ones. The test image is a vest dress." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Available garment classes:\")\n", + "for name in garment_classes:\n", + " print(\" -\", name)\n", + "\n", + "giq.landmark.plot(image_path=\"./test_image/cloth_3.jpg\", figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 1: load the model. Note that `model_class` is an instance here, not a class.\n", + "HRNet = giq.landmark.detection.load_model(\n", + " model_path=\"./models/hrnet.pth\",\n", + " model_class=PoseHighResolutionNet(),\n", + " device=device,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 2: detect\n", + "coords, maxvals, detection_dict = giq.landmark.detect(\n", + " class_name=\"vest dress\",\n", + " class_dict=garment_classes,\n", + " image_path=\"./test_image/cloth_3.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " device=device,\n", + ")\n", + "\n", + "giq.landmark.plot(\n", + " image_path=\"./test_image/cloth_3.jpg\",\n", + " coordinate=coords,\n", + " figsize=(3, 3),\n", + " color=\"green\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Understand the output\n", + "\n", + "`detect` returns three things:\n", + "\n", + "| Value | Meaning |\n", + "|---|---|\n", + "| `coords` | landmark coordinates in pixels, shaped `(1, n_landmarks, 2)` |\n", + "| `maxvals` | the model's confidence for each landmark, from 0 to 1 |\n", + "| `detection_dict` | the full per-landmark record, keyed by landmark ID |\n", + "\n", + "`detection_dict` is what the refinement, derivation, and measurement stages consume." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"coords shape :\", coords.shape)\n", + "print(\"maxvals shape:\", maxvals.shape)\n", + "\n", + "for i, (xy, conf) in enumerate(zip(coords[0], maxvals[0])):\n", + " print(f\"landmark {i + 1:>2}: x={xy[0]:7.1f} y={xy[1]:7.1f} confidence={conf.item():.3f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Detected landmarks can sit slightly off the true garment edge, and some\n", + "useful points are not predicted at all. Both problems are solved in the\n", + "[landmark refinement and derivation tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_landmark_refinement_and_derivation.ipynb)." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/tutorial_landmark_refinement_and_derivation/tutorial_landmark_refinement_and_derivation.ipynb b/working/examples/tutorial_landmark_refinement_and_derivation/tutorial_landmark_refinement_and_derivation.ipynb new file mode 100644 index 0000000..a0280b9 --- /dev/null +++ b/working/examples/tutorial_landmark_refinement_and_derivation/tutorial_landmark_refinement_and_derivation.ipynb @@ -0,0 +1,320 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - GarmentIQ Landmark Refinement and Derivation\n", + "\n", + "Detected landmarks often sit slightly off the garment edge, and some useful points are not\n", + "predicted by the model at all. **Refinement** snaps detected landmarks onto the true\n", + "boundary using a segmentation mask, while **derivation** computes entirely new landmarks\n", + "from geometric rules.\n", + "\n", + "This tutorial shows how to run both steps, why each one needs a segmentation mask, and how\n", + "to plot detected and derived points together." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Detect landmarks and a mask](#detect)\n", + "3. [Refine the landmarks](#refine)\n", + "4. [Derive new landmarks](#derive)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download the test image and both models. On Colab you can keep\n", + "this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import numpy as np\n", + "import torch\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes\n", + "from garmentiq.segmentation.model_definition.birefnet import (\n", + " BiRefNet,\n", + " load_birefnet_config,\n", + ")\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own: every model loader and every\n", + "# inference function takes a `device` argument that defaults to \"cpu\". Pass it\n", + "# explicitly to use a GPU (\"cuda\") or Apple Silicon (\"mps\").\n", + "if torch.cuda.is_available():\n", + " device = \"cuda\"\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + "else:\n", + " device = \"cpu\"\n", + "\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the test image and the pretrained models\n", + "\n", + "!mkdir -p ./test_image\n", + "!wget -q -O ./test_image/cloth_3.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", + "\n", + "# HRNet, for landmark detection\n", + "!mkdir -p ./models\n", + "!wget -q -O ./models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", + "\n", + "# BiRefNet, for the segmentation mask both steps need\n", + "!mkdir -p ./models/birefnet\n", + "!wget -q -O ./models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Detect landmarks and a mask\n", + "\n", + "Refinement and derivation both work from a segmentation mask, so we run detection and\n", + "segmentation first. Detection is covered in the\n", + "[landmark detection tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_landmark_detection.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "HRNet = giq.landmark.detection.load_model(\n", + " model_path=\"./models/hrnet.pth\",\n", + " model_class=PoseHighResolutionNet(),\n", + " device=device,\n", + ")\n", + "\n", + "coords, maxvals, detection_dict = giq.landmark.detect(\n", + " class_name=\"vest dress\",\n", + " class_dict=garment_classes,\n", + " image_path=\"./test_image/cloth_3.jpg\",\n", + " model=HRNet,\n", + " scale_std=200.0,\n", + " resize_dim=[288, 384],\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " device=device,\n", + ")\n", + "\n", + "giq.landmark.plot(\n", + " image_path=\"./test_image/cloth_3.jpg\",\n", + " coordinate=coords,\n", + " figsize=(3, 3),\n", + " color=\"green\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "birefnet = giq.segmentation.load_model(\n", + " model_class=BiRefNet,\n", + " model_path=\"./models/birefnet/model.safetensors\",\n", + " model_args=load_birefnet_config(),\n", + " device=device,\n", + ")\n", + "\n", + "original_img, mask = giq.segmentation.extract(\n", + " model=birefnet,\n", + " image_path=\"./test_image/cloth_3.jpg\",\n", + " resize_dim=(1024, 1024),\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " device=device,\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=mask, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Refine the landmarks\n", + "\n", + "`refine` searches a small window around each detected point and moves it onto the mask\n", + "boundary. `window_size` sets how far it may travel, and `ksize` and `sigmaX` control the\n", + "Gaussian blur applied to the mask before the search, which smooths away jagged edges." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "refined_coords, refined_detection_dict = giq.landmark.refine(\n", + " class_name=\"vest dress\",\n", + " detection_np=coords,\n", + " detection_conf=maxvals,\n", + " detection_dict=detection_dict,\n", + " mask=mask,\n", + " window_size=5,\n", + " ksize=(11, 11),\n", + " sigmaX=0.0,\n", + ")\n", + "\n", + "shift = np.linalg.norm(refined_coords[0] - coords[0], axis=1)\n", + "for i, d in enumerate(shift):\n", + " print(f\"landmark {i + 1:>2}: moved {d:6.2f} px\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giq.landmark.plot(\n", + " image_path=\"./test_image/cloth_3.jpg\",\n", + " coordinate=refined_coords,\n", + " figsize=(3, 3),\n", + " color=\"green\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Derive new landmarks\n", + "\n", + "Some measurement points are not predicted by the model. They are marked\n", + "`predefined: False` in the garment class definition and computed instead, by intersecting\n", + "a geometric construction with the mask.\n", + "\n", + "`derivation_dict` holds the rules. `derive` returns the newly derived coordinates and an\n", + "updated detection dictionary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "derived_coords, derived_detection_dict = giq.landmark.derive(\n", + " class_name=\"vest dress\",\n", + " detection_dict=refined_detection_dict,\n", + " derivation_dict=giq.landmark.derivation_dict.derivation_dict,\n", + " landmark_coords=refined_coords,\n", + " np_mask=mask,\n", + ")\n", + "\n", + "print(\"Derived landmarks:\")\n", + "for landmark_id, xy in derived_coords.items():\n", + " print(f\" id {landmark_id}: x={xy[0]:.1f}, y={xy[1]:.1f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the refined landmarks together with a derived one to see where it\n", + "landed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "derived_id = list(derived_coords.keys())[0]\n", + "\n", + "giq.landmark.plot(\n", + " image_path=\"./test_image/cloth_3.jpg\",\n", + " coordinate=np.concatenate(\n", + " (refined_coords, np.array([[derived_coords[derived_id]]])), axis=1\n", + " ),\n", + " figsize=(3, 3),\n", + " color=\"green\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Defining your own derivation rules is covered in the custom measurement\n", + "instruction advanced usage notebook." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/tutorial_matting/tutorial_matting.ipynb b/working/examples/tutorial_matting/tutorial_matting.ipynb new file mode 100644 index 0000000..fcf8dec --- /dev/null +++ b/working/examples/tutorial_matting/tutorial_matting.ipynb @@ -0,0 +1,490 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - GarmentIQ Matting\n", + "\n", + "Segmentation gives every pixel a yes or no answer, which leaves hard, stair-stepped edges\n", + "and discards semi-transparent detail such as lace or loose fibers. **Matting** instead\n", + "predicts a continuous alpha value per pixel, so a garment composites onto a new background\n", + "without a cut-out look.\n", + "\n", + "Matting always needs guidance from a segmentation step first. This tutorial covers the two\n", + "models GarmentIQ supports, ViTMatte and Matting Anything, the trimap that guides them, and\n", + "three practical pairings." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Matting with ViTMatte and a trimap](#vitmatte)\n", + "3. [Matting with Matting Anything](#mam)\n", + "4. [Matting from a text prompt with SAM 3](#sam3)\n", + "5. [Choosing a pairing](#choosing)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download the test image and model weights. On Colab you can keep\n", + "this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import numpy as np\n", + "import torch\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.segmentation.model_definition.birefnet import (\n", + " BiRefNet,\n", + " load_birefnet_config,\n", + ")\n", + "from garmentiq.segmentation.model_definition.sam import (\n", + " SamModel,\n", + " Sam3Model,\n", + " load_sam_config,\n", + " load_sam_processor,\n", + ")\n", + "from garmentiq.matting.model_definition.vitmatte import (\n", + " VitMatteForImageMatting,\n", + " load_vitmatte_config,\n", + " load_vitmatte_processor,\n", + ")\n", + "from garmentiq.matting.model_definition.mam import load_mam\n", + "\n", + "# Matting runs at full image resolution. Apple Silicon (\"mps\") can return degenerate\n", + "# output on very large images, so CUDA or CPU is preferred here.\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the test image and the model weights\n", + "\n", + "!mkdir -p ./test_image\n", + "!wget -q -O ./test_image/cloth_1.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg\n", + "\n", + "# BiRefNet, a prompt-free segmentation model\n", + "!mkdir -p ./models/birefnet\n", + "!wget -q -O ./models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", + "\n", + "# ViTMatte, small\n", + "!mkdir -p ./models/vitmatte\n", + "!wget -q -O ./models/vitmatte/model.safetensors \\\n", + " https://huggingface.co/hustvl/vitmatte-small-composition-1k/resolve/main/model.safetensors\n", + "\n", + "# SAM 1, base\n", + "!mkdir -p ./models/sam_b\n", + "!wget -q -O ./models/sam_b/model.safetensors \\\n", + " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors\n", + "\n", + "# Matting Anything, about 408 MB. It bundles a frozen SAM, but GarmentIQ reads only the\n", + "# matting decoder from it and reuses the SAM you load separately.\n", + "!mkdir -p ./models/mam\n", + "!wget -q -O ./models/mam/mam_sam_vitb.pth \\\n", + " https://huggingface.co/spaces/shi-labs/Matting-Anything/resolve/main/checkpoints/mam_sam_vitb.pth\n", + "\n", + "# SAM 3 is gated, see the SAM 3 section below.\n", + "!mkdir -p ./models/sam3\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Matting with ViTMatte and a trimap\n", + "\n", + "ViTMatte is guided by a **trimap**: white (255) is definitely foreground, black (0) is\n", + "definitely background, and gray (128) is the unknown band where the model is free to\n", + "predict soft alpha.\n", + "\n", + "`generate_trimap` builds one from any segmentation mask. It erodes the mask to get\n", + "confident foreground, dilates it to get confident background, and marks the ring between\n", + "them as unknown. We start from BiRefNet, which needs no prompt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "birefnet = giq.segmentation.load_model(\n", + " model_class=BiRefNet,\n", + " model_path=\"./models/birefnet/model.safetensors\",\n", + " model_args=load_birefnet_config(),\n", + " device=device,\n", + ")\n", + "\n", + "original_img, mask_biref = giq.segmentation.extract(\n", + " model=birefnet,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " resize_dim=(1024, 1024),\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " device=device,\n", + ")\n", + "\n", + "trimap = giq.matting.generate_trimap(mask_biref, erode_size=15, dilate_size=15)\n", + "\n", + "print(\"trimap values:\", np.unique(trimap).tolist())\n", + "giq.segmentation.plot(image_np=trimap, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now run ViTMatte. `composite` blends the image onto a new background\n", + "using the alpha values, rather than cutting it out along a hard edge." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vitmatte = giq.matting.load_model(\n", + " model_class=VitMatteForImageMatting,\n", + " model_path=\"./models/vitmatte/model.safetensors\",\n", + " model_args={\"config\": load_vitmatte_config(\"vitmatte-small-composition-1k\")},\n", + " device=device,\n", + ")\n", + "vitmatte_processor = load_vitmatte_processor(\"vitmatte-small-composition-1k\")\n", + "\n", + "image_np, alpha = giq.matting.matte(\n", + " model=vitmatte,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=vitmatte_processor,\n", + " trimap=trimap,\n", + " device=device,\n", + ")\n", + "\n", + "composited = giq.matting.composite(\n", + " image_np=image_np,\n", + " alpha_np=alpha,\n", + " background_color=(102, 255, 102),\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=alpha, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=composited, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Let GarmentIQ build the trimap\n", + "\n", + "Passing `mask=` instead of `trimap=` derives the trimap internally, which is the usual way\n", + "to chain segmentation into matting. `trimap_args` forwards options to `generate_trimap`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_, alpha_auto = giq.matting.matte(\n", + " model=vitmatte,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=vitmatte_processor,\n", + " mask=mask_biref,\n", + " trimap_args={\"erode_size\": 15, \"dilate_size\": 15},\n", + " device=device,\n", + ")\n", + "\n", + "print(\"identical to passing the trimap explicitly:\", np.array_equal(alpha, alpha_auto))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tuning the unknown band\n", + "\n", + "The band width decides where soft alpha may appear. Too narrow and the edges stay hard;\n", + "too wide and the model has to guess across large areas." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for size in [5, 15, 35]:\n", + " tm = giq.matting.generate_trimap(mask_biref, erode_size=size, dilate_size=size)\n", + " _, a = giq.matting.matte(\n", + " model=vitmatte,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=vitmatte_processor,\n", + " trimap=tm,\n", + " device=device,\n", + " )\n", + " soft = int(((a > 10) & (a < 245)).sum())\n", + " print(f\"erode/dilate={size:>2} | unknown band px={int((tm == 128).sum()):>9,} \"\n", + " f\"| soft alpha px={soft:>9,}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Matting with Matting Anything\n", + "\n", + "Matting Anything needs no trimap. It prompts a frozen SAM and refines the resulting mask\n", + "into an alpha matte directly, so it is structurally tied to SAM and takes a SAM style\n", + "`prompt` instead of a trimap.\n", + "\n", + "`load_mam` reads only the matting decoder from the checkpoint and pairs it with a SAM model\n", + "you already loaded, so SAM is not loaded twice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sam = giq.segmentation.load_model(\n", + " model_class=SamModel,\n", + " model_path=\"./models/sam_b/model.safetensors\",\n", + " model_args={\"config\": load_sam_config(\"sam-vit-b\")},\n", + " device=device,\n", + ")\n", + "sam_processor = load_sam_processor(\"sam-vit-b\")\n", + "\n", + "mam = load_mam(\n", + " checkpoint_path=\"./models/mam/mam_sam_vitb.pth\",\n", + " sam_model=sam,\n", + " sam_processor=sam_processor,\n", + " device=device,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> **Prompt quality matters.** MAM keeps SAM's highest-confidence mask, and\n", + "> an ambiguous prompt can make SAM return a sub-part or even the background, which inverts\n", + "> the matte. A box drawn tightly around the garment is the reliable choice, so here we\n", + "> derive one from a segmentation pass first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_, mask_sam = giq.segmentation.extract(\n", + " model=sam,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam_processor,\n", + " prompt={\"boxes\": [[[200, 200, 1600, 2200]]]},\n", + " device=device,\n", + ")\n", + "\n", + "ys, xs = np.where(mask_sam > 127)\n", + "tight_box = [float(xs.min()), float(ys.min()), float(xs.max()), float(ys.max())]\n", + "print(\"tight garment box:\", [round(v) for v in tight_box])\n", + "\n", + "# Note there is no trimap anywhere in this call\n", + "image_np, alpha_mam = giq.matting.matte(\n", + " model=mam,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " prompt={\"boxes\": [[tight_box]]},\n", + " device=device,\n", + ")\n", + "\n", + "composited_mam = giq.matting.composite(\n", + " image_np=image_np,\n", + " alpha_np=alpha_mam,\n", + " background_color=(102, 255, 102),\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=alpha_mam, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=composited_mam, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Matting from a text prompt with SAM 3\n", + "\n", + "ViTMatte accepts a trimap from *any* segmentation mask, so it is not limited to BiRefNet.\n", + "Here the mask comes from SAM 3, which selects the garment from a natural-language\n", + "description.\n", + "\n", + "> **SAM 3 is gated.** Accept the licence at\n", + "> [facebook/sam3](https://huggingface.co/facebook/sam3), download `model.safetensors`, and\n", + "> place it at `./models/sam3/model.safetensors`. Only the weights are gated: GarmentIQ\n", + "> bundles SAM 3's configuration, processor, and tokenizer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "SAM3_WEIGHTS = \"./models/sam3/model.safetensors\"\n", + "HAS_SAM3 = os.path.exists(SAM3_WEIGHTS)\n", + "\n", + "if not HAS_SAM3:\n", + " print(f\"SAM 3 weights not found at {SAM3_WEIGHTS} - the SAM 3 cells will be skipped.\")\n", + "else:\n", + " print(\"SAM 3 weights found.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if HAS_SAM3:\n", + " sam3 = giq.segmentation.load_model(\n", + " model_class=Sam3Model,\n", + " model_path=SAM3_WEIGHTS,\n", + " model_args={\"config\": load_sam_config(\"sam3\")},\n", + " device=device,\n", + " )\n", + "\n", + " _, mask_sam3 = giq.segmentation.extract(\n", + " model=sam3,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=load_sam_processor(\"sam3\"),\n", + " prompt={\"text\": \"t-shirt\"},\n", + " device=device,\n", + " )\n", + "\n", + " # Exactly the same matting call as before, only the mask differs\n", + " image_np, alpha_sam3 = giq.matting.matte(\n", + " model=vitmatte,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=vitmatte_processor,\n", + " mask=mask_sam3,\n", + " trimap_args={\"erode_size\": 15, \"dilate_size\": 15},\n", + " device=device,\n", + " )\n", + "\n", + " composited_sam3 = giq.matting.composite(\n", + " image_np=image_np,\n", + " alpha_np=alpha_sam3,\n", + " background_color=(102, 255, 102),\n", + " )\n", + "\n", + " giq.segmentation.plot(image_np=alpha_sam3, figsize=(3, 3))\n", + " giq.segmentation.plot(image_np=composited_sam3, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Choosing a pairing\n", + "\n", + "The two matting models differ in what they need to be guided by, which is what decides the\n", + "segmentation model they pair with.\n", + "\n", + "| Model | Guidance | Trimap | Notes |\n", + "|---|---|:---:|---|\n", + "| **ViTMatte** | a trimap | required | Configs bundled for `small` and `base`; the trimap can be derived from any segmentation mask automatically |\n", + "| **Matting Anything** | a SAM prompt | not needed | Refines a SAM mask, and reuses the same SAM you already loaded for segmentation |\n", + "\n", + "That gives three natural pairings:\n", + "\n", + "| Segmentation | Matting | Guidance | Prompt needed |\n", + "|---|---|---|---|\n", + "| BiRefNet | ViTMatte | trimap from the mask | no, fully automatic |\n", + "| SAM 1 or SAM 2 | Matting Anything | a SAM prompt, points or boxes | yes, geometric |\n", + "| SAM 3 | ViTMatte | trimap from a text-prompted mask | yes, text |\n", + "\n", + "ViTMatte works from any mask, so it pairs with any segmentation model. Matting Anything\n", + "consumes SAM's own image embeddings rather than just its mask, so it only works with SAM.\n", + "\n", + "> **Apple Silicon note.** Above roughly 3.5 megapixels the MPS backend has been observed to\n", + "> return a degenerate ViTMatte matte that differs substantially from the CPU result, and it\n", + "> can exhaust GPU memory. GarmentIQ emits a `RuntimeWarning` in that case. Use\n", + "> `device=\"cpu\"`, or downscale the image, for very large inputs.\n", + "\n", + "To run matting as part of the full measurement pipeline, see the\n", + "[tailor tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_tailor.ipynb)." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/tutorial_segmentation/tutorial_segmentation.ipynb b/working/examples/tutorial_segmentation/tutorial_segmentation.ipynb new file mode 100644 index 0000000..21f0b09 --- /dev/null +++ b/working/examples/tutorial_segmentation/tutorial_segmentation.ipynb @@ -0,0 +1,596 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - GarmentIQ Segmentation\n", + "\n", + "Segmentation separates a garment from its background and produces the mask that the later\n", + "GarmentIQ stages depend on. GarmentIQ exposes four segmentation backends through one\n", + "consistent API: BiRefNet, which needs no prompt at all, and SAM 1, SAM 2, and SAM 3, which\n", + "are guided by prompts.\n", + "\n", + "This tutorial shows how to load each model, how to pass point, box, label, and text\n", + "prompts through a single unified `prompt` dictionary, and how to change an image\n", + "background once you have a mask." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Segmentation using BiRefNet](#birefnet)\n", + "3. [Segmentation using SAM 1](#sam1)\n", + "4. [Segmentation using SAM 2](#sam2)\n", + "5. [Segmentation using SAM 3](#sam3)\n", + "6. [Which prompts each model supports](#capabilities)\n", + "7. [Reference notes](#notes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download the test image and the model weights. On Colab you can\n", + "keep this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import torch\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.segmentation.model_definition.birefnet import (\n", + " BiRefNet,\n", + " load_birefnet_config,\n", + ")\n", + "from garmentiq.segmentation.model_definition.sam import (\n", + " SamModel,\n", + " Sam2Model,\n", + " Sam3Model,\n", + " load_sam_config,\n", + " load_sam_processor,\n", + ")\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own: every model loader and every\n", + "# inference function takes a `device` argument that defaults to \"cpu\". Pass it\n", + "# explicitly to use a GPU (\"cuda\") or Apple Silicon (\"mps\").\n", + "if torch.cuda.is_available():\n", + " device = \"cuda\"\n", + "elif torch.backends.mps.is_available():\n", + " device = \"mps\"\n", + "else:\n", + " device = \"cpu\"\n", + "\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the test image and the model weights\n", + "\n", + "!mkdir -p ./test_image\n", + "!wget -q -O ./test_image/cloth_1.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_1.jpg\n", + "\n", + "# BiRefNet\n", + "!mkdir -p ./models/birefnet\n", + "!wget -q -O ./models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", + "\n", + "# SAM 1, base\n", + "!mkdir -p ./models/sam_b\n", + "!wget -q -O ./models/sam_b/model.safetensors \\\n", + " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors\n", + "\n", + "# SAM 2.1, tiny\n", + "!mkdir -p ./models/sam2_t\n", + "!wget -q -O ./models/sam2_t/model.safetensors \\\n", + " https://huggingface.co/facebook/sam2.1-hiera-tiny/resolve/main/model.safetensors\n", + "\n", + "# Grounding DINO gives SAM 1 and SAM 2 text prompts. The whole directory is needed\n", + "# because the processor bundles a tokenizer.\n", + "!mkdir -p ./models/gdino\n", + "for _f in [\n", + " \"config.json\", \"preprocessor_config.json\", \"tokenizer.json\",\n", + " \"tokenizer_config.json\", \"special_tokens_map.json\", \"vocab.txt\",\n", + " \"model.safetensors\",\n", + "]:\n", + " !wget -q -O ./models/gdino/{_f} https://huggingface.co/IDEA-Research/grounding-dino-tiny/resolve/main/{_f}\n", + "\n", + "# SAM 3 is gated, see the SAM 3 section below.\n", + "!mkdir -p ./models/sam3\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Segmentation using BiRefNet\n", + "\n", + "BiRefNet is prompt free: give it an image and it returns a mask. Note that\n", + "`segmentation.extract` is called without any `prompt` argument here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "birefnet = giq.segmentation.load_model(\n", + " model_class=BiRefNet,\n", + " model_path=\"./models/birefnet/model.safetensors\",\n", + " model_args=load_birefnet_config(),\n", + " device=device,\n", + ")\n", + "\n", + "original_img, mask_biref = giq.segmentation.extract(\n", + " model=birefnet,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " resize_dim=(1024, 1024),\n", + " normalize_mean=[0.485, 0.456, 0.406],\n", + " normalize_std=[0.229, 0.224, 0.225],\n", + " device=device,\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=original_img, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=mask_biref, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With a mask in hand, `change_background_color` replaces everything outside\n", + "the garment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bg_modified = giq.segmentation.change_background_color(\n", + " image_np=original_img,\n", + " mask_np=mask_biref,\n", + " background_color=[102, 255, 102],\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=bg_modified, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Segmentation using SAM 1\n", + "\n", + "SAM models are prompted. All prompt types travel in one `prompt` dictionary, which accepts\n", + "the keys `points`, `labels`, `boxes`, and `text`. Here we use **points** and **boxes**.\n", + "\n", + "The test image is 1800 x 2400 pixels, and prompts are given in pixel coordinates." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sam = giq.segmentation.load_model(\n", + " model_class=SamModel,\n", + " model_path=\"./models/sam_b/model.safetensors\",\n", + " model_args={\"config\": load_sam_config(\"sam-vit-b\")},\n", + " device=device,\n", + ")\n", + "sam_processor = load_sam_processor(\"sam-vit-b\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Point prompt\n", + "\n", + "A point says \"the object I want is here\". The nesting is\n", + "`[[[x, y]]]`: a list of images, each holding a list of points." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "original_img, mask_point = giq.segmentation.extract(\n", + " model=sam,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam_processor,\n", + " prompt={\"points\": [[[900, 1200]]]},\n", + " device=device,\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=mask_point, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Box prompt\n", + "\n", + "A box is `[x_min, y_min, x_max, y_max]`. It is usually more reliable than a single point,\n", + "because a point can latch onto a sub-part such as a printed logo." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_, mask_box = giq.segmentation.extract(\n", + " model=sam,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam_processor,\n", + " prompt={\"boxes\": [[[200, 200, 1600, 2200]]]},\n", + " device=device,\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=mask_box, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Segmentation using SAM 2\n", + "\n", + "SAM 2 loads and runs through exactly the same calls as SAM 1, only the model class and the\n", + "variant name change. Here we show the two prompt types not yet covered: **labels** and\n", + "**text**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sam2 = giq.segmentation.load_model(\n", + " model_class=Sam2Model,\n", + " model_path=\"./models/sam2_t/model.safetensors\",\n", + " model_args={\"config\": load_sam_config(\"sam2.1-hiera-tiny\")},\n", + " device=device,\n", + ")\n", + "sam2_processor = load_sam_processor(\"sam2.1-hiera-tiny\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Label prompt\n", + "\n", + "`labels` annotates each point: **1** means \"include this region\", **0** means \"exclude\n", + "it\". Negative points are how you carve away a part the model wrongly grabbed, for example\n", + "keeping the shirt while rejecting the collar area." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# One positive point on the garment body\n", + "_, mask_pos = giq.segmentation.extract(\n", + " model=sam2,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam2_processor,\n", + " prompt={\"points\": [[[900, 1200]]], \"labels\": [[1]]},\n", + " device=device,\n", + ")\n", + "\n", + "# The same point, plus a negative point that pushes the mask away from the upper chest\n", + "_, mask_neg = giq.segmentation.extract(\n", + " model=sam2,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam2_processor,\n", + " prompt={\"points\": [[[900, 1200], [900, 700]]], \"labels\": [[1, 0]]},\n", + " device=device,\n", + ")\n", + "\n", + "print(\"positive only coverage:\", round(float((mask_pos > 127).mean()), 4))\n", + "print(\"positive+negative coverage:\", round(float((mask_neg > 127).mean()), 4))\n", + "\n", + "giq.segmentation.plot(image_np=mask_pos, figsize=(3, 3))\n", + "giq.segmentation.plot(image_np=mask_neg, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Text prompt\n", + "\n", + "Neither SAM 1 nor SAM 2 has a text encoder, so a phrase cannot reach them directly.\n", + "GarmentIQ first turns the phrase into boxes with **Grounding DINO**, then uses those boxes\n", + "as an ordinary box prompt. Pass a grounding model and the pipeline handles the rest.\n", + "\n", + "Grounding is covered on its own in the\n", + "[grounding tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_grounding.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from garmentiq.grounding import load_grounding_model, load_grounding_processor\n", + "\n", + "grounder = load_grounding_model(\"./models/gdino\", device=device)\n", + "grounding_processor = load_grounding_processor(\"./models/gdino\")\n", + "\n", + "_, mask_text = giq.segmentation.extract(\n", + " model=sam2,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam2_processor,\n", + " prompt={\"text\": \"a shirt\"},\n", + " grounding_model=grounder,\n", + " grounding_processor=grounding_processor,\n", + " grounding_args={\"box_threshold\": 0.3, \"text_threshold\": 0.3, \"max_boxes\": 1},\n", + " device=device,\n", + ")\n", + "\n", + "giq.segmentation.plot(image_np=mask_text, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Segmentation using SAM 3\n", + "\n", + "SAM 3 is the first generation that understands **text natively**, so no grounding model is\n", + "needed.\n", + "\n", + "> **SAM 3 is gated.** Meta distributes the weights under a licence you must accept, so\n", + "> they cannot be bundled or downloaded automatically. Accept the licence at\n", + "> [facebook/sam3](https://huggingface.co/facebook/sam3), download `model.safetensors`, and\n", + "> place it at `./models/sam3/model.safetensors`.\n", + ">\n", + "> Only the *weights* are gated. GarmentIQ ships SAM 3's configuration, processor, and\n", + "> tokenizer inside the package, so nothing else has to be fetched and the model loads\n", + "> fully offline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "SAM3_WEIGHTS = \"./models/sam3/model.safetensors\"\n", + "HAS_SAM3 = os.path.exists(SAM3_WEIGHTS)\n", + "\n", + "if not HAS_SAM3:\n", + " print(f\"SAM 3 weights not found at {SAM3_WEIGHTS} - the SAM 3 cells will be skipped.\")\n", + "else:\n", + " print(\"SAM 3 weights found.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if HAS_SAM3:\n", + " # No `model_dir` argument: the config, processor and tokenizer come from the\n", + " # bundled package files, so only the weights are read from disk.\n", + " sam3 = giq.segmentation.load_model(\n", + " model_class=Sam3Model,\n", + " model_path=SAM3_WEIGHTS,\n", + " model_args={\"config\": load_sam_config(\"sam3\")},\n", + " device=device,\n", + " )\n", + " sam3_processor = load_sam_processor(\"sam3\")\n", + "\n", + " _, mask_sam3 = giq.segmentation.extract(\n", + " model=sam3,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam3_processor,\n", + " prompt={\"text\": \"t-shirt\"},\n", + " device=device,\n", + " )\n", + "\n", + " giq.segmentation.plot(image_np=mask_sam3, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Point prompts with SAM 3\n", + "\n", + "SAM 3's detector is an open-vocabulary detector, so it is prompted by *describing* an\n", + "object, not by clicking one, and it rejects point prompts. The same checkpoint also\n", + "contains a **tracker**, which does carry the familiar SAM style prompt encoder.\n", + "`load_sam3_tracker` reassembles it from the weights you already downloaded." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if HAS_SAM3:\n", + " from garmentiq.segmentation.model_definition.sam import (\n", + " load_sam3_tracker,\n", + " load_sam3_tracker_processor,\n", + " )\n", + "\n", + " sam3_tracker = load_sam3_tracker(model_path=SAM3_WEIGHTS, device=device)\n", + "\n", + " _, mask_sam3_point = giq.segmentation.extract(\n", + " model=sam3_tracker,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=load_sam3_tracker_processor(),\n", + " prompt={\"points\": [[[540, 530]]], \"labels\": [[1]]},\n", + " device=device,\n", + " )\n", + "\n", + " giq.segmentation.plot(image_np=mask_sam3_point, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Which prompts each model supports\n", + "\n", + "BiRefNet takes no prompt. The SAM family differs, and SAM 3 is the odd one out: it gains\n", + "text but loses points. GarmentIQ bundles every configuration listed below, so only the\n", + "weights ever need downloading.\n", + "\n", + "| Variant | `model_type` | Points | Labels | Boxes | Text | Config bundled |\n", + "|---|---|:---:|:---:|:---:|:---:|:---:|\n", + "| BiRefNet | not applicable, prompt free | — | — | — | — | yes |\n", + "| SAM 1 | `sam-vit-b`, `sam-vit-l`, `sam-vit-h` | yes | yes | yes | via grounding | yes |\n", + "| SAM 2.1 | `sam2.1-hiera-tiny`, `-small`, `-base-plus`, `-large` | yes | yes | yes | via grounding | yes |\n", + "| SAM 3 detector | `sam3` | no | no | yes | yes, native | yes, weights gated |\n", + "| SAM 3 tracker | `load_sam3_tracker()` | yes | yes | yes | no | yes, same checkpoint |\n", + "\n", + "GarmentIQ validates the prompt against the model, so an unsupported combination raises a\n", + "clear error instead of silently returning a meaningless mask. You can query the table\n", + "programmatically too." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from garmentiq.segmentation.model_definition.sam import (\n", + " sam_capabilities,\n", + " ALL_SAM_MODELS,\n", + ")\n", + "\n", + "for variant in ALL_SAM_MODELS:\n", + " caps = sam_capabilities(variant)\n", + " print(f\"{variant:<24} -> {', '.join(sorted(k for k, v in caps.items() if v))}\")\n", + "\n", + "# A prompt is always required for SAM, otherwise the mask would be arbitrary.\n", + "try:\n", + " giq.segmentation.extract(\n", + " model=sam,\n", + " image_path=\"./test_image/cloth_1.jpg\",\n", + " processor=sam_processor,\n", + " device=device,\n", + " )\n", + "except ValueError as e:\n", + " print(\"\\nExpected error:\", e)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Reference notes\n", + "\n", + "**Prompt nesting is normalized for you.** SAM 1 and SAM 2 disagree on how deeply prompts\n", + "must be nested, and SAM 2 raises if given SAM 1's shape. GarmentIQ reshapes the prompt to\n", + "whatever the model expects, so the same `prompt` dictionary works across versions.\n", + "\n", + "**Each SAM generation needs a recent enough `transformers`.** The classes are imported\n", + "lazily, so GarmentIQ itself imports fine on older releases, and asking for a model your\n", + "release does not carry raises a clear `ImportError` telling you to upgrade.\n", + "\n", + "| Model | First `transformers` release |\n", + "|---|---|\n", + "| SAM 1 | 4.29 |\n", + "| ViTMatte | 4.34 |\n", + "| Grounding DINO | 4.40 |\n", + "| SAM 2 | 4.56 |\n", + "| SAM 3 | 5.0 |\n", + "\n", + "Installing `garmentiq[sam2]` or `garmentiq[sam3]` pulls the required floor automatically.\n", + "\n", + "**Choosing a processor backend.** `load_sam_processor` resizes with PIL by default, which\n", + "matches GarmentIQ's historical behavior so masks stay reproducible across `transformers`\n", + "releases. Pass `backend=\"torchvision\"` for a faster processor; on this tutorial's test\n", + "image the two agree to within 19 pixels out of 4.3 million. The older `use_fast` flag still\n", + "works, mapping `True` to `\"torchvision\"` and `False` to `\"pil\"`." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/tutorial_tailor/tutorial_tailor.ipynb b/working/examples/tutorial_tailor/tutorial_tailor.ipynb new file mode 100644 index 0000000..53ce31c --- /dev/null +++ b/working/examples/tutorial_tailor/tutorial_tailor.ipynb @@ -0,0 +1,527 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - GarmentIQ Tailor\n", + "\n", + "The tailor agent runs the whole GarmentIQ pipeline end to end: classification,\n", + "segmentation, optional matting, landmark detection, refinement, derivation, and\n", + "measurement. It processes a folder of images and writes masks, annotated images, and\n", + "measurement files into an output directory.\n", + "\n", + "This tutorial shows how to configure a tailor agent, how to read the metadata table it\n", + "returns, how to swap the segmentation backend, and how to enable alpha matting for cleaner\n", + "cutouts." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#prerequisites)\n", + "2. [Measure with BiRefNet](#birefnet)\n", + "3. [Read the results](#results)\n", + "4. [Measure with SAM](#sam)\n", + "5. [Add alpha matting](#matting)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Prerequisites\n", + "\n", + "Install the package and download three test images and every model in the pipeline. On\n", + "Colab you can keep this section collapsed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Install GarmentIQ\n", + "!pip install garmentiq -q" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @title Import GarmentIQ and choose a device\n", + "\n", + "import json\n", + "\n", + "import torch\n", + "\n", + "import garmentiq as giq\n", + "from garmentiq.classification.model_definition import tinyViT\n", + "from garmentiq.landmark.detection.model_definition import PoseHighResolutionNet\n", + "from garmentiq.garment_classes import garment_classes\n", + "from garmentiq.landmark.derivation.derivation_dict import derivation_dict\n", + "from garmentiq.segmentation.model_definition.birefnet import (\n", + " BiRefNet,\n", + " load_birefnet_config,\n", + ")\n", + "from garmentiq.segmentation.model_definition.sam import (\n", + " SamModel,\n", + " load_sam_config,\n", + " load_sam_processor,\n", + ")\n", + "from garmentiq.matting.model_definition.vitmatte import (\n", + " VitMatteForImageMatting,\n", + " load_vitmatte_config,\n", + " load_vitmatte_processor,\n", + ")\n", + "\n", + "# GarmentIQ never grabs an accelerator on its own: every model loader and every\n", + "# inference function takes a `device` argument that defaults to \"cpu\".\n", + "# The matting stage below runs at full image resolution, and Apple Silicon (\"mps\")\n", + "# can run out of GPU memory and return a subtly wrong matte there, so this tutorial\n", + "# prefers CUDA or CPU. Without `do_matte=True`, \"mps\" is fine.\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form" + }, + "outputs": [], + "source": [ + "# @title Download the test images and every model\n", + "\n", + "# cloth_2 is a short sleeve top, cloth_3 a vest dress, cloth_4 a skirt\n", + "!mkdir -p ./test_image\n", + "!wget -q -O ./test_image/cloth_2.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_2.jpg\n", + "!wget -q -O ./test_image/cloth_3.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_3.jpg\n", + "!wget -q -O ./test_image/cloth_4.jpg \\\n", + " https://raw.githubusercontent.com/lygitdata/GarmentIQ/refs/heads/gh-pages/asset/img/cloth_4.jpg\n", + "\n", + "!mkdir -p ./models\n", + "\n", + "# Classification\n", + "!wget -q -O ./models/tiny_vit_inditex_finetuned.pt \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/tiny_vit_inditex_finetuned.pt\n", + "\n", + "# Landmark detection\n", + "!wget -q -O ./models/hrnet.pth \\\n", + " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", + "\n", + "# Segmentation, BiRefNet\n", + "!mkdir -p ./models/birefnet\n", + "!wget -q -O ./models/birefnet/model.safetensors \\\n", + " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors\n", + "\n", + "# Segmentation, SAM 1 base\n", + "!mkdir -p ./models/sam_b\n", + "!wget -q -O ./models/sam_b/model.safetensors \\\n", + " https://huggingface.co/facebook/sam-vit-base/resolve/main/model.safetensors\n", + "\n", + "# Matting, ViTMatte small\n", + "!mkdir -p ./models/vitmatte\n", + "!wget -q -O ./models/vitmatte/model.safetensors \\\n", + " https://huggingface.co/hustvl/vitmatte-small-composition-1k/resolve/main/model.safetensors\n", + "\n", + "print(\"Downloads finished.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Measure with BiRefNet\n", + "\n", + "A tailor agent is configured once and then reused. `model_dir` is the shared root, and\n", + "each `*_model_path` is given relative to it.\n", + "\n", + "`summary()` prints the configuration and the steps that will run, which is worth checking\n", + "before a long batch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "CLASSIFICATION_ARGS = {\n", + " \"num_classes\": len(garment_classes),\n", + " \"img_size\": (120, 184),\n", + " \"patch_size\": 6,\n", + " \"resize_dim\": (120, 184),\n", + " \"normalize_mean\": [0.8047, 0.7808, 0.7769],\n", + " \"normalize_std\": [0.2957, 0.3077, 0.3081],\n", + "}\n", + "\n", + "LANDMARK_ARGS = {\n", + " \"scale_std\": 200.0,\n", + " \"resize_dim\": [288, 384],\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tailor_biref = giq.tailor(\n", + " input_dir=\"./test_image\",\n", + " model_dir=\"./models\",\n", + " output_dir=\"./output_biref\",\n", + " class_dict=garment_classes,\n", + " do_refine=True,\n", + " do_derive=True,\n", + " derivation_dict=derivation_dict,\n", + " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", + " classification_model_class=tinyViT,\n", + " classification_model_args=CLASSIFICATION_ARGS,\n", + " segmentation_model_path=\"birefnet/model.safetensors\",\n", + " segmentation_model_class=BiRefNet,\n", + " segmentation_model_args={\n", + " \"model_config\": load_birefnet_config(),\n", + " \"resize_dim\": (1024, 1024),\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + " \"background_color\": [102, 255, 102],\n", + " },\n", + " landmark_detection_model_path=\"hrnet.pth\",\n", + " landmark_detection_model_class=PoseHighResolutionNet(),\n", + " landmark_detection_model_args=LANDMARK_ARGS,\n", + " device=device,\n", + ")\n", + "\n", + "tailor_biref.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metadata, outputs = tailor_biref.measure(\n", + " save_segmentation_image=True,\n", + " save_measurement_image=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Read the results\n", + "\n", + "`measure` returns a metadata table and the raw outputs. The metadata holds the path of\n", + "every file written, which makes the results easy to look up." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(metadata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The masks\n", + "for image in metadata[\"mask_image\"]:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The images with the background replaced\n", + "for image in metadata[\"bg_modified_image\"]:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The images annotated with the measured landmarks\n", + "for image in metadata[\"measurement_image\"]:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The measurements themselves\n", + "for json_path in metadata[\"measurement_json\"]:\n", + " with open(json_path) as fh:\n", + " print(f\"{json_path}:\")\n", + " print(json.dumps(json.load(fh), indent=4, sort_keys=True))\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Measure with SAM\n", + "\n", + "Swapping the segmentation backend only changes the three `segmentation_*` arguments. SAM\n", + "is prompted, so its arguments carry a processor and a prompt that is applied to every\n", + "image in the batch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tailor_sam = giq.tailor(\n", + " input_dir=\"./test_image\",\n", + " model_dir=\"./models\",\n", + " output_dir=\"./output_sam\",\n", + " class_dict=garment_classes,\n", + " do_refine=False,\n", + " do_derive=True,\n", + " derivation_dict=derivation_dict,\n", + " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", + " classification_model_class=tinyViT,\n", + " classification_model_args=CLASSIFICATION_ARGS,\n", + " segmentation_model_path=\"sam_b/model.safetensors\",\n", + " segmentation_model_class=SamModel,\n", + " segmentation_model_args={\n", + " \"model_config\": {\"config\": load_sam_config(\"sam-vit-b\")},\n", + " \"processor\": load_sam_processor(\"sam-vit-b\"),\n", + " \"prompt\": {\"points\": [[[1000, 900]]]},\n", + " \"background_color\": [102, 255, 102],\n", + " },\n", + " landmark_detection_model_path=\"hrnet.pth\",\n", + " landmark_detection_model_class=PoseHighResolutionNet(),\n", + " landmark_detection_model_args=LANDMARK_ARGS,\n", + " device=device,\n", + ")\n", + "\n", + "tailor_sam.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metadata_sam, outputs_sam = tailor_sam.measure(\n", + " save_segmentation_image=True,\n", + " save_measurement_image=True,\n", + ")\n", + "\n", + "for image in metadata_sam[\"measurement_image\"]:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Add alpha matting\n", + "\n", + "Set `do_matte=True` and supply a matting model to replace the hard segmentation cutout\n", + "with a soft alpha matte. The pipeline then produces an alpha matte and an\n", + "alpha-composited image alongside the usual outputs, and landmark detection runs on the\n", + "composited image.\n", + "\n", + "Matting itself is covered in the\n", + "[matting tutorial](https://colab.research.google.com/github/lygitdata/GarmentIQ/blob/main/test/tutorial_matting.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tailor_matte = giq.tailor(\n", + " input_dir=\"./test_image\",\n", + " model_dir=\"./models\",\n", + " output_dir=\"./output_matte\",\n", + " class_dict=garment_classes,\n", + " do_refine=True,\n", + " do_derive=True,\n", + " derivation_dict=derivation_dict,\n", + " classification_model_path=\"tiny_vit_inditex_finetuned.pt\",\n", + " classification_model_class=tinyViT,\n", + " classification_model_args=CLASSIFICATION_ARGS,\n", + " segmentation_model_path=\"birefnet/model.safetensors\",\n", + " segmentation_model_class=BiRefNet,\n", + " segmentation_model_args={\n", + " \"model_config\": load_birefnet_config(),\n", + " \"resize_dim\": (1024, 1024),\n", + " \"normalize_mean\": [0.485, 0.456, 0.406],\n", + " \"normalize_std\": [0.229, 0.224, 0.225],\n", + " \"background_color\": [102, 255, 102],\n", + " },\n", + " landmark_detection_model_path=\"hrnet.pth\",\n", + " landmark_detection_model_class=PoseHighResolutionNet(),\n", + " landmark_detection_model_args=LANDMARK_ARGS,\n", + " do_matte=True,\n", + " matting_model_path=\"vitmatte/model.safetensors\",\n", + " matting_model_class=VitMatteForImageMatting,\n", + " matting_model_args={\n", + " \"model_config\": {\n", + " \"config\": load_vitmatte_config(\"vitmatte-small-composition-1k\")\n", + " },\n", + " \"processor\": load_vitmatte_processor(\"vitmatte-small-composition-1k\"),\n", + " \"trimap_args\": {\"erode_size\": 15, \"dilate_size\": 15},\n", + " \"background_color\": [102, 255, 102],\n", + " },\n", + " device=device,\n", + ")\n", + "\n", + "tailor_matte.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metadata_matte, outputs_matte = tailor_matte.measure(\n", + " save_segmentation_image=True,\n", + " save_measurement_image=True,\n", + " save_matting_image=True,\n", + ")\n", + "\n", + "# Two new columns appear in the metadata\n", + "print([c for c in metadata_matte.columns if \"matte\" in c or \"matting\" in c])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The alpha mattes\n", + "for image in metadata_matte[\"matte_image\"]:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))\n", + "\n", + "# The alpha-composited images\n", + "for image in metadata_matte[\"matte_composite_image\"]:\n", + " giq.landmark.plot(image_path=image, figsize=(3, 3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How matting interacts with the rest of the pipeline\n", + "\n", + "**Matting requires segmentation.** The segmentation mask is what supplies ViTMatte's\n", + "trimap and Matting Anything's guidance, so `do_matte=True` forces the segmentation stage to\n", + "run. Standalone matting has no such requirement: call `garmentiq.matting.matte` with\n", + "whatever image and trimap you already have.\n", + "\n", + "**Matting also feeds landmark detection.** The pose model works best on a\n", + "background-replaced image, so when matting is enabled the alpha composite is what gets\n", + "measured:\n", + "\n", + "| `do_matte` | segmentation `background_color` | matting `background_color` | Detection runs on | Composited onto |\n", + "|:---:|:---:|:---:|---|---|\n", + "| yes | set | set | soft alpha composite | matting color |\n", + "| yes | set | — | soft alpha composite | segmentation color |\n", + "| yes | — | set | soft alpha composite | matting color |\n", + "| yes | — | — | soft alpha composite | white, a neutral default |\n", + "| no | set | — | hard background-modified image | segmentation color |\n", + "| no | — | — | the original image | – |\n", + "\n", + "**Which outputs you get depends on what you asked for:**\n", + "\n", + "| Output | Requires |\n", + "|---|---|\n", + "| `matte_image`, the alpha matte itself | `save_matting_image=True`, no color needed |\n", + "| `matte_composite_image` | `save_matting_image=True` **and** a matting `background_color` |\n", + "| `bg_modified_image` | a segmentation `background_color` |\n", + "\n", + "**The configuration is validated up front**, so a mistake fails immediately rather than\n", + "part way through a long batch:\n", + "\n", + "| Situation | Result |\n", + "|---|---|\n", + "| `do_matte=True` without a matting model | `ValueError` naming the missing argument |\n", + "| `do_matte=True` with ViTMatte but no `processor` | `ValueError` |\n", + "| `do_matte=True` with Matting Anything but no `prompt` | `ValueError` |\n", + "| `save_matting_image=True` without `do_matte=True` | `ValueError` |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Asking to save matting output when matting is disabled\n", + "try:\n", + " tailor_biref.measure(save_matting_image=True)\n", + "except ValueError as e:\n", + " print(\"Expected error:\", e)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/working/examples/web_landmark_detection/main.ipynb b/working/examples/web_landmark_detection/main.ipynb index 0540c08..6b08114 100644 --- a/working/examples/web_landmark_detection/main.ipynb +++ b/working/examples/web_landmark_detection/main.ipynb @@ -27,9 +27,9 @@ "outputs": [], "source": [ "%%bash\n", - "mkdir -p /app/working/examples/web_landmark_detection/models\n", - "if [ ! -f /app/working/examples/web_landmark_detection/models/hrnet.pth ]; then\n", - " wget -q -O /app/working/examples/web_landmark_detection/models/hrnet.pth \\\n", + "mkdir -p ./models\n", + "if [ ! -f ./models/hrnet.pth ]; then\n", + " wget -q -O ./models/hrnet.pth \\\n", " https://huggingface.co/lygitdata/garmentiq/resolve/main/hrnet.pth\n", "else\n", " echo \"Model file already exists, skipping download.\"\n", @@ -49,7 +49,7 @@ "# load model once\n", "print(\"Loading HRNet model…\")\n", "HRNet = giq.landmark.detection.load_model(\n", - " model_path=\"/app/working/examples/web_landmark_detection/models/hrnet.pth\",\n", + " model_path=\"./models/hrnet.pth\",\n", " model_class=PoseHighResolutionNet()\n", ")\n", "print(\"Model loaded.\")" diff --git a/working/examples/web_segmentation/main.ipynb b/working/examples/web_segmentation/main.ipynb index 275a839..5daf9f6 100644 --- a/working/examples/web_segmentation/main.ipynb +++ b/working/examples/web_segmentation/main.ipynb @@ -24,9 +24,9 @@ "outputs": [], "source": [ "# Download BiRefNet model\n", - "!mkdir -p /app/working/examples/web_segmentation/models/birefnet\n", + "!mkdir -p ./models/birefnet\n", "\n", - "!wget -q -O /app/working/examples/web_segmentation/models/birefnet/model.safetensors \\\n", + "!wget -q -O ./models/birefnet/model.safetensors \\\n", " https://huggingface.co/lygitdata/BiRefNet_garmentiq_backup/resolve/main/model.safetensors" ] }, @@ -51,7 +51,7 @@ "print(\"Loading BiRefNet model…\")\n", "birefnet = giq.segmentation.load_model(\n", " model_class=BiRefNet,\n", - " model_path=\"/app/working/examples/web_segmentation/models/birefnet/model.safetensors\",\n", + " model_path=\"./models/birefnet/model.safetensors\",\n", " model_args=load_birefnet_config()\n", ")\n", "print(\"Model loaded.\")"