add APQ-ViT

example/APQ_ViT/.gitignore

@@ -0,0 +1,2 @@
+weights/
+__pycache__/

example/APQ_ViT/README.md

@@ -0,0 +1,40 @@
+# APQ-ViT
+
+## 1. 简介
+本示例介绍了一种用于 Vision Transformer (ViT) 模型的后训练量化方法（APQ-ViT）。APQ-ViT 是一种简单而高效的 post-training 量化方法,无需重新训练即可实现模型量化。
+
+## 2. Benchmark
+| 模型            | w4a4  | w4a8  | w8a8  |
+|-----------------|-------|-------|-------|
+| ViT-S/16        | 47.95 | 72.30 | 81.25 |
+
+
+## 3. APQ-ViT 的测试
+
+APQ-ViT 整体方法如图所示，详情见论文 [Towards Accurate Post-Training Quantization for Vision Transformer](https://arxiv.org/abs/2303.14341)
+
+![arch](arch.png)
+
+### 3.1 准备环境
+- PaddlePaddle >= 2.4 （可从Paddle官网下载安装）
+
+安装 paddlepaddle：
+```shell
+# CPU
+pip install paddlepaddle==2.4.2
+# GPU 以Ubuntu、CUDA 11.2为例
+python -m pip install paddlepaddle-gpu==2.4.2.post112 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html
+```
+
+### 3.2 准备数据集
+
+本示例在 ImageNet 上进行分类实验。
+
+### 3.3 校准与测试
+
+```sh
+CUDA_VISIBLE_DEVICES=2 python3 test_vit.py \
+    --model_config_path ./configs/vit_base_patch16_224.yaml \
+    --model_path /path/to/model/weight \
+```
+

example/APQ_ViT/configs/vit_base_patch16_224.yaml

@@ -0,0 +1,33 @@
+DATA:
+    IMAGE_SIZE: 224
+    CROP_PCT: 0.875
+    DATA_PATH: /mnt/disk1/datasets/imagenet
+    BATCH_SIZE: 4
+    BATCH_SIZE_EVAL: 4
+MODEL:
+    TYPE: vit
+    NAME: vit_base_patch16_224
+    PATCH_SIZE: 16
+    MLP_RATIO: 4.0
+    QKV_BIAS: true
+    EMBED_DIM: 768
+    DEPTH: 12
+    NUM_HEADS: 12
+    DROPOUT: 0.1  # same as paper
+    ATTENTION_DROPOUT: 0.1  # same as paper
+TRAIN:
+    NUM_EPOCHS: 300  # same as paper
+    WARMUP_EPOCHS: 32  # ~10k steps (4096 batch size)
+    WEIGHT_DECAY: 0.3  # same as paper
+    BASE_LR: 3e-3
+    WARMUP_START_LR: 1e-6
+    END_LR: 0.0
+    GRAD_CLIP: 1.0
+    ACCUM_ITER: 1
+    OPTIMIZER:
+        NAME: 'AdamW'
+        BETAS: (0.9, 0.999)
+
+VALIDATE_FREQ: 1
+SAVE_FREQ: 1
+REPORT_FREQ: 20

example/APQ_ViT/configs/vit_small_patch32_224.yaml

@@ -0,0 +1,31 @@
+DATA:
+    IMAGE_SIZE: 224
+    CROP_PCT: 0.875
+MODEL:
+    TYPE: vit
+    NAME: vit_small_patch32_224
+    PATCH_SIZE: 32
+    MLP_RATIO: 4.0
+    QKV_BIAS: true
+    EMBED_DIM: 384
+    DEPTH: 12
+    NUM_HEADS: 6
+    DROPOUT: 0.1  # same as paper
+    ATTENTION_DROPOUT: 0.1  # same as paper
+TRAIN:
+    NUM_EPOCHS: 300  # same as paper
+    WARMUP_EPOCHS: 32  # ~10k steps (4096 batch size)
+    WEIGHT_DECAY: 0.3  # same as paper
+    BASE_LR: 3e-3
+    WARMUP_START_LR: 1e-6
+    END_LR: 0.0
+    GRAD_CLIP: 1.0
+    ACCUM_ITER: 1
+    OPTIMIZER:
+        NAME: 'AdamW'
+        BETAS: (0.9, 0.999)
+
+VALIDATE_FREQ: 1
+SAVE_FREQ: 10
+REPORT_FREQ: 100
+

example/APQ_ViT/quant_layers/bit_type.py

@@ -0,0 +1,43 @@
+import paddle
+import numpy as np
+
+
+class BitType:
+
+    def __init__(self, bits, signed, name=None):
+        self.bits = bits
+        self.signed = signed
+        if name is not None:
+            self.name = name
+        else:
+            self.update_name()
+
+    @property
+    def upper_bound(self):
+        if not self.signed:
+            return 2 ** self.bits - 1
+        return 2 ** (self.bits - 1) - 1
+
+    @property
+    def lower_bound(self):
+        if not self.signed:
+            return 0
+        return -2 ** (self.bits - 1)
+
+    @property
+    def range(self):
+        return 2 ** self.bits
+
+    def update_name(self):
+        self.name = ''
+        if not self.signed:
+            self.name = self.name + 'uint'
+        else:
+            self.name = self.name + 'int'
+        self.name += '{}'.format(self.bits)
+
+
+BIT_TYPE_LIST = [BitType(4, False, 'uint4'), BitType(4, True, 'int4'),
+    BitType(8, True, 'int8'), BitType(8, False, 'uint8'), BitType(6, False,
+    'uint6'), BitType(6, True, 'int6')]
+BIT_TYPE_DICT = {bit_type.name: bit_type for bit_type in BIT_TYPE_LIST}