Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+onnx/detr.axmodel filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, AXERA
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1,3 +1,28 @@
----
-license: bsd-3-clause
----
+# RF-DETR.axera
+RF-DETR DEMO on AXERA NPU
+
+### 1. 工程下载
+```  
+gh repo clone AXERA-TECH/Deformable-Detr.axera
+```
+
+### 2. 模型导出
+```
+pip3 install -r tools/requirements.txt
+python3 export_onnx.py --variant small --out-dir onnx
+```
+### 3. Pulsar2 
+```
+onnxsim rf-detr-small.onnx sim.onnx
+pulsar2 build --config onnx/config.json
+```
+### 4. 板端运行
+```
+python src/infer.py --model detr.axmodel --img ./assets/test.jpg  --thresh 0.5
+```
+### 5. 结果展示
+#### AXMODEL
+![axmodel_result](./asserts/ax_result.jpg)
+
+#### ONNX
+![onnx_result](./asserts/onnx_result.jpg)

asserts/imagenet-calib.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e99da619eedfd33ca8e07d2629e61e92bc5da9e81823ae84a8e1c28a6fe56781
+size 16281600

onnx/.gitattributes

@@ -0,0 +1 @@
+*.onnx filter=lfs diff=lfs merge=lfs -text

onnx/config.json

@@ -0,0 +1,46 @@
+{
+  "input": "./onnx/sim.onnx",
+  "output_dir": "./out",
+  "output_name": "detr.axmodel",
+  "model_type": "ONNX",
+  "target_hardware": "AX650",
+  "npu_mode": "NPU3",
+  "quant": {
+    "input_configs": [
+      {
+        "tensor_name": "DEFAULT",
+        "calibration_dataset": "./asserts/imagenet-calib.tar",
+        "calibration_size": 128,
+        "calibration_mean": [123.675, 116.28, 103.53],
+        "calibration_std": [58.395, 57.12, 57.375],
+        "calibration_format": "Image"
+      }
+    ],
+    "calibration_method": "MinMax",
+    "precision_analysis": true,
+    "precision_analysis_method": "EndToEnd",
+    "precision_analysis_mode": "NPUBackend",
+    "conv_bias_data_type": "FP32",
+    "layer_configs": [
+      {
+        "start_tensor_names": ["DEFAULT"],
+        "end_tensor_names": ["DEFAULT"],
+        "data_type": "U16"
+      }
+    ]
+  },
+  "input_processors": [
+    {
+      "tensor_name": "DEFAULT",
+      "tensor_format": "RGB",
+      "tensor_layout": "NCHW",
+      "src_dtype": "U8",
+      "src_layout": "NHWC",
+      "mean": [123.675, 116.28, 103.53],
+      "std": [58.395, 57.12, 57.375]
+    }
+  ],
+  "compiler": {
+    "enable_slice_mode": true
+  }
+}

onnx/detr.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a60c1d0711edddc754a7ea250993839e3e101cd4b8daf71535bedc5560c1acc1
+size 33966166

onnx/rf-detr-small.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:793b4d89e3aff9ee4a95c43af54cce78162a8d3aa2c75333ea188f12e323c1e2
+size 121913199

onnx/sim.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac48eca7f3dddd9c1d436999c65a1ffbbcb0e63758dbd89cdd95fc71133d0c06
+size 118451286

src/infer.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+import argparse
+import colorsys
+from pathlib import Path
+
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont
+
+try:
+    import axengine as ort
+
+    BACKEND = "axengine"
+    print("Running on AXera NPU (axengine)...")
+except ImportError:
+    import onnxruntime as ort
+
+    BACKEND = "onnxruntime"
+    print("Running on CPU/GPU (onnxruntime)...")
+
+
+MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+
+CLASSES = [
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic_light",
+    "fire_hydrant", "stop_sign", "parking_meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
+    "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
+    "skis", "snowboard", "sports_ball", "kite", "baseball_bat", "baseball_glove", "skateboard", "surfboard",
+    "tennis_racket", "bottle", "wine_glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+    "sandwich", "orange", "broccoli", "carrot", "hot_dog", "pizza", "donut", "cake", "chair", "couch",
+    "potted_plant", "bed", "dining_table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard",
+    "cell_phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
+    "scissors", "teddy_bear", "hair_drier", "toothbrush",
+]
+COCO_IDS = [
+    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21,
+    22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
+    46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65,
+    67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90,
+]
+CLASS_NAME_BY_ID = {cid: name for cid, name in zip(COCO_IDS, CLASSES)}
+
+
+def sigmoid(x: np.ndarray) -> np.ndarray:
+    return 1.0 / (1.0 + np.exp(-np.clip(x, -88.0, 88.0)))
+
+
+def get_numpy_dtype(input_meta: object) -> np.dtype:
+    if hasattr(input_meta, "dtype"):
+        return np.dtype(input_meta.dtype)
+
+    ort_type = getattr(input_meta, "type", "")
+    mapping = {
+        "tensor(float)": np.float32,
+        "tensor(float16)": np.float16,
+        "tensor(uint8)": np.uint8,
+        "tensor(int8)": np.int8,
+        "tensor(int32)": np.int32,
+        "tensor(int64)": np.int64,
+    }
+    if ort_type not in mapping:
+        raise ValueError(f"Unsupported input type: {ort_type}")
+    return np.dtype(mapping[ort_type])
+
+
+def preprocess(
+    image_path: str,
+    input_h: int,
+    input_w: int,
+    layout: str,
+    dtype: np.dtype,
+    letterbox: bool,
+) -> tuple[np.ndarray, Image.Image, dict[str, float]]:
+    raw_image = Image.open(image_path).convert("RGB")
+    orig_w, orig_h = raw_image.size
+    if letterbox:
+        scale_x = min(input_w / orig_w, input_h / orig_h)
+        scale_y = scale_x
+        resized_w = max(1, int(round(orig_w * scale_x)))
+        resized_h = max(1, int(round(orig_h * scale_y)))
+        pad_x = (input_w - resized_w) // 2
+        pad_y = (input_h - resized_h) // 2
+
+        resized = raw_image.resize((resized_w, resized_h), Image.Resampling.BILINEAR)
+        canvas = Image.new("RGB", (input_w, input_h), (0, 0, 0))
+        canvas.paste(resized, (pad_x, pad_y))
+        image = np.array(canvas)
+    else:
+        resized = raw_image.resize((input_w, input_h), Image.Resampling.BILINEAR)
+        image = np.array(resized)
+        pad_x = 0.0
+        pad_y = 0.0
+        scale_x = input_w / orig_w
+        scale_y = input_h / orig_h
+
+    if BACKEND == "axengine":
+        if layout == "NHWC":
+            tensor = image[None, ...].astype(dtype, copy=False)
+        else:
+            tensor = image.transpose(2, 0, 1)[None, ...].astype(dtype, copy=False)
+    else:
+        if layout == "NHWC":
+            if dtype != np.uint8:
+                raise ValueError(f"NHWC input only supports uint8 in this simple script, got {dtype}")
+            tensor = image[None, ...].astype(np.uint8)
+        else:
+            tensor = image.astype(np.float32) / 255.0
+            tensor = (tensor - MEAN) / STD
+            tensor = tensor.transpose(2, 0, 1)[None, ...].astype(dtype)
+
+    meta = {
+        "orig_w": float(orig_w),
+        "orig_h": float(orig_h),
+        "scale_x": float(scale_x),
+        "scale_y": float(scale_y),
+        "pad_x": float(pad_x),
+        "pad_y": float(pad_y),
+        "input_w": float(input_w),
+        "input_h": float(input_h),
+    }
+    return tensor, raw_image, meta
+
+
+def decode(outputs: list[np.ndarray], meta: dict[str, float], thresh: float) -> list[tuple[np.ndarray, float, int, int]]:
+    dets = outputs[0][0]
+    labels = outputs[1][0]
+
+    if dets.shape[-1] != 4:
+        dets, labels = labels, dets
+
+    boxes = dets
+    logits = labels[:, :-1]
+    probs = sigmoid(logits)
+    input_w = meta["input_w"]
+    input_h = meta["input_h"]
+    orig_w = meta["orig_w"]
+    orig_h = meta["orig_h"]
+    scale_x = meta["scale_x"]
+    scale_y = meta["scale_y"]
+    pad_x = meta["pad_x"]
+    pad_y = meta["pad_y"]
+
+    flat = probs.reshape(-1)
+    topk = min(300, flat.size)
+    top_idx = np.argpartition(-flat, topk - 1)[:topk]
+    top_idx = top_idx[np.argsort(-flat[top_idx])]
+
+    num_classes = probs.shape[1]
+    results = []
+    for rank, idx in enumerate(top_idx.tolist()):
+        query_id = idx // num_classes
+        label_id = idx % num_classes
+        score = float(flat[idx])
+        if score < thresh:
+            continue
+
+        cx, cy, bw, bh = boxes[query_id]
+        x1 = ((cx - bw / 2.0) * input_w - pad_x) / scale_x
+        y1 = ((cy - bh / 2.0) * input_h - pad_y) / scale_y
+        x2 = ((cx + bw / 2.0) * input_w - pad_x) / scale_x
+        y2 = ((cy + bh / 2.0) * input_h - pad_y) / scale_y
+
+        x1 = max(0.0, min(orig_w, x1))
+        y1 = max(0.0, min(orig_h, y1))
+        x2 = max(0.0, min(orig_w, x2))
+        y2 = max(0.0, min(orig_h, y2))
+
+        results.append((np.array([x1, y1, x2, y2]), score, label_id, query_id))
+    return results
+
+
+def color_for_label(label_id: int) -> tuple[int, int, int]:
+    hue = (label_id * 0.61803398875) % 1.0
+    r, g, b = colorsys.hsv_to_rgb(hue, 0.75, 1.0)
+    return int(r * 255), int(g * 255), int(b * 255)
+
+
+def draw(raw_img: Image.Image, results: list[tuple[np.ndarray, float, int, int]], output_path: str) -> None:
+    draw_obj = ImageDraw.Draw(raw_img)
+    try:
+        font = ImageFont.truetype("DejaVuSans.ttf", 18)
+    except OSError:
+        font = ImageFont.load_default()
+
+    for box, score, label_id, query_id in results:
+        x1, y1, x2, y2 = box.tolist()
+        name = CLASS_NAME_BY_ID.get(label_id, f"obj_{label_id}")
+        text = f"{name} {score:.2f}"
+        color = color_for_label(label_id)
+
+        draw_obj.rectangle([x1, y1, x2, y2], outline=color, width=3)
+        draw_obj.rectangle([x1, max(0, y1 - 22), x1 + 140, y1], fill=color)
+        draw_obj.text((x1 + 2, max(0, y1 - 20)), text, fill="black", font=font)
+        print(f"query={query_id:3d} class={name:<15} score={score:.4f} box=({x1:.1f}, {y1:.1f}, {x2:.1f}, {y2:.1f})")
+
+    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+    raw_img.save(output_path)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=str, required=True)
+    parser.add_argument("--img", type=str, required=True)
+    parser.add_argument("--output", type=str, default="result.jpg")
+    parser.add_argument("--thresh", type=float, default=0.3)
+    parser.add_argument("--letterbox", action="store_true", help="use letterbox resize instead of direct resize")
+    args = parser.parse_args()
+
+    session = ort.InferenceSession(args.model)
+    input_meta = session.get_inputs()[0]
+    shape = [int(x) for x in input_meta.shape]
+    dtype = get_numpy_dtype(input_meta)
+
+    if shape[1] in (1, 3, 4):
+        layout = "NCHW"
+        input_h, input_w = shape[2], shape[3]
+    else:
+        layout = "NHWC"
+        input_h, input_w = shape[1], shape[2]
+
+    print(f"input_name={input_meta.name} shape={shape} dtype={dtype} layout={layout}")
+
+    img_tensor, raw_img, meta = preprocess(args.img, input_h, input_w, layout, dtype, args.letterbox)
+    outputs = session.run(None, {input_meta.name: img_tensor})
+
+    results = decode(outputs, meta, args.thresh)
+    print(f"Detected {len(results)} objects.")
+
+    draw(raw_img, results, args.output)
+    print(f"Result saved to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

tools/export_onnx.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+# Reference: https://github.com/infracv/rf-detr-cpp/blob/develop/trt-files/scripts/export_onnx.py
+"""Export an RF-DETR PyTorch checkpoint to ONNX, plus a meta-sidecar JSON.
+
+The sidecar is consumed by `rfdetr/core/engine_meta.hpp`. It captures the
+variant identity, input H/W, query count, normalization stats, and color
+order — i.e. everything the C++ runtime needs that is NOT recoverable from the
+engine's tensor shapes alone.
+
+Example:
+    python export_onnx.py --variant small --out-dir onnx
+    python export_onnx.py --variant seg-large --weights /tmp/seg-large.pth
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+VARIANT_TABLE: Dict[str, Dict[str, Any]] = {
+    "nano":        {"resolution": 384, "num_queries": 300, "patch": 16, "has_masks": False,
+                    "ctors": ("RFDETRNano",)},
+    "small":       {"resolution": 512, "num_queries": 300, "patch": 16, "has_masks": False,
+                    "ctors": ("RFDETRSmall",)},
+    "medium":      {"resolution": 576, "num_queries": 300, "patch": 16, "has_masks": False,
+                    "ctors": ("RFDETRMedium",)},
+    "base":        {"resolution": 560, "num_queries": 300, "patch": 14, "has_masks": False,
+                    "ctors": ("RFDETRBase",)},
+    "large":       {"resolution": 704, "num_queries": 300, "patch": 16, "has_masks": False,
+                    "ctors": ("RFDETRLarge",)},
+    "seg-nano":    {"resolution": 312, "num_queries": 100, "patch": 12, "has_masks": True,
+                    "ctors": ("RFDETRSegNano",)},
+    "seg-small":   {"resolution": 384, "num_queries": 100, "patch": 12, "has_masks": True,
+                    "ctors": ("RFDETRSegSmall",)},
+    "seg-medium":  {"resolution": 432, "num_queries": 200, "patch": 12, "has_masks": True,
+                    "ctors": ("RFDETRSegMedium",)},
+    "seg-large":   {"resolution": 504, "num_queries": 200, "patch": 12, "has_masks": True,
+                    "ctors": ("RFDETRSegLarge",)},
+    "seg-xlarge":  {"resolution": 624, "num_queries": 300, "patch": 12, "has_masks": True,
+                    "ctors": ("RFDETRSegXLarge",)},
+    "seg-2xlarge": {"resolution": 768, "num_queries": 300, "patch": 12, "has_masks": True,
+                    "ctors": ("RFDETRSeg2XLarge",)},
+    "seg-preview": {"resolution": 432, "num_queries": 200, "patch": 12, "has_masks": True,
+                    "ctors": ("RFDETRSegPreview",)},
+}
+
+
+def resolve_ctor(rfdetr_module, ctor_names):
+    for name in ctor_names:
+        if hasattr(rfdetr_module, name):
+            return getattr(rfdetr_module, name)
+    raise AttributeError(
+        f"none of {ctor_names} are exposed by `rfdetr` — upstream may have renamed the variant. "
+        f"Edit VARIANT_TABLE in this script to match."
+    )
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--variant", required=True, choices=sorted(VARIANT_TABLE.keys()),
+                    help="RF-DETR variant to export")
+    ap.add_argument("--weights", default=None,
+                    help="path to .pth checkpoint (default: upstream pretrained)")
+    ap.add_argument("--out-dir", default="onnx",
+                    help="output directory for .onnx and .json sidecar")
+    ap.add_argument("--name", default=None,
+                    help="output basename (default: rf-detr-<variant>)")
+    ap.add_argument("--opset", type=int, default=17)
+    ap.add_argument("--dynamic-batch", action="store_true",
+                    help="export with batch dimension as a dynamic axis")
+    args = ap.parse_args()
+
+    spec = VARIANT_TABLE[args.variant]
+
+    try:
+        import rfdetr  # type: ignore
+    except ImportError:
+        sys.exit("rfdetr Python package not installed (pip install rfdetr)")
+
+    ctor = resolve_ctor(rfdetr, spec["ctors"])
+    model_kwargs = {"pretrain_weights": args.weights} if args.weights else {}
+    model = ctor(**model_kwargs)
+
+    out_dir = Path(args.out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    basename = args.name or f"rf-detr-{args.variant}"
+
+    # rfdetr's `.export(format="onnx", output_dir=...)` typically writes
+    # `inference_model.onnx` (and possibly a sim'd variant) into output_dir.
+    # We invoke it, then locate the produced .onnx and rename to our convention.
+    print(f"[export_onnx] exporting variant={args.variant} to {out_dir}/")
+    model.export(
+        format="onnx",
+        output_dir=str(out_dir),
+        opset_version=args.opset,
+        dynamic_batch=args.dynamic_batch,
+    )
+
+    # Find the most recently produced .onnx in out_dir; rename to <basename>.onnx.
+    candidates = sorted(out_dir.glob("*.onnx"), key=lambda p: p.stat().st_mtime, reverse=True)
+    if not candidates:
+        sys.exit(f"export reported success but no .onnx file appeared in {out_dir}")
+    src = candidates[0]
+    dst = out_dir / f"{basename}.onnx"
+    if src.resolve() != dst.resolve():
+        shutil.move(str(src), str(dst))
+        print(f"[export_onnx] renamed {src.name} -> {dst.name}")
+
+    print(f"[export_onnx] wrote {dst}")
+
+
+
+if __name__ == "__main__":
+    main()

tools/requirements.txt

@@ -0,0 +1,3 @@
+rfdetr>=1.7.0
+onnx>=1.15
+onnxsim>=0.4.36