Update README.md

README.md

@@ -121,8 +121,8 @@ conversation = [
     {
         "role": "user",
         "content": [
-            {"type": "image", "url": "https://huggingface.co/microsoft/kosmos-2-patch14-224/resolve/main/snowman.jpg"},
-            {"type": "text", "text": "이 이미지에는 무엇이 보이나요?"},
+            {"type": "image", "url": "https://huggingface.co/NCSOFT/VARCO-VISION-2.0-14B/resolve/main/demo.jpg"},
+            {"type": "text", "text": "이 이미지에 표시된 것은 무엇인가요?"},
         ],
     },
 ]
@@ -135,45 +135,86 @@ inputs = processor.apply_chat_template(
     return_tensors="pt"
 ).to(model.device, torch.float16)
 
-generate_ids = model.generate(**inputs, max_new_tokens=1024, do_sample=False)
-outputs = processor.decode(generate_ids[0], skip_special_tokens=True)
-print(outputs)
+generate_ids = model.generate(**inputs, max_new_tokens=1024)
+generate_ids_trimmed = [
+    out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generate_ids)
+]
+output = processor.decode(generate_ids_trimmed[0], skip_special_tokens=True)
+print(output)
 ```
-The following shows the input required for using OCR with text localization, along with the corresponding output:
+
+<details>
+<summary>Multi image inference</summary>
+
 ```python
-# INPUT
-image_file = "./assets/ocr.jpg"
-raw_image = Image.open(image_file)
 conversation = [
     {
         "role": "user",
         "content": [
-            {"type": "text", "text": ""},
-            {"type": "image"},
+            {"type": "image", "image": "file:///path/to/image1.jpg"},
+            {"type": "image", "image": "file:///path/to/image2.jpg"},
+            {"type": "text", "text": "이미지 간의 유사점을 파악하세요."},
         ],
     },
 ]
 
-# OUTPUT
-"""
-<char>백범로</char><bbox>0.172, 0.266, 0.328, 0.341</bbox>
-<char>124번길</char><bbox>0.347, 0.266, 0.512, 0.341</bbox>
-<char>Baekbeom-ro</char><bbox>0.171, 0.337, 0.433, 0.392</bbox>
-<char>124</char><bbox>0.444, 0.341, 0.508, 0.392</bbox>
-<char>만수주공아파트</char><bbox>0.109, 0.531, 0.335, 0.601</bbox>
-<char>시흥</char><bbox>0.443, 0.518, 0.522, 0.581</bbox>
-<char>시청</char><bbox>0.711, 0.521, 0.811, 0.594</bbox>
-<char>Mansu</char><bbox>0.102, 0.601, 0.181, 0.648</bbox>
-<char>Jugong</char><bbox>0.186, 0.601, 0.273, 0.658</bbox>
-<char>Apt</char><bbox>0.28, 0.601, 0.327, 0.651</bbox>
-<char>42</char><bbox>0.377, 0.601, 0.416, 0.648</bbox>
-<char>Shieung</char><bbox>0.445, 0.578, 0.53, 0.625</bbox>
-<char>인천대공원</char><bbox>0.43, 0.621, 0.609, 0.684</bbox>
-<char>모래내시장역</char><bbox>0.651, 0.59, 0.873, 0.665</bbox>
-<char>IncheonGrand</char><bbox>0.432, 0.681, 0.561, 0.723</bbox>
-<char>Park</char><bbox>0.564, 0.681, 0.611, 0.723</bbox>
-"""
+inputs = processor.apply_chat_template(
+    conversation,
+    add_generation_prompt=True,
+    tokenize=True,
+    return_dict=True,
+    return_tensors="pt"
+).to(model.device, torch.float16)
+
+generate_ids = model.generate(**inputs, max_new_tokens=1024)
+generate_ids_trimmed = [
+    out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generate_ids)
+]
+output = processor.decode(generate_ids_trimmed[0], skip_special_tokens=True)
+print(output)
 ```
-<div align="center">
-    <img src="./ocr.jpg" width="100%" />
-</div>
+</details>
+
+<details>
+<summary>OCR inference</summary>
+
+```python
+from PIL import Image
+
+image = Image.open("file:///path/to/image.jpg")
+
+# Image upscaling for OCR performance boost
+w, h = image.size
+target_size = 2304
+if max(w, h) < target_size:
+    scaling_factor = target_size / max(w, h)
+    new_w = int(w * scaling_factor)
+    new_h = int(h * scaling_factor)
+    image = image.resize((new_w, new_h))
+
+conversation = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "image", "image": image},
+            {"type": "text", "text": "<ocr>"},
+        ],
+    },
+]
+
+inputs = processor.apply_chat_template(
+    conversation,
+    add_generation_prompt=True,
+    tokenize=True,
+    return_dict=True,
+    return_tensors="pt"
+).to(model.device, torch.float16)
+
+generate_ids = model.generate(**inputs, max_new_tokens=1024)
+generate_ids_trimmed = [
+    out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generate_ids)
+]
+output = processor.decode(generate_ids_trimmed[0], skip_special_tokens=False)
+print(output)
+```
+</details>