update: rord

.gitignore

@@ -17,3 +17,7 @@ third_party/REKD
 Dockerfile
 hloc/matchers/dedode.py
 gradio_cached_examples
+
+hloc/matchers/quadtree.py
+third_party/QuadTreeAttention
+desktop.ini

app.py

@@ -14,6 +14,9 @@ This Space demonstrates [Image Matching WebUI](https://github.com/Vincentqyw/ima
 
 🔎 For more details about supported local features and matchers, please refer to https://github.com/Vincentqyw/image-matching-webui
 
+🚀 All algorithms run on CPU for inference on HF, causing slow speeds and high latency. For faster inference, please download the [source code](https://github.com/Vincentqyw/image-matching-webui) for local deployment or check [openxlab space](https://github.com/Vincentqyw/image-matching-webui) and [direct URL](https://g-app-center-083997-7409-n9elr1.openxlab.space) 
+
+🐛 Your feedback is valuable to me. Please do not hesitate to report any bugs [here](https://github.com/Vincentqyw/image-matching-webui/issues).
 """
 
 
@@ -102,9 +105,9 @@ def run(config):
                     )
 
                 with gr.Row():
-                    button_reset = gr.Button(label="Reset", value="Reset")
+                    button_reset = gr.Button(value="Reset")
                     button_run = gr.Button(
-                        label="Run Match", value="Run Match", variant="primary"
+                         value="Run Match", variant="primary"
                     )
 
                 with gr.Accordion("Advanced Setting", open=False):
@@ -242,7 +245,7 @@ def run(config):
                     )
                     with gr.Accordion("Open for More: Geometry info", open=False):
                         geometry_result = gr.JSON(label="Reconstructed Geometry")
-
+            
             # callbacks
             match_image_src.change(
                 fn=ui_change_imagebox,

common/utils.py

@@ -48,6 +48,7 @@ def gen_examples():
         pairs = list(combinations(imgs_list, 2))
         selected = random.sample(range(len(pairs)), count)
         return [pairs[i] for i in selected]
+
     # image pair path
     path = "datasets/sacre_coeur/mapping"
     pairs = gen_images_pairs(path, len(example_matchers))
@@ -176,7 +177,10 @@ def compute_geom(
         if H is not None:
             geo_info["Homography"] = H.tolist()
             _, H1, H2 = cv2.stereoRectifyUncalibrated(
-                mkpts0.reshape(-1, 2), mkpts1.reshape(-1, 2), F, imgSize=(w1, h1)
+                mkpts0.reshape(-1, 2),
+                mkpts1.reshape(-1, 2),
+                F,
+                imgSize=(w1, h1),
             )
             geo_info["H1"] = H1.tolist()
             geo_info["H2"] = H2.tolist()
@@ -504,6 +508,11 @@ matcher_zoo = {
         "config_feature": extract_features.confs["d2net-ss"],
         "dense": False,
     },
+    "rord": {
+        "config": match_features.confs["NN-mutual"],
+        "config_feature": extract_features.confs["rord"],
+        "dense": False,
+    },
     # "d2net-ms": {
     #     "config": match_features.confs["NN-mutual"],
     #     "config_feature": extract_features.confs["d2net-ms"],

hloc/extract_features.py

@@ -115,6 +115,18 @@ confs = {
             "resize_max": 1600,
         },
     },
+    "rord": {
+        "output": "feats-rord-ss-n5000-r1600",
+        "model": {
+            "name": "rord",
+            "multiscale": False,
+            "max_keypoints": 5000,
+        },
+        "preprocessing": {
+            "grayscale": False,
+            "resize_max": 1600,
+        },
+    },
     "rootsift": {
         "output": "feats-rootsift-n5000-r1600",
         "model": {

hloc/extractors/alike.py

@@ -1,6 +1,5 @@
 import sys
 from pathlib import Path
-import subprocess
 import torch
 
 from ..utils.base_model import BaseModel

hloc/extractors/d2net.py

@@ -10,7 +10,6 @@ sys.path.append(str(d2net_path))
 from lib.model_test import D2Net as _D2Net
 from lib.pyramid import process_multiscale
 
-
 class D2Net(BaseModel):
     default_conf = {
         "model_name": "d2_tf.pth",

hloc/extractors/darkfeat.py

@@ -1,11 +1,8 @@
 import sys
 from pathlib import Path
 import subprocess
-import logging
-
 from ..utils.base_model import BaseModel
-
-logger = logging.getLogger(__name__)
+from .. import logger
 
 darkfeat_path = Path(__file__).parent / "../../third_party/DarkFeat"
 sys.path.append(str(darkfeat_path))

hloc/extractors/dedode.py

@@ -1,10 +1,10 @@
 import sys
 from pathlib import Path
 import subprocess
-import logging
 import torch
 from PIL import Image
 from ..utils.base_model import BaseModel
+from .. import logger
 import torchvision.transforms as transforms
 
 dedode_path = Path(__file__).parent / "../../third_party/DeDoDe"
@@ -14,8 +14,6 @@ from DeDoDe import dedode_detector_L, dedode_descriptor_B
 from DeDoDe.utils import to_pixel_coords
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-logger = logging.getLogger(__name__)
-
 
 class DeDoDe(BaseModel):
     default_conf = {

hloc/extractors/example.py

@@ -2,7 +2,7 @@ import sys
 from pathlib import Path
 import subprocess
 import torch
-import logging
+from .. import logger
 
 from ..utils.base_model import BaseModel
 
@@ -12,8 +12,6 @@ sys.path.append(str(example_path))
 # import some modules here
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-logger = logging.getLogger(__name__)
-
 
 class Example(BaseModel):
     # change to your default configs

hloc/extractors/fire_local.py

@@ -1,13 +1,12 @@
 from pathlib import Path
 import subprocess
-import logging
 import sys
 import torch
 import torchvision.transforms as tvf
 
 from ..utils.base_model import BaseModel
+from .. import logger
 
-logger = logging.getLogger(__name__)
 fire_path = Path(__file__).parent / "../../third_party/fire"
 
 sys.path.append(str(fire_path))

hloc/extractors/netvlad.py

@@ -1,6 +1,5 @@
 from pathlib import Path
 import subprocess
-import logging
 import numpy as np
 import torch
 import torch.nn as nn
@@ -9,8 +8,7 @@ import torchvision.models as models
 from scipy.io import loadmat
 
 from ..utils.base_model import BaseModel
-
-logger = logging.getLogger(__name__)
+from .. import logger
 
 EPS = 1e-6
 

hloc/extractors/rord.py

@@ -0,0 +1,75 @@
+import sys
+from pathlib import Path
+import subprocess
+import torch
+
+from ..utils.base_model import BaseModel
+from .. import logger
+
+rord_path = Path(__file__).parent / "../../third_party/RoRD"
+sys.path.append(str(rord_path))
+from lib.model_test import D2Net as _RoRD
+from lib.pyramid import process_multiscale
+
+class RoRD(BaseModel):
+    default_conf = {
+        "model_name": "rord.pth",
+        "checkpoint_dir": rord_path / "models",
+        "use_relu": True,
+        "multiscale": False,
+        "max_keypoints": 1024,
+    }
+    required_inputs = ["image"]
+    weight_urls = {
+        "rord.pth": "https://drive.google.com/uc?id=12414ZGKwgPAjNTGtNrlB4VV9l7W76B2o&confirm=t",
+    }
+    proxy = "http://localhost:1080"
+
+    def _init(self, conf):
+        model_path = conf["checkpoint_dir"] / conf["model_name"]
+        link = self.weight_urls[conf["model_name"]]
+        if not model_path.exists():
+            model_path.parent.mkdir(exist_ok=True)
+            cmd_wo_proxy = ["gdown", link, "-O", str(model_path)]
+            cmd = ["gdown", link, "-O", str(model_path), "--proxy", self.proxy]
+            logger.info(
+                f"Downloading the RoRD model with `{cmd_wo_proxy}`."
+            )
+            try:
+                subprocess.run(cmd_wo_proxy, check=True)
+            except subprocess.CalledProcessError as e:
+                logger.info(f"Downloading the RoRD model with `{cmd}`.")
+                try:
+                    subprocess.run(cmd, check=True)
+                except subprocess.CalledProcessError as e:
+                    logger.error(f"Failed to download the RoRD model.")
+                    raise e
+        logger.info("RoRD model loaded.")
+        self.net = _RoRD(
+            model_file=model_path, use_relu=conf["use_relu"], use_cuda=False
+        )
+
+    def _forward(self, data):
+        image = data["image"]
+        image = image.flip(1)  # RGB -> BGR
+        norm = image.new_tensor([103.939, 116.779, 123.68])
+        image = image * 255 - norm.view(1, 3, 1, 1)  # caffe normalization
+
+        if self.conf["multiscale"]:
+            keypoints, scores, descriptors = process_multiscale(image, self.net)
+        else:
+            keypoints, scores, descriptors = process_multiscale(
+                image, self.net, scales=[1]
+            )
+        keypoints = keypoints[:, [1, 0]]  # (x, y) and remove the scale
+
+        idxs = scores.argsort()[-self.conf["max_keypoints"] or None :]
+        keypoints = keypoints[idxs, :2]
+        descriptors = descriptors[idxs]
+        scores = scores[idxs]
+
+        return {
+            "keypoints": torch.from_numpy(keypoints)[None],
+            "scores": torch.from_numpy(scores)[None],
+            "descriptors": torch.from_numpy(descriptors.T)[None],
+        }

hloc/matchers/aspanformer.py

@@ -4,9 +4,8 @@ from ..utils.base_model import BaseModel
 from ..utils import do_system
 from pathlib import Path
 import subprocess
-import logging
 
-logger = logging.getLogger(__name__)
+from .. import logger
 
 sys.path.append(str(Path(__file__).parent / "../../third_party"))
 from ASpanFormer.src.ASpanFormer.aspanformer import ASpanFormer as _ASpanFormer
@@ -77,7 +76,9 @@ class ASpanFormer(BaseModel):
 
         # update: match threshold
         _config["aspan"]["match_coarse"]["thr"] = conf["match_threshold"]
-        _config["aspan"]["match_coarse"]["skh_iters"] = conf["sinkhorn_iterations"]
+        _config["aspan"]["match_coarse"]["skh_iters"] = conf[
+            "sinkhorn_iterations"
+        ]
 
         self.net = _ASpanFormer(config=_config["aspan"])
         weight_path = model_path

hloc/matchers/dkm.py

@@ -3,16 +3,14 @@ from pathlib import Path
 import torch
 from PIL import Image
 import subprocess
-import logging
 from ..utils.base_model import BaseModel
+from .. import logger
 
 sys.path.append(str(Path(__file__).parent / "../../third_party"))
 from DKM.dkm import DKMv3_outdoor
 
 dkm_path = Path(__file__).parent / "../../third_party/DKM"
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-logger = logging.getLogger(__name__)
-
 
 class DKMv3(BaseModel):
     default_conf = {

hloc/matchers/gluestick.py

@@ -1,11 +1,9 @@
 import sys
 from pathlib import Path
 import subprocess
-import logging
 import torch
 from ..utils.base_model import BaseModel
-
-logger = logging.getLogger(__name__)
+from .. import logger
 
 gluestick_path = Path(__file__).parent / "../../third_party/GlueStick"
 sys.path.append(str(gluestick_path))

hloc/matchers/lightglue.py

@@ -1,9 +1,8 @@
 import sys
 from pathlib import Path
-import logging
 from ..utils.base_model import BaseModel
+from .. import logger
 
-logger = logging.getLogger(__name__)
 lightglue_path = Path(__file__).parent / "../../third_party/LightGlue"
 sys.path.append(str(lightglue_path))
 from lightglue import LightGlue as LG

hloc/matchers/roma.py

@@ -1,10 +1,10 @@
 import sys
 from pathlib import Path
 import subprocess
-import logging
 import torch
 from PIL import Image
 from ..utils.base_model import BaseModel
+from .. import logger
 
 roma_path = Path(__file__).parent / "../../third_party/Roma"
 sys.path.append(str(roma_path))
@@ -12,8 +12,6 @@ sys.path.append(str(roma_path))
 from roma.models.model_zoo.roma_models import roma_model
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-logger = logging.getLogger(__name__)
-
 
 class Roma(BaseModel):
     default_conf = {

hloc/matchers/sgmnet.py

@@ -1,12 +1,12 @@
 import sys
 from pathlib import Path
 import subprocess
-import logging
 import torch
 from PIL import Image
 from collections import OrderedDict, namedtuple
 from ..utils.base_model import BaseModel
 from ..utils import do_system
+from .. import logger
 
 sgmnet_path = Path(__file__).parent / "../../third_party/SGMNet"
 sys.path.append(str(sgmnet_path))
@@ -14,7 +14,6 @@ sys.path.append(str(sgmnet_path))
 from sgmnet import matcher as SGM_Model
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-logger = logging.getLogger(__name__)
 
 
 class SGMNet(BaseModel):

hloc/matchers/sold2.py

@@ -4,18 +4,13 @@ from ..utils.base_model import BaseModel
 import torch
 
 from ..utils.base_model import BaseModel
+from .. import logger
+import subprocess
 
 sold2_path = Path(__file__).parent / "../../third_party/SOLD2"
 sys.path.append(str(sold2_path))
 
 from sold2.model.line_matcher import LineMatcher
-from sold2.misc.visualize_util import (
-    plot_images,
-    plot_lines,
-    plot_line_matches,
-    plot_color_line_matches,
-    plot_keypoints,
-)
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
@@ -36,9 +31,21 @@ class SOLD2(BaseModel):
         "image1",
     ]
 
+    weight_urls = {
+        "sold2_wireframe.tar": "https://www.polybox.ethz.ch/index.php/s/blOrW89gqSLoHOk/download",
+    }
     # Initialize the line matcher
     def _init(self, conf):
         checkpoint_path = conf["checkpoint_dir"] / conf["weights"]
+
+        # Download the model.
+        if not checkpoint_path.exists():
+            checkpoint_path.parent.mkdir(exist_ok=True)
+            link = self.weight_urls[conf["weights"]]
+            cmd = ["wget", link, "-O", str(checkpoint_path)]
+            logger.info(f"Downloading the SOLD2 model with `{cmd}`.")
+            subprocess.run(cmd, check=True)
+
         mode = "dynamic"  # 'dynamic' or 'static'
         match_config = {
             "model_cfg": {

hloc/utils/__init__.py

@@ -1,7 +1,7 @@
 import os
 import logging
-
-logger = logging.getLogger(__name__)
+import sys
+from .. import logger
 
 
 def do_system(cmd, verbose=False):

third_party/RoRD/LICENSE

@@ -0,0 +1,251 @@
+Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree to be
+bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-NoDerivatives 4.0 International Public License
+("Public License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You such
+rights in consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+Section 1 – Definitions.
+
+    a. Adapted Material means material subject to Copyright and Similar Rights
+       that is derived from or based upon the Licensed Material and in which
+       the Licensed Material is translated, altered, arranged, transformed, or
+       otherwise modified in a manner requiring permission under the Copyright
+       and Similar Rights held by the Licensor. For purposes of this Public
+       License, where the Licensed Material is a musical work, performance, or
+       sound recording, Adapted Material is always produced where the Licensed
+       Material is synched in timed relation with a moving image.
+    b. Copyright and Similar Rights means copyright and/or similar rights
+       closely related to copyright including, without limitation,
+       performance, broadcast, sound recording, and Sui Generis Database
+       Rights, without regard to how the rights are labeled or categorized.
+       For purposes of this Public License, the rights specified in Section
+       2(b)(1)-(2) are not Copyright and Similar Rights.
+    c. Effective Technological Measures means those measures that, in the
+       absence of proper authority, may not be circumvented under laws
+       fulfilling obligations under Article 11 of the WIPO Copyright Treaty
+       adopted on December 20, 1996, and/or similar international agreements.
+    d. Exceptions and Limitations means fair use, fair dealing, and/or any
+       other exception or limitation to Copyright and Similar Rights that
+       applies to Your use of the Licensed Material.
+    e. Licensed Material means the artistic or literary work, database, or
+       other material to which the Licensor applied this Public License.
+    f. Licensed Rights means the rights granted to You subject to the terms
+       and conditions of this Public License, which are limited to all
+       Copyright and Similar Rights that apply to Your use of the Licensed
+       Material and that the Licensor has authority to license.
+    g. Licensor means the individual(s) or entity(ies) granting rights under
+       this Public License.
+    h. NonCommercial means not primarily intended for or directed towards
+       commercial advantage or monetary compensation. For purposes of this
+       Public License, the exchange of the Licensed Material for other
+       material subject to Copyright and Similar Rights by digital
+       file-sharing or similar means is NonCommercial provided there is no
+       payment of monetary compensation in connection with the exchange.
+    i. Share means to provide material to the public by any means or process
+       that requires permission under the Licensed Rights, such as
+       reproduction, public display, public performance, distribution,
+       dissemination, communication, or importation, and to make material
+       available to the public including in ways that members of the public
+       may access the material from a place and at a time individually chosen
+       by them.
+    j. Sui Generis Database Rights means rights other than copyright resulting
+       from Directive 96/9/EC of the European Parliament and of the Council of
+       11 March 1996 on the legal protection of databases, as amended and/or
+       succeeded, as well as other essentially equivalent rights anywhere in
+       the world.
+    k. You means the individual or entity exercising the Licensed Rights under
+       this Public License. Your has a corresponding meaning.
+
+Section 2 – Scope.
+
+    a. License grant.
+        1. Subject to the terms and conditions of this Public License, the
+           Licensor hereby grants You a worldwide, royalty-free,
+           non-sublicensable, non-exclusive, irrevocable license to exercise
+           the Licensed Rights in the Licensed Material to:
+            A. reproduce and Share the Licensed Material, in whole or in part,
+               for NonCommercial purposes only; and
+            B. produce and reproduce, but not Share, Adapted Material for
+               NonCommercial purposes only.
+        2. Exceptions and Limitations. For the avoidance of doubt, where
+           Exceptions and Limitations apply to Your use, this Public License
+           does not apply, and You do not need to comply with its terms and
+           conditions.
+        3. Term. The term of this Public License is specified in Section 6(a).
+        4. Media and formats; technical modifications allowed. The Licensor
+           authorizes You to exercise the Licensed Rights in all media and
+           formats whether now known or hereafter created, and to make
+           technical modifications necessary to do so. The Licensor waives
+           and/or agrees not to assert any right or authority to forbid You
+           from making technical modifications necessary to exercise the
+           Licensed Rights, including technical modifications necessary to
+           circumvent Effective Technological Measures. For purposes of this
+           Public License, simply making modifications authorized by this
+           Section 2(a)(4) never produces Adapted Material.
+        5. Downstream recipients.
+            A. Offer from the Licensor – Licensed Material. Every recipient of
+               the Licensed Material automatically receives an offer from the
+               Licensor to exercise the Licensed Rights under the terms and
+               conditions of this Public License.
+            B. No downstream restrictions. You may not offer or impose any
+               additional or different terms or conditions on, or apply any
+               Effective Technological Measures to, the Licensed Material if
+               doing so restricts exercise of the Licensed Rights by any
+               recipient of the Licensed Material.
+        6. No endorsement. Nothing in this Public License constitutes or may
+           be construed as permission to assert or imply that You are, or that
+           Your use of the Licensed Material is, connected with, or sponsored,
+           endorsed, or granted official status by, the Licensor or others
+           designated to receive attribution as provided in Section
+           3(a)(1)(A)(i).
+
+    b. Other rights.
+        1. Moral rights, such as the right of integrity, are not licensed
+           under this Public License, nor are publicity, privacy, and/or other
+           similar personality rights; however, to the extent possible, the
+           Licensor waives and/or agrees not to assert any such rights held by
+           the Licensor to the limited extent necessary to allow You to
+           exercise the Licensed Rights, but not otherwise.
+        2. Patent and trademark rights are not licensed under this Public
+           License.
+        3. To the extent possible, the Licensor waives any right to collect
+           royalties from You for the exercise of the Licensed Rights, whether
+           directly or through a collecting society under any voluntary or
+           waivable statutory or compulsory licensing scheme. In all other
+           cases the Licensor expressly reserves any right to collect such
+           royalties, including when the Licensed Material is used other than
+           for NonCommercial purposes.
+
+Section 3 – License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the following conditions.
+
+    a. Attribution.
+
+        1. If You Share the Licensed Material, You must:
+            A. retain the following if it is supplied by the Licensor with the
+               Licensed Material:
+                i. identification of the creator(s) of the Licensed Material
+                   and any others designated to receive attribution, in any
+                   reasonable manner requested by the Licensor (including by
+                   pseudonym if designated);
+                ii. a copyright notice;
+                iii. a notice that refers to this Public License;
+                iv. a notice that refers to the disclaimer of warranties;
+                v. a URI or hyperlink to the Licensed Material to the extent
+                   reasonably practicable;
+            B. indicate if You modified the Licensed Material and retain an
+               indication of any previous modifications; and
+            C. indicate the Licensed Material is licensed under this Public
+               License, and include the text of, or the URI or hyperlink to,
+               this Public License.
+
+            For the avoidance of doubt, You do not have permission under this
+            Public License to Share Adapted Material.
+
+        2. You may satisfy the conditions in Section 3(a)(1) in any reasonable
+           manner based on the medium, means, and context in which You Share
+           the Licensed Material. For example, it may be reasonable to satisfy
+           the conditions by providing a URI or hyperlink to a resource that
+           includes the required information.
+        3. If requested by the Licensor, You must remove any of the
+           information required by Section 3(a)(1)(A) to the extent reasonably
+           practicable.
+
+Section 4 – Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that apply to
+Your use of the Licensed Material:
+
+    a. for the avoidance of doubt, Section 2(a)(1) grants You the right to
+       extract, reuse, reproduce, and Share all or a substantial portion of
+       the contents of the database for NonCommercial purposes only and
+       provided You do not Share Adapted Material;
+    b. if You include all or a substantial portion of the database contents in
+       a database in which You have Sui Generis Database Rights, then the
+       database in which You have Sui Generis Database Rights (but not its
+       individual contents) is Adapted Material; and
+    c. You must comply with the conditions in Section 3(a) if You Share all or
+       a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not replace
+Your obligations under this Public License where the Licensed Rights include
+other Copyright and Similar Rights.
+
+Section 5 – Disclaimer of Warranties and Limitation of Liability.
+
+    a. Unless otherwise separately undertaken by the Licensor, to the extent
+       possible, the Licensor offers the Licensed Material as-is and
+       as-available, and makes no representations or warranties of any kind
+       concerning the Licensed Material, whether express, implied, statutory,
+       or other. This includes, without limitation, warranties of title,
+       merchantability, fitness for a particular purpose, non-infringement,
+       absence of latent or other defects, accuracy, or the presence or
+       absence of errors, whether or not known or discoverable. Where
+       disclaimers of warranties are not allowed in full or in part, this
+       disclaimer may not apply to You.
+    b. To the extent possible, in no event will the Licensor be liable to You
+       on any legal theory (including, without limitation, negligence) or
+       otherwise for any direct, special, indirect, incidental, consequential,
+       punitive, exemplary, or other losses, costs, expenses, or damages
+       arising out of this Public License or use of the Licensed Material,
+       even if the Licensor has been advised of the possibility of such
+       losses, costs, expenses, or damages. Where a limitation of liability is
+       not allowed in full or in part, this limitation may not apply to You.
+    c. The disclaimer of warranties and limitation of liability provided above
+       shall be interpreted in a manner that, to the extent possible, most
+       closely approximates an absolute disclaimer and waiver of all
+       liability.
+
+Section 6 – Term and Termination.
+
+    a. This Public License applies for the term of the Copyright and Similar
+       Rights licensed here. However, if You fail to comply with this Public
+       License, then Your rights under this Public License terminate
+       automatically.
+    b. Where Your right to use the Licensed Material has terminated under
+       Section 6(a), it reinstates:
+        1. automatically as of the date the violation is cured, provided it is
+           cured within 30 days of Your discovery of the violation; or
+        2. upon express reinstatement by the Licensor.
+
+      For the avoidance of doubt, this Section 6(b) does not affect any right
+      the Licensor may have to seek remedies for Your violations of this
+      Public License.
+
+    c. For the avoidance of doubt, the Licensor may also offer the Licensed
+       Material under separate terms or conditions or stop distributing the
+       Licensed Material at any time; however, doing so will not terminate
+       this Public License.
+    d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
+
+Section 7 – Other Terms and Conditions.
+
+    a. The Licensor shall not be bound by any additional or different terms or
+       conditions communicated by You unless expressly agreed.
+    b. Any arrangements, understandings, or agreements regarding the Licensed
+       Material not stated herein are separate from and independent of the
+       terms and conditions of this Public License.
+
+Section 8 – Interpretation.
+
+    a. For the avoidance of doubt, this Public License does not, and shall not
+       be interpreted to, reduce, limit, restrict, or impose conditions on any
+       use of the Licensed Material that could lawfully be made without
+       permission under this Public License.
+    b. To the extent possible, if any provision of this Public License is
+       deemed unenforceable, it shall be automatically reformed to the minimum
+       extent necessary to make it enforceable. If the provision cannot be
+       reformed, it shall be severed from this Public License without
+       affecting the enforceability of the remaining terms and conditions.
+    c. No term or condition of this Public License will be waived and no
+       failure to comply consented to unless expressly agreed to by the Licensor.
+    d. Nothing in this Public License constitutes or may be interpreted as a
+       limitation upon, or waiver of, any privileges and immunities that apply
+       to the Licensor or You, including from the legal processes of any
+       jurisdiction or authority.

third_party/RoRD/configs/camera.txt

@@ -0,0 +1,3 @@
+382.1996765136719 381.8395690917969 312.7102355957031 247.72047424316406 1000.0
+
+

third_party/RoRD/configs/train_scenes.txt

@@ -0,0 +1,7 @@
+temple_nara_japan
+brandenburg_gate
+taj_mahal
+buckingham_palace
+grand_place_brussels
+hagia_sophia_interior
+westminster_abbey

third_party/RoRD/configs/train_scenes_small.txt

@@ -0,0 +1 @@
+brandenburg_gate

third_party/RoRD/demo/register.py

@@ -0,0 +1,265 @@
+import numpy as np
+import copy
+import argparse
+import os, sys
+import open3d as o3d
+from sys import argv
+from PIL import Image
+import math
+import cv2
+import torch
+
+sys.path.append("../")
+from lib.extractMatchTop import getPerspKeypoints, getPerspKeypointsEnsemble, siftMatching
+from lib.model_test import D2Net
+
+#### Cuda ####
+use_cuda = torch.cuda.is_available()
+device = torch.device('cuda:0' if use_cuda else 'cpu')
+
+#### Argument Parsing ####
+parser = argparse.ArgumentParser(description='RoRD ICP evaluation')
+
+parser.add_argument(
+	'--rgb1', type=str, default = 'rgb/rgb2_1.jpg',
+	help='path to the rgb image1'
+)
+parser.add_argument(
+	'--rgb2', type=str, default = 'rgb/rgb2_2.jpg',
+	help='path to the rgb image2'
+)
+
+parser.add_argument(
+	'--depth1', type=str, default = 'depth/depth2_1.png',
+	help='path to the depth image1'
+)
+
+parser.add_argument(
+	'--depth2', type=str, default = 'depth/depth2_2.png',
+	help='path to the depth image2'
+)
+
+parser.add_argument(
+	'--model_rord', type=str, default = '../models/rord.pth',
+	help='path to the RoRD model for evaluation'
+)
+
+parser.add_argument(
+	'--model_d2', type=str,
+	help='path to the vanilla D2-Net model for evaluation'
+)
+
+parser.add_argument(
+	'--model_ens', action='store_true',
+	help='ensemble model of RoRD + D2-Net'
+)
+
+parser.add_argument(
+	'--sift', action='store_true',
+	help='Sift'
+)
+
+parser.add_argument(
+	'--camera_file', type=str, default='../configs/camera.txt',
+	help='path to the camera intrinsics file. In order: focal_x, focal_y, center_x, center_y, scaling_factor.'
+)
+
+parser.add_argument(
+	'--viz3d', action='store_true',
+	help='visualize the pointcloud registrations'
+)
+
+args = parser.parse_args()
+
+if args.model_ens: # Change default paths accordingly for ensemble
+	model1_ens = '../../models/rord.pth'
+	model2_ens = '../../models/d2net.pth'
+
+def draw_registration_result(source, target, transformation):
+	source_temp = copy.deepcopy(source)
+	target_temp = copy.deepcopy(target)
+	source_temp.transform(transformation)
+
+	target_temp += source_temp
+	# print("Saved registered PointCloud.")
+	# o3d.io.write_point_cloud("registered.pcd", target_temp)
+
+	trgSph.append(source_temp); trgSph.append(target_temp)
+	axis1 = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.5, origin=[0, 0, 0])
+	axis2 = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.5, origin=[0, 0, 0])
+	axis2.transform(transformation)
+	trgSph.append(axis1); trgSph.append(axis2)
+	print("Showing registered PointCloud.")
+	o3d.visualization.draw_geometries(trgSph)
+
+
+def readDepth(depthFile):
+	depth = Image.open(depthFile)
+	if depth.mode != "I":
+		raise Exception("Depth image is not in intensity format")
+
+	return np.asarray(depth)
+
+def readCamera(camera):
+	with open (camera, "rt") as file:
+		contents = file.read().split()
+
+	focalX = float(contents[0])
+	focalY = float(contents[1])
+	centerX = float(contents[2])
+	centerY = float(contents[3])
+	scalingFactor = float(contents[4])
+
+	return focalX, focalY, centerX, centerY, scalingFactor
+
+def getPointCloud(rgbFile, depthFile, pts):
+	thresh = 15.0
+
+	depth = readDepth(depthFile)
+	rgb = Image.open(rgbFile)
+
+	points = []
+	colors = []
+
+	corIdx = [-1]*len(pts)
+	corPts = [None]*len(pts)
+	ptIdx = 0
+
+	for v in range(depth.shape[0]):
+		for u in range(depth.shape[1]):
+			Z = depth[v, u] / scalingFactor
+			if Z==0: continue
+			if (Z > thresh): continue
+
+			X = (u - centerX) * Z / focalX
+			Y = (v - centerY) * Z / focalY
+
+			points.append((X, Y, Z))
+			colors.append(rgb.getpixel((u, v)))
+
+			if((u, v) in pts):
+				# print("Point found.")
+				index = pts.index((u, v))
+				corIdx[index] = ptIdx
+				corPts[index] = (X, Y, Z)
+
+			ptIdx = ptIdx+1
+
+	points = np.asarray(points)
+	colors = np.asarray(colors)
+
+	pcd = o3d.geometry.PointCloud()
+	pcd.points = o3d.utility.Vector3dVector(points)
+	pcd.colors = o3d.utility.Vector3dVector(colors/255)
+
+	return pcd, corIdx, corPts
+
+
+def convertPts(A):
+	X = A[0]; Y = A[1]
+
+	x = [];	y = []
+
+	for i in range(len(X)):
+		x.append(int(float(X[i])))
+
+	for i in range(len(Y)):
+		y.append(int(float(Y[i])))
+
+	pts = []
+	for i in range(len(x)):
+		pts.append((x[i], y[i]))
+
+	return pts
+
+
+def getSphere(pts):
+	sphs = []
+
+	for ele in pts:
+		if(ele is not None):
+			sphere = o3d.geometry.TriangleMesh.create_sphere(radius=0.03)
+			sphere.paint_uniform_color([0.9, 0.2, 0])
+
+			trans = np.identity(4)
+			trans[0, 3] = ele[0]
+			trans[1, 3] = ele[1]
+			trans[2, 3] = ele[2]
+
+			sphere.transform(trans)
+			sphs.append(sphere)
+
+	return sphs
+
+
+def get3dCor(src, trg):
+	corr = []
+
+	for sId, tId in zip(src, trg):
+		if(sId != -1 and tId != -1):
+			corr.append((sId, tId))
+
+	corr = np.asarray(corr)
+
+	return corr
+
+if __name__ == "__main__":
+
+	focalX, focalY, centerX, centerY, scalingFactor = readCamera(args.camera_file)
+
+	rgb_name_src = os.path.basename(args.rgb1)
+	H_name_src = os.path.splitext(rgb_name_src)[0] + '.npy'
+	srcH = os.path.join(os.path.dirname(args.rgb1), H_name_src)
+	rgb_name_trg = os.path.basename(args.rgb2)
+	H_name_trg = os.path.splitext(rgb_name_trg)[0] + '.npy'
+	trgH = os.path.join(os.path.dirname(args.rgb2), H_name_trg)
+
+	use_cuda = torch.cuda.is_available()
+	device = torch.device('cuda:0' if use_cuda else 'cpu')
+	model1 = D2Net(model_file=args.model_d2)
+	model1 = model1.to(device)
+	model2 = D2Net(model_file=args.model_rord)
+	model2 = model2.to(device)
+
+	if args.model_rord:
+		srcPts, trgPts, matchImg, matchImgOrtho = getPerspKeypoints(args.rgb1, args.rgb2, srcH, trgH, model2, device)
+	elif args.model_d2:
+		srcPts, trgPts, matchImg, matchImgOrtho = getPerspKeypoints(args.rgb1, args.rgb2, srcH, trgH, model1, device)
+	elif args.model_ens:
+		model1 = D2Net(model_file=model1_ens)
+		model1 = model1.to(device)
+		model2 = D2Net(model_file=model2_ens)
+		model2 = model2.to(device)
+		srcPts, trgPts, matchImg, matchImgOrtho = getPerspKeypointsEnsemble(model1, model2, args.rgb1, args.rgb2, srcH, trgH, device)
+	elif args.sift:
+		srcPts, trgPts, matchImg, matchImgOrtho = siftMatching(args.rgb1, args.rgb2, srcH, trgH, device)
+
+	#### Visualization ####
+	print("\nShowing matches in perspective and orthographic view. Press q\n")
+	cv2.imshow('Orthographic view', matchImgOrtho)
+	cv2.imshow('Perspective view', matchImg)
+	cv2.waitKey()
+
+	srcPts = convertPts(srcPts)
+	trgPts = convertPts(trgPts)
+
+	srcCld, srcIdx, srcCor = getPointCloud(args.rgb1, args.depth1, srcPts)
+	trgCld, trgIdx, trgCor = getPointCloud(args.rgb2, args.depth2, trgPts)
+
+	srcSph = getSphere(srcCor)
+	trgSph = getSphere(trgCor)
+	axis = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.5, origin=[0, 0, 0])
+	srcSph.append(srcCld); srcSph.append(axis)
+	trgSph.append(trgCld); trgSph.append(axis)
+
+	corr = get3dCor(srcIdx, trgIdx)
+
+	p2p = o3d.registration.TransformationEstimationPointToPoint()
+	trans_init = p2p.compute_transformation(srcCld, trgCld, o3d.utility.Vector2iVector(corr))
+	print("Transformation matrix: \n", trans_init)
+
+	if args.viz3d:
+		# o3d.visualization.draw_geometries(srcSph)
+		# o3d.visualization.draw_geometries(trgSph)
+
+		draw_registration_result(srcCld, trgCld, trans_init)

third_party/RoRD/demo/rgb/rgb1_1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99396bb9e7c265b8bad5237806d37d8fd9d92a772e118f6de22668f1db011948
+size 200

third_party/RoRD/demo/rgb/rgb1_2.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f29bb750adcb50b497192ecbd554cf3cd74c3f1c9809d41994c5acd1654179f2
+size 200

third_party/RoRD/demo/rgb/rgb2_1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b911f2c3962789f99f31fc78313262ec3fa257b9dd8887d318f69fa7a303c04
+size 200

third_party/RoRD/demo/rgb/rgb2_2.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b82950927db25768fe129af2138f535faf3da2789c87e7c98957c90d8423f2
+size 200

third_party/RoRD/demo/rgb/rgb3_1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3262c4ce815dad042112aed3f3a082806fc3dab62ee6bd02492ec94abbf6987
+size 200