OpenCV feature matching multiple objects

Question

How can I find multiple objects of one type on one image. I use ORB feature finder and brute force matcher (opencv = 3.2.0).

My source code:

import numpy as np
import cv2
from matplotlib import pyplot as plt

MIN_MATCH_COUNT = 10

img1 = cv2.imread('box.png', 0)  # queryImage
img2 = cv2.imread('box1.png', 0) # trainImage

#img2 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)

# Initiate ORB detector
# 
orb = cv2.ORB_create(10000, 1.2, nlevels=9, edgeThreshold = 4)
#orb = cv2.ORB_create()

# find the keypoints and descriptors with SIFT
kp1, des1 = orb.detectAndCompute(img1, None)
kp2, des2 = orb.detectAndCompute(img2, None)

FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
search_params = dict(checks = 50)

flann = cv2.FlannBasedMatcher(index_params, search_params)

des1 = np.float32(des1)
des2 = np.float32(des2)

# matches = flann.knnMatch(des1, des2, 2)

bf = cv2.BFMatcher()
matches = bf.knnMatch(des1, des2, k=2)

# store all the good matches as per Lowe's ratio test.
good = []
for m,n in matches:
    if m.distance < 0.7*n.distance:
        good.append(m)

if len(good)>3:
    src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
    dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)

    M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 2)

    if M is None:
        print ("No Homography")
    else:
        matchesMask = mask.ravel().tolist()

        h,w = img1.shape
        pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
        dst = cv2.perspectiveTransform(pts,M)

        img2 = cv2.polylines(img2,[np.int32(dst)],True,255,3, cv2.LINE_AA)

else:
    print ("Not enough matches are found - %d/%d" % (len(good),MIN_MATCH_COUNT))
    matchesMask = None

draw_params = dict(matchColor = (0,255,0), # draw matches in green color
                   singlePointColor = None,
                   matchesMask = matchesMask, # draw only inliers
                   flags = 2)

img3 = cv2.drawMatches(img1,kp1,img2,kp2,good,None,**draw_params)

plt.imshow(img3, 'gray'),plt.show()

But it can find only one instance of query image.

Query Image

Test Image

Result

So its found only one image from two. What I am doing wrong?

Find the first object, compute transformation, mask area of the found object, repeat until you get all the objects. — Michał Gacka
– Michał Gacka, Commented Mar 21, 2017 at 23:08
@m3h0w, may be: 1. calculate features 2. compute transformation 3. find first object 4. mask area repeat until get all objects — V. Gai
– V. Gai, Commented Mar 22, 2017 at 8:37
Haven't got time to read the docs right now but I think that it is a fair assumption that the matching algorithm is looking for the best matching object and not for multiple objects. — Michał Gacka
– Michał Gacka, Commented Mar 22, 2017 at 9:30
@V.Gai You can also check in the literature what are the common approaches to deal with this situation (strict feature matching deals with matching descriptors, there is no object assumption). The SIFT paper of Lowe proposes an approach based on Hough voting. I found recently this paper: MAC-RANSAC: a robust algorithm for the recognition of multiple objects but it should exist many others. — Catree
– Catree, Commented Mar 22, 2017 at 10:08

M Z · Accepted Answer · 2021-07-22 19:51:56Z

11

My source to find multiple objects using ORB descriptors

import cv2
from matplotlib import pyplot as plt

MIN_MATCH_COUNT = 10

img1 = cv2.imread('box.png', 0)  # queryImage
img2 = cv2.imread('box1.png', 0) # trainImage

orb = cv2.ORB_create(10000, 1.2, nlevels=8, edgeThreshold = 5)

# find the keypoints and descriptors with ORB
kp1, des1 = orb.detectAndCompute(img1, None)
kp2, des2 = orb.detectAndCompute(img2, None)

import numpy as np
from sklearn.cluster import MeanShift, estimate_bandwidth

x = np.array([kp2[0].pt])

for i in xrange(len(kp2)):
    x = np.append(x, [kp2[i].pt], axis=0)

x = x[1:len(x)]

bandwidth = estimate_bandwidth(x, quantile=0.1, n_samples=500)

ms = MeanShift(bandwidth=bandwidth, bin_seeding=True, cluster_all=True)
ms.fit(x)
labels = ms.labels_
cluster_centers = ms.cluster_centers_

labels_unique = np.unique(labels)
n_clusters_ = len(labels_unique)
print("number of estimated clusters : %d" % n_clusters_)

s = [None] * n_clusters_
for i in xrange(n_clusters_):
    l = ms.labels_
    d, = np.where(l == i)
    print(d.__len__())
    s[i] = list(kp2[xx] for xx in d)

des2_ = des2

for i in xrange(n_clusters_):

    kp2 = s[i]
    l = ms.labels_
    d, = np.where(l == i)
    des2 = des2_[d, ]

    FLANN_INDEX_KDTREE = 0
    index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
    search_params = dict(checks = 50)

    flann = cv2.FlannBasedMatcher(index_params, search_params)

    des1 = np.float32(des1)
    des2 = np.float32(des2)

    matches = flann.knnMatch(des1, des2, 2)

    # store all the good matches as per Lowe's ratio test.
    good = []
    for m,n in matches:
        if m.distance < 0.7*n.distance:
            good.append(m)

    if len(good)>3:
        src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
        dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)

        M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 2)

        if M is None:
            print ("No Homography")
        else:
            matchesMask = mask.ravel().tolist()

            h,w = img1.shape
            pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
            dst = cv2.perspectiveTransform(pts,M)

            img2 = cv2.polylines(img2,[np.int32(dst)],True,255,3, cv2.LINE_AA)

            draw_params = dict(matchColor=(0, 255, 0),  # draw matches in green color
                               singlePointColor=None,
                               matchesMask=matchesMask,  # draw only inliers
                               flags=2)

            img3 = cv2.drawMatches(img1, kp1, img2, kp2, good, None, **draw_params)

            plt.imshow(img3, 'gray'), plt.show()

    else:
        print ("Not enough matches are found - %d/%d" % (len(good),MIN_MATCH_COUNT))
        matchesMask = None

Result images

edited Jul 22, 2021 at 19:51

M Z

4,8392 gold badges15 silver badges28 bronze badges

answered Mar 28, 2017 at 17:49

V. Gai

5004 gold badges10 silver badges32 bronze badges

Sign up to request clarification or add additional context in comments.

4 Comments

Suge Over a year ago

Your answer is the best over all the searching, but I need a c++ implementation, I can't find alternatives of the estimate_bandwidth and MeanShift in c++, could you point me out? How can I segment the found good matches to clusters in c++?Thank you so much!

V. Gai Over a year ago

Hi! I think you can find MeanShift for c++ here: docs.opencv.org/3.4.3/dc/d6b/…. And about clustring here: docs.opencv.org/2.4/modules/core/doc/clustering.html

Suge Over a year ago

That MeanShift is far different with the one in Python scikit.lean, it used to track object in camera and can not track multiple objects. While the kmeans of OpenCV needs to estimate the clusters count first, this situation made me helpless.

Suge Over a year ago

I post my trying to this thread, would you please take a look at it? Very appreciate it, you way is much too perfect!stackoverflow.com/questions/52425355/…

V. Gai · Accepted Answer · 2017-03-25 23:26:34Z

1

To solve this task can be used next approaches:

answered Mar 25, 2017 at 23:26

V. Gai

5004 gold badges10 silver badges32 bronze badges

Comments

gurkan · Accepted Answer · 2023-12-07 18:49:20Z

Here is my approach. You can use ORB or SIFT to do the matching. After the first match, the matched area is filled with near-neighbor pixels for the next matching. This can make sure keypoints will not be occupied by the first matching.

import cv2
import numpy as np
import matplotlib.pyplot as plt

def find_matching_boxes(image, template, detector_method, params):

    # Parameters and their default values
    MAX_MATCHING_OBJECTS = params.get('max_matching_objects', 10)
    SIFT_DISTANCE_THRESHOLD = params.get('SIFT_distance_threshold', 0.5)
    BEST_MATCHES_POINTS = params.get('best_matches_points', 20)

    # Initialize the detector and matcher
    if detector_method == "SIFT":
        detector = cv2.SIFT_create()
        bf = cv2.BFMatcher()
    elif detector_method == "ORB":
        detector = cv2.ORB_create(fastThreshold=5, edgeThreshold=10)
        bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
    else:
        raise ValueError("Unsupported detector method")

    # Find keypoints and descriptors for the template
    keypoints2, descriptors2 = detector.detectAndCompute(template, None)

    matched_boxes = []
    matching_img = image.copy()

    for i in range(MAX_MATCHING_OBJECTS):
        # Match descriptors
        keypoints1, descriptors1 = detector.detectAndCompute(matching_img, None)

        if detector_method == "SIFT":
            # Matching strategy for SIFT
            matches = bf.knnMatch(descriptors1, descriptors2, k=2)
            good_matches = [m for m, n in matches if m.distance < SIFT_DISTANCE_THRESHOLD * n.distance]
            good_matches = sorted(good_matches, key=lambda x: x.distance)[:BEST_MATCHES_POINTS]

        elif detector_method == "ORB":
            # Matching strategy for ORB
            matches = bf.match(descriptors1, descriptors2)
            matches = sorted(matches, key=lambda x: x.distance)
            good_matches = matches[:BEST_MATCHES_POINTS]
        
        else:
            raise ValueError("Unsupported detector method")

        # Extract location of good matches
        points1 = np.float32([keypoints1[m.queryIdx].pt for m in good_matches])
        points2 = np.float32([keypoints2[m.trainIdx].pt for m in good_matches])

        # Find homography for drawing the bounding box
        try:
            H, _ = cv2.findHomography(points2, points1, cv2.RANSAC, 2)
        except cv2.error:
            print("No more matching box")
            break

        # Transform the corners of the template to the matching points in the image
        h, w = template.shape[:2]
        corners = np.float32([[0, 0], [0, h-1], [w-1, h-1], [w-1, 0]]).reshape(-1, 1, 2)
        transformed_corners = cv2.perspectiveTransform(corners, H)
        matched_boxes.append(transformed_corners)

        # You can uncomment the following lines to see the matching process
        # Draw the bounding box
        img1_with_box = matching_img.copy()
        matching_result = cv2.drawMatches(img1_with_box, keypoints1, template, keypoints2, good_matches, None, flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
        cv2.polylines(matching_result, [np.int32(transformed_corners)], True, (255, 0, 0), 3, cv2.LINE_AA)
        plt.imshow(matching_result, cmap='gray')
        plt.show()

        # Create a mask and fill the matched area with near neighbors
        matching_img2 = cv2.cvtColor(matching_img, cv2.COLOR_BGR2GRAY) 
        mask = np.ones_like(matching_img2) * 255
        cv2.fillPoly(mask, [np.int32(transformed_corners)], 0)
        mask = cv2.bitwise_not(mask)
        matching_img = cv2.inpaint(matching_img, mask, 3, cv2.INPAINT_TELEA)

    return matched_boxes

# Example usage:
img1 = cv2.imread('i_bast.png') # Image
img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)

template = cv2.imread('t_bast.png') # Template
template = cv2.cvtColor(template, cv2.COLOR_BGR2RGB)

params = {
    'max_matching_objects': 5,
    'SIFT_distance_threshold': 0.5,
    'best_matches_points': 20
}

# Change to "SIFT" or "ORB" depending on your requirement
matched_boxes = find_matching_boxes(img1, template, "ORB", params) 

# Draw the bounding boxes on the original image
for box in matched_boxes:
    cv2.polylines(img1, [np.int32(box)], True, (0, 255, 0), 3, cv2.LINE_AA)

plt.imshow(img1)
plt.show()

The workflow is like this:

Use the original image to do the matching:
Fill the matching area with the nearest pixels,and then repeat the matching process:
Check the result:

You can also check out my repo:
https://github.com/prob1995/multi_scale_multi_object_template_matching

If you're using my repo for this thread, make sure to modify the matching method to ORB. Like this:

# Change to "SIFT" or "ORB" depending on your requirement
matched_boxes = find_matching_boxes(img1, template, "ORB", params)

and params::

params = {
    'max_matching_objects': 5,
    'SIFT_distance_threshold': 0.5,
    'best_matches_points': 20
}

Collectives™ on Stack Overflow

OpenCV feature matching multiple objects

3 Answers 3

4 Comments

Comments

Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

4 Comments

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related