# Panoramic Stitching
###### tags: `Computer Vision` `perception`
## Getting started
```python!
%matplotlib inline
import cv2
import random
import numpy as np
import matplotlib.pyplot as plt
from google.colab.patches import cv2_imshow
]: %%capture
! wget -O img1.jpg "https://drive.google.com/uc?,→export=download&id=1omMydL6ADxq_vW5gl_1EFhdzT9kaMhUt"
! wget -O img2.jpg "https://drive.google.com/uc?,→export=download&id=12lxB1ArAlwGn97XgBgt-SFyjE7udMGvf"
```
## Visualize Input Images
```python!
img1 = plt.imread('img1.jpg')
img2 = plt.imread('img2.jpg')
def plot_imgs(img1, img2):
fig, ax = plt.subplots(1, 2, figsize=(15, 20))
for a in ax:
a.set_axis_off()
ax[0].imshow(img1)
ax[1].imshow(img2)
plot_imgs(img1, img2)
```
## a)Feature Extraction
### 1)Compute ORB Features
```python!
def get_orb_features(img):
'''
Compute ORB features using cv2 library functions.
Use default parameters when computing the keypoints.
Hint: you will need cv2.ORB_create() and some related functions
Input:
img: cv2 image
Returns:
keypoints: a list of cv2 keypoints
descriptors: a list of ORB descriptors
'''
orb = cv2.ORB_create()
keypoints, descriptors = orb.detectAndCompute(img, None)
return keypoints, descriptors
```
### 2)Match Keypoints
```python!
def match_keypoints(desc_1, desc_2, ratio=0.75):
'''
Compute matches between feature descriptors of two images using
Lowe's ratio test. You may use cv2 library functions.
Hint: you may need to use cv2.DescriptorMatcher_create or cv2.BFMatcher
and some related functions
Input:
desc_1, desc_2: list of feature descriptors
Return:
matches: list of feature matches
'''
bf = cv2.BFMatcher_create()
matche = bf.knnMatch(desc_1,desc_2,k=2)
# Apply ratio test
matches = []
matches_points = []
for m,n in matche:
if m.distance < ratio * n.distance:
matches_points.append((m.trainIdx, m.queryIdx))
matches.append(m)
return matches, matches_points
```
```python!
kp_1, desc_1 = get_orb_features(img1)
kp_2, desc_2 = get_orb_features(img2)
kp_img1 = cv2.drawKeypoints(img1, kp_1, None, color=(0,255,0), flags=0)
kp_img2 = cv2.drawKeypoints(img2, kp_2, None, color=(0,255,0), flags=0)
print('keypoints for img1 and img2')
plot_imgs(kp_img1, kp_img2)
matches, matches_points = match_keypoints(desc_1, desc_2)
match_plot = cv2.drawMatches(img1, kp_1, img2, kp_2, matches[:4], None, flags=2)
print("orb feature matches")
cv2_imshow(match_plot)
```
## b) Find Homography Matrix
```python!
def find_homography(pts_1, pts_2):
'''
Use either nonlinear least squares or direct linear transform
to find a homography that estimates the transformation mapping from pts_1
to pts_2.
e.g. If x is in pts_1 and y is in pts_2, then y = H * x
Hint if using nonlinear least square:
The objective function to optimize here is:
||pts_1 - cart(H*homog(pts_2))||^2 where homog(x) converts x into
homogeneous coordinates and cart(x) converts x to cartesian coordinates.
You can use scipy.optimize.least_squares for this.
Hint if using direct linear transform:
The solution is given by the right-singular vector with the smallest singular value in the singular vector decomposition.
You can use np.linalg.svd for this.
Input:
pts_1, pts_2: (N, 2) matrix
Return:
H: the resultant homography matrix (3 x 3)
'''
P = []
N = len(pts_1)
# try
src_pts = pts_2
dst_pts = pts_1
x1 = []
y1 = []
# calculate homography matrix
for i in range(0,N):
x1 = (src_pts[i][0], dst_pts[i][0])
y1 = (src_pts[i][1], dst_pts[i][1])
P.append([-x1[0], -y1[0], -1, 0, 0, 0, x1[0] * x1[1], y1[0] * x1[1], x1[1]])
P.append([0, 0, 0, -x1[0], -y1[0], -1, x1[0] * y1[1], y1[0] * y1[1], y1[1]])
# print("P",P)
[U, S, Vt] = np.linalg.svd(P)
H = Vt[-1].reshape(3, 3)
H /= H[2][2]
return H
```
## c)Implement RANSAC
```python!
def transform_ransac(pts_1, pts_2, verbose=False):
'''
Implements RANSAC to estimate homography matrix.
Hint: Follow the RANSAC steps outlined in the lecture slides.
Hint: Try num iterations = 1000 (not mandatory)
Hint: Threshold ε =2 (ε here refers to the L2 distance between two points)
Input:
pts_1, pts_2: (N, 2) matrices
Return:
best_model: homography matrix with most inliers
'''
n_iterations = 10
threshold = 172
final_inliers = []
final_f = None
best_model = None
inliners_best = 0
# best_dist = float('inf')
inliers = 0
n_rows = pts_1.shape[0]
for itr in range(n_iterations):
random_indices = np.random.choice(n_rows, size=4)
pts1_4 = pts_1[random_indices, :]
pts2_4 = pts_2[random_indices, :]
H = find_homography(pts_1, pts_2)
# print("pts1_4",pts1_4)
# src = [src_pts 1]
for j in range(n_rows):
# print("pts_2[j,:]",pts_2[j,:])
# print("pts2_4",pts2_4)
# src = np.pad(pts2_4, [(0, 0), (0, 1)], constant_values=1)
src = np.hstack((pts_1[j,:], np.ones(1)))
# print("src",src)
# pts = H * src
# print("pts1_4",pts1_4)
pts_pred = np.dot(H, src.T).T
# print("pts_pred",pts_pred[:2])
# normalize and throw z=1
# pts1_pred = pts_pred[:2]*pts_pred[-1]
pts_pred = (pts_pred / pts_pred[-1].reshape(-1, 1))[:, 0:2]
# print("pts_pred",pts_pred)
# print("pts_1[j,:]",pts_1[j,:])
# print("pts_1[j,:]",pts_1[j,:])
dist = np.linalg.norm(pts_pred - pts_1[j,:])
# dist = np.sqrt(np.sum(np.square(pts_pred - pts_2[j,:]), axis=1))
# print("dist",dist)
# print("distvec",distvec)
# dist = np.mean(distvec[distvec < 172])
if abs(dist) < threshold and dist!=np.nan:
print("dist",dist)
inliers+=1
final_inliers.append(j)
# print("j",j)
# inlier = np.count_nonzero(distvec < threshold)
# if len(indices) > inliners_best:
# inliners_best = len(indices)
# final_indices = indices
# best_model = H_4
# print("final_inliers",final_inliers)
pts1 = pts_1[final_inliers, :]
pts2 = pts_2[final_inliers, :]
best_model = find_homography(pts1, pts2)
print("pts1",pts1)
print("pts2",pts2)
# if inlier > inliners_best or (inlier is inliners_best and dist < best_dist):
# best_inlier = inlier
# best_dist = dist
# best_model = H_4
return best_model
```
## d) Panoramic Stitching
```python!
def panoramic_stitching(img1, img2):
'''
Given a pair of overlapping images, generate a panoramic image.
Hint: use the functions that you've written in the previous parts.
Input:
img1, img2: cv2 images
Return:
final_img: cv2 image of panorama
'''
kp_1, desc_1 = get_orb_features(img1)
kp_2, desc_2 = get_orb_features(img2)
matches, matches_points = match_keypoints(desc_1, desc_2)
matches_points = matches_points
# print('matches_points',matches_points)
image1_kp = np.float32(
[kp_1[i].pt for (_, i) in matches_points])
image2_kp = np.float32(
[kp_2[i].pt for (i, _) in matches_points])
# print('image1_kp',image1_kp)
# print('image2_kp',image2_kp)
# H, status = cv2.findHomography(image2_kp,image1_kp, cv2.RANSAC,5.0)
# H, status = cv2.findHomography(image2_kp,image1_kp, cv2.RANSAC,0.5)
H_my = transform_ransac(image2_kp, image1_kp)
H_onlyh = find_homography(image2_kp, image1_kp)
# print("H_onlyh",H_onlyh)
print("H_my",H_my)
# print('H',H)
# apply perspective wrap to stitch images together
final_img = cv2.warpPerspective(img2, H_my, (img2.shape[1] + img1.shape[1], img2.shape[0] * 2))
final_img[0:img1.shape[0], 0:img1.shape[1]] = img1
return final_img
```
## Result
```python!
result = panoramic_stitching(img1, img2)
cv2_imshow(result)
```
Sign in with Wallet
Connect another wallet