1. auto trim to sam length, 2. change png to jpg, 3. support skip mot…

…ion segmentation

.gitignore

@@ -260,12 +260,14 @@ paket-files/
 __pycache__/
 *.pyc
 
-
 *.png
+*.jpg
 *.mat
 *.asv
 *.jpg
 *.bmp
 *.mat
 *.dll
 *.exe
+~case-cuhk_lib/
+case*

README.md

@@ -6,37 +6,39 @@ Demo code for the works below:
 
 [Video Demo](https://youtu.be/IDzFvqRb40Y) on YouTube:
 
-[![Video Demo](https://img.youtube.com/vi/IDzFvqRb40Y/0.jpg)](https://youtu.be/IDzFvqRb40Y) 
+[![Video Demo](https://img.youtube.com/vi/IDzFvqRb40Y/0.jpg)](https://youtu.be/IDzFvqRb40Y)
 
 ## Dataset
 ___Including video clips from previous works, see Readme.MD for more details)___
 
-[Google Drive Link](https://drive.google.com/drive/folders/11ybVIQCOj-ojEYLvW85ZtqI4I_8Zd9lI?usp=sharing) 
+[Google Drive Link](https://drive.google.com/drive/folders/11ybVIQCOj-ojEYLvW85ZtqI4I_8Zd9lI?usp=sharing)
+
 [BaiduYun](https://pan.baidu.com/s/1bZd1k6#list/path=%2F)
 
+***Notes on your own test data:** The proposed method is using traditional SIFT + Mesh-based warping to conduct stitching, which cannot handle the case where the parallax is too large, or the light condition is too different. To put it in a simple way, if a single pair of frames from the input video pair can't be stitched by Content-Preserve Warping, then this video stitching method will also fail.
 
 ## External libraries and code used:
  - Shankar Rao's Motion Segmentation Code: http://perception.csl.illinois.edu/coding/motion/#Software
  - CVX: http://cvxr.com/cvx/
  - vlfeat: http://www.vlfeat.org/
  - peter kovesi matlab toolbox: http://www.peterkovesi.com/matlabfns/
  - Liu Shuaicheng's As-similar-as-possible Warping code: http://www.liushuaicheng.org/SIGGRAPH2013/index.htm
- 
+
 ## How to use this demo code:
 
-1. In the folder `/case-cuhk_lib`, extract video frames of ***case17-l.mp4*** to folder `/left`, and extract video frames of ***case17-r.mp4*** to folder `/right`. After the frame extracation, each folder should contain 400 png files. The file names should be indexed properly. (e.g. ***001.png 002.png*** ...)
+1. In the folder `/case-cuhk_lib`, extract video frames of ***case17-l.mp4*** to folder `/left`, and extract video frames of ***case17-r.mp4*** to folder `/right`. After the frame extracation, each folder should contain 400 jpg files. The file names should be indexed properly. (e.g. ***001.jpg 002.jpg*** ...). A script `video2frames.m` is provided. Modify the filename
 
-2. You may need to install `CVX` if you have not. 
+2. You may need to install `CVX` if you have not.
 
-3. Set MATLAB path to `/Stitching-1.1.0`, run `RunStitching.m`. The generated output frames will be in the auto-created subfolder under `/case-cuhk_lib`. (`res_demo` if you didn't change the output path). 
+3. Set MATLAB path to `/Stitching-1.1.0`, run `RunStitching.m`. The generated output frames will be in the auto-created subfolder under `/case-cuhk_lib`. (`res_demo` if you didn't change the output path). **Warning:** the motion segmentation code from Shankar Rao is very slow, if your video doesn't have large foreground object, set `SKIP_BACKGROUND_SEGMENTATION = true` in `RunStitching.m` and treat all trajectories as background. For further details, see the code comments.
 
-4. Build the OpenCV project in `/SeamCut` (You need to set OpenCV's include and library path manually) and copy the executable (e.g. `SeamCut.exe`) to the folder containing the output frames (***1.png***, ***A1.png***, ***B1.png***, ***2.png***, ***A2.png***, ***B2.png***, ...). This program finds the continuous optimal seam by GraphCut algorithm and use OpenCV's multi-band blending function. 
+4. Build the OpenCV project in `/SeamCut` (You need to set OpenCV's include and library path manually) and copy the executable (e.g. `SeamCut.exe`) to the folder containing the output frames (***1.jpg***, ***A1.jpg***, ***B1.jpg***, ***2.jpg***, ***A2.jpg***, ***B2.jpg***, ...). This program finds the continuous optimal seam by GraphCut algorithm and use OpenCV's multi-band blending function.
 
-5. Run `./SeamCut 5 1 400 1 0.2` to see the final result. Blended frames are saved as ***D1.png***, ***D2.png***, ...
+5. Run `./SeamCut 5 1 400 1 0.2` to see the final result. Blended frames are saved as ***D1.jpg***, ***D2.jpg***, ...
 
 For more details, please read the comments
 
-## Please cite our papers:
+## Please cite our papers if you use the code or our dataset.
 
 @article{nie2018dynamic,
   title={Dynamic Video Stitching via Shakiness Removing},
@@ -57,6 +59,6 @@ For more details, please read the comments
   year={2016},
   organization={ACM}
 }
- 
-Code by Tan Su, Zhensong Zhang and Yongwei Nie. For research purpose ONLY. 
+
+Code by Tan Su, Zhensong Zhang and Yongwei Nie. For research purpose ONLY.
 [![996.icu](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu)

SeamCut/SeamCut.vcxproj.filters

@@ -11,7 +11,7 @@
     </Filter>
     <Filter Include="Resource Files">
       <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
-      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav</Extensions>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;jpg;wav</Extensions>
     </Filter>
   </ItemGroup>
   <ItemGroup>

SeamCut/main.cpp

@@ -42,7 +42,7 @@ Mat FillHoles(const Mat& src)
 	//assume input is uint8 B & W (0 or 1)
 	//this function imitates imfill(image,'hole')
 	Mat src1 = src.clone();
-	
+
 	Mat dst = Mat::zeros(src.rows, src.cols, CV_8UC1);
 
 	std::vector<std::vector<Point> > contours;
@@ -64,7 +64,7 @@ Mat FindBoundaries(Mat src)
 	erode(src1, eroded,Mat());
 	absdiff(src1, eroded, dst);
 
-	return dst;	
+	return dst;
 }
 
 void process_err(char *errInfo)
@@ -87,11 +87,11 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 	Mat edge0, edge1;
 	Mat A0sMask, B0sMask, A1sMask, B1sMask;
 
-	sprintf(filename, "%s/A%d.png", srcFolder, start);
+	sprintf(filename, "%s/A%d.jpg", srcFolder, start);
 	A1 = imread(filename);
 	resize(A1, A1s, Size(), scale, scale);
 	getEdgeMap(50, A1s, A1e);
-	sprintf(filename, "%s/B%d.png", srcFolder, start);
+	sprintf(filename, "%s/B%d.jpg", srcFolder, start);
 	B1 = imread(filename);
 	resize(B1, B1s, Size(), scale, scale);
 	getEdgeMap(50, B1s, B1e);
@@ -128,11 +128,11 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 		A0sMask = A1sMask;
 		B0sMask = B1sMask;
 
-		sprintf(filename, "%s/A%d.png", srcFolder, frameIndex + 1);
+		sprintf(filename, "%s/A%d.jpg", srcFolder, frameIndex + 1);
 		A1 = imread(filename);
 		resize(A1, A1s, Size(), scale, scale);
 		getEdgeMap(50, A1s, A1e);
-		sprintf(filename, "%s/B%d.png", srcFolder, frameIndex + 1);
+		sprintf(filename, "%s/B%d.jpg", srcFolder, frameIndex + 1);
 		B1 = imread(filename);
 		resize(B1, B1s, Size(), scale, scale);
 		getEdgeMap(50, B1s, B1e);
@@ -150,15 +150,15 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 		Mat imgAll = Mat::zeros(A0sMask.size(), CV_8UC1);
 		bitwise_or(A0sMask, B0sMask, imgAll);
 
-		//imwrite("D:/test.png", imgAll);
+		//imwrite("D:/test.jpg", imgAll);
 
 
 		edge0 = A0e + B0e;
 		edge1 = A1e + B1e;
 
-		//sprintf_s(filename, "%s/edge0.png", tmpFolder);
+		//sprintf_s(filename, "%s/edge0.jpg", tmpFolder);
 		//imwrite(filename, edge0);
-		//sprintf_s(filename, "%s/edge1.png", tmpFolder);
+		//sprintf_s(filename, "%s/edge1.jpg", tmpFolder);
 		//imwrite(filename, edge1);
 
 
@@ -208,7 +208,7 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 							g.add_tweights(idx0, 0, INT_MAX);
 						}
 					}
-					
+
 				}
 			}
 		}
@@ -258,7 +258,7 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 				}
 
 				// Add back edge
-				//norm(a0, b1) + norm(a1, b0) + 
+				//norm(a0, b1) + norm(a1, b0) +
 				double cap3 = weight * (norm(a0, a1) + A0e.at<uchar>(y, x) + norm(b0, b1) + B0e.at<uchar>(y, x) + norm(a0, b1) + A1e.at<uchar>(y, x) + norm(a1, b0) + B1e.at<uchar>(y, x)) * 0.01;
 				g.add_edge(idx0, idx1, (int)(weight * (norm(a0, b1) + A0e.at<uchar>(y, x) + B1e.at<uchar>(y, x)) * 0.02 * stable), (int)(weight * (norm(a1, b0) + A1e.at<uchar>(y, x) + B0e.at<uchar>(y, x)) * 0.02 * stable));
 
@@ -269,14 +269,14 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 		std::cout << frameIndex << "\tmax flow: " << flow << endl;
 
 
-		
+
 
 		graphcut = A0.clone();
 		halfA = A0.clone();
 		halfB = B0.clone();
 		halfA_mask = Mat::zeros(A0.rows, A0.cols, CV_8U);
 		halfB_mask = halfA_mask.clone();
-		
+
 		namedWindow("Result", WINDOW_AUTOSIZE);
 
 		if (frameIndex == start)
@@ -315,15 +315,15 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 					}
 				}
 			}
-			//sprintf(filename, "%s/C%d.png", trgFolder, frameIndex);
+			//sprintf(filename, "%s/C%d.jpg", trgFolder, frameIndex);
 			//imwrite(filename, graphcut);
-			sprintf(filename, "%s/C%dA.png", trgFolder, frameIndex);
+			sprintf(filename, "%s/C%dA.jpg", trgFolder, frameIndex);
 			imwrite(filename, halfA);
-			sprintf(filename, "%s/C%dB.png", trgFolder, frameIndex);
+			sprintf(filename, "%s/C%dB.jpg", trgFolder, frameIndex);
 			imwrite(filename, halfB);
-			/*sprintf(filename, "../C%dA_m.png", frameIndex);
+			/*sprintf(filename, "../C%dA_m.jpg", frameIndex);
 			imwrite(filename, halfA_mask);
-			sprintf(filename, "../C%dB_m.png", frameIndex);
+			sprintf(filename, "../C%dB_m.jpg", frameIndex);
 			imwrite(filename, halfB_mask);*/
 			Ptr<detail::Blender> blender;
 			blender = detail::Blender::createDefault(detail::Blender::MULTI_BAND, true);
@@ -335,7 +335,7 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 			Mat res, res_mask;
 			blender->blend(res, res_mask);
 			res.convertTo(res, CV_8UC3);
-			std::sprintf(filename, "%s/D%d.png", trgFolder, frameIndex);
+			std::sprintf(filename, "%s/D%d.jpg", trgFolder, frameIndex);
 			imwrite(filename, res);
 			imshow("result", res); waitKey(1);
 		}
@@ -371,19 +371,19 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 				}
 			}
 		}
-		//sprintf(filename, "%s/mask%d.png", trgFolder, frameIndex);
+		//sprintf(filename, "%s/mask%d.jpg", trgFolder, frameIndex);
 		//imwrite(filename, saveMask);
 
 
-		//sprintf(filename, "%s/C%d.png", trgFolder, frameIndex + 1);
+		//sprintf(filename, "%s/C%d.jpg", trgFolder, frameIndex + 1);
 		//imwrite(filename, graphcut);
-		sprintf(filename, "%s/C%dA.png", trgFolder, frameIndex + 1);
+		sprintf(filename, "%s/C%dA.jpg", trgFolder, frameIndex + 1);
 		imwrite(filename, halfA);
-		sprintf(filename, "%s/C%dB.png", trgFolder, frameIndex + 1);
+		sprintf(filename, "%s/C%dB.jpg", trgFolder, frameIndex + 1);
 		imwrite(filename, halfB);
-		//sprintf(filename, "../C%dA_m.png", frameIndex + 1);
+		//sprintf(filename, "../C%dA_m.jpg", frameIndex + 1);
 		//imwrite(filename, halfA_mask);
-		//sprintf(filename, "../C%dB_m.png", frameIndex + 1);
+		//sprintf(filename, "../C%dB_m.jpg", frameIndex + 1);
 		//imwrite(filename, halfB_mask);
 		Ptr<detail::Blender> blender;
 		blender = detail::Blender::createDefault(detail::Blender::MULTI_BAND, true);
@@ -395,7 +395,7 @@ void findSeamAndBlend(int stable, int start, int end, int ab, float scale)
 		Mat res, res_mask;
 		blender->blend(res, res_mask);
 		res.convertTo(res, CV_8UC3);
-		sprintf(filename, "%s/D%d.png", trgFolder, frameIndex + 1);
+		sprintf(filename, "%s/D%d.jpg", trgFolder, frameIndex + 1);
 		imwrite(filename, res);
 		cv::imshow("result", res); waitKey(1);
 	}
@@ -418,5 +418,5 @@ int main(int argc, char **argv)
 		cout << "Usage: ./SeamCut [1-5] start end [01] scale" << endl;
 	}
 	return 0;
-	
+
 }

Stitching-1.1.0/GetGraph.m

@@ -1,6 +1,4 @@
 function [Path, graph, goodA, goodB] = GetGraph( trackA, trackB, CP, ppf, alpha, beta)
-%UNTITLED2 Summary of this function goes here
-%   Detailed explanation goes here
     nWindow = trackA.nWindow;
     nLabel = trackA.nLabel;
     videoH = trackA.videoHeight;
@@ -82,6 +80,9 @@
             deltaEdges = zeros(wSize, nLabel, nLabel);
             for fIndex = 1:wSize
                 frameIndex = left + fIndex - 1;
+                if (frameIndex > trackA.nFrame || frameIndex > trackB.nFrame) 
+                   break;
+                end
                 labelMasks = zeros(videoH, videoW, nLabel, 2);
                 fprintf('%4d', frameIndex);
                 if frameIndex < left + wSize / 2

Stitching-1.1.0/PrintBackground.m

@@ -11,25 +11,25 @@ function PrintBackground( input, tracks, backList, savePath )
         error('frame number not match');
     end
     for frameIndex = 1:nFrames
-        fileName = fileList(frameIndex).name;         
-        frame = imread([input fileName]);        
+        fileName = fileList(frameIndex).name;
+        frame = imread([input fileName]);
         hasPoints = tracks.head <= frameIndex & tracks.tail >= frameIndex;
         backPoints = hasPoints & backList == 1;
         forePoints = hasPoints & backList == -1;
-        
-        backPos = squeeze(tracks.points(backPoints, frameIndex, :));           
+
+        backPos = squeeze(tracks.points(backPoints, frameIndex, :));
         forePos = squeeze(tracks.points(forePoints, frameIndex, :));
         if size(backPos, 2) == 1
-            backPos = backPos'; 
+            backPos = backPos';
         end
         if size(forePos, 2) == 1
-            forePos = forePos'; 
+            forePos = forePos';
         end
         frame = DrawFeature(frame, backPos, 'cyan', 'o');
         frame = DrawFeature(frame, forePos, 'red', 'o');
-%         frame = insertMarker(frame, backPos, 's', 'color', 'green');        
-%         frame = insertMarker(frame, forePos, 's', 'color', 'red');        
-        imwrite(frame, [savePath int2str(frameIndex) '.png']);
+%         frame = insertMarker(frame, backPos, 's', 'color', 'green');
+%         frame = insertMarker(frame, forePos, 's', 'color', 'red');
+        imwrite(frame, [savePath int2str(frameIndex) '.jpg']);
 %         imshow(frame);
     end
 

Stitching-1.1.0/PrintFeature.m

@@ -25,8 +25,8 @@ function PrintFeature( input_A, input_B, all, back, outPath)
         if ~exist(outPath, 'dir')
             mkdir(outPath);
         end
-        imwrite(IA, [outPath '/A' int2str(frameIndex) '.png']);
-        imwrite(IB, [outPath '/B' int2str(frameIndex) '.png']);
+        imwrite(IA, [outPath '/A' int2str(frameIndex) '.jpg']);
+        imwrite(IB, [outPath '/B' int2str(frameIndex) '.jpg']);
     end
 end
 

Stitching-1.1.0/PrintLabel.m

@@ -1,6 +1,4 @@
-function PrintLabel( input, tracks, savePath, backList)
-%UNTITLED2 Summary of this function goes here
-%   Detailed explanation goes here
+function PrintLabel( input, tracks, savePath)
     fileList = dir(input);
     fileList = fileList(3:length(fileList));
     nFrames = length(fileList);
@@ -13,15 +11,15 @@ function PrintLabel( input, tracks, savePath, backList)
     colors = {'black', 'red', 'green', 'magenta', 'blue', 'cyan', 'yellow', 'blue', 'cyan', 'yellow'};
     shapes = {'x', 'o', '+', '*', 's', 'o', '+', '*', 's', 'o'};
     label_num = 9;
-    parfor frameIndex = 1:nFrames
+    for frameIndex = 1:nFrames
         fileName = fileList(frameIndex).name;
-        frame = imread([input fileName]); 
+        frame = imread([input fileName]);
         hasPoint = tracks.points(:, frameIndex, 1) ~= 0 | tracks.points(:, frameIndex, 2) ~= 0;
         points = squeeze(tracks.points(hasPoint, frameIndex, :));
         labels = squeeze(tracks.labels(hasPoint, frameIndex, :));
         I1 = frame;
         I2 = frame;
-        for labelIndex = 1:label_num        
+        for labelIndex = 1:label_num
             pos = squeeze(points(labels(:, 1) == labelIndex, :));
 %             I1 = insertMarker(I1, pos, shapes{labelIndex+1}, 'color', colors{labelIndex+1});
             I1 = DrawFeature(I1, pos, colors{labelIndex+1}, 'o');
@@ -31,8 +29,8 @@ function PrintLabel( input, tracks, savePath, backList)
         end
 %         figure(1); imshow(I1);
 %         figure(2); imshow(I2);
-        imwrite(I1, [savePath int2str(frameIndex) '_1.png']);
-        imwrite(I2, [savePath int2str(frameIndex) '_2.png']);
+        imwrite(I1, [savePath int2str(frameIndex) '_1.jpg']);
+        imwrite(I2, [savePath int2str(frameIndex) '_2.jpg']);
     end
 
 end