diff --git a/Prj-Win/Prj-Win/Prj-Win.vcxproj b/Prj-Win/Prj-Win/Prj-Win.vcxproj index 35affa2..1e8cc08 100644 --- a/Prj-Win/Prj-Win/Prj-Win.vcxproj +++ b/Prj-Win/Prj-Win/Prj-Win.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -22,32 +22,32 @@ {69FAD143-D7C9-4804-A186-90254BD80549} Win32Proj PrjWin - 8.1 + 10.0.17763.0 Application true - v140 + v141 Unicode Application false - v140 + v141 true Unicode Application true - v140 + v141 Unicode Application false - v140 + v141 true Unicode @@ -76,9 +76,9 @@ true - D:\Prj-Win\lpr\include;D:\opencv\build\include\opencv2;D:\opencv\build\include\opencv;D:\opencv\build\include;$(IncludePath) - D:\opencv\build\x64\vc14\lib;$(LibraryPath) - $(SolutionDir)Build + D:\Prj-Win\lpr\include;D:\opencv\build\include;D:\opencv\build\include\opencv2;$(IncludePath) + D:\opencv\build\x64\vc15\lib;$(LibraryPath) + $(SolutionDir)Build\ false @@ -112,7 +112,7 @@ Console true - opencv_world330d.lib;%(AdditionalDependencies) + opencv_world400d.lib;opencv_world400.lib;%(AdditionalDependencies) @@ -161,15 +161,18 @@ + + + diff --git a/Prj-Win/Prj-Win/Prj-Win.vcxproj.filters b/Prj-Win/Prj-Win/Prj-Win.vcxproj.filters index cea3788..94d12dd 100644 --- a/Prj-Win/Prj-Win/Prj-Win.vcxproj.filters +++ b/Prj-Win/Prj-Win/Prj-Win.vcxproj.filters @@ -48,6 +48,9 @@ 源文件 + + 头文件 + @@ -71,5 +74,11 @@ 源文件\test + + 源文件 + + + 源文件 + \ No newline at end of file diff --git a/Prj-Win/lpr/include/CNNRecognizer.h b/Prj-Win/lpr/include/CNNRecognizer.h index ad491a0..a577733 100644 --- a/Prj-Win/lpr/include/CNNRecognizer.h +++ b/Prj-Win/lpr/include/CNNRecognizer.h @@ -1,5 +1,5 @@ // -// Created by 庾金科 on 21/10/2017. +// Created by Jack Yu on 21/10/2017. // #ifndef SWIFTPR_CNNRECOGNIZER_H diff --git a/Prj-Win/lpr/include/Pipeline.h b/Prj-Win/lpr/include/Pipeline.h index 53e4580..ed078b1 100644 --- a/Prj-Win/lpr/include/Pipeline.h +++ b/Prj-Win/lpr/include/Pipeline.h @@ -1,85 +1,60 @@ -// -// Created by �׽�� on 22/10/2017. -// - -#ifndef SWIFTPR_PIPLINE_H -#define SWIFTPR_PIPLINE_H - -#include "PlateDetection.h" -#include "PlateSegmentation.h" -#include "CNNRecognizer.h" -#include "PlateInfo.h" -#include "FastDeskew.h" -#include "FineMapping.h" -#include "Recognizer.h" - -namespace pr{ - class PipelinePR{ - public: - GeneralRecognizer *generalRecognizer; - PlateDetection *plateDetection; - PlateSegmentation *plateSegmentation; - FineMapping *fineMapping; - PipelinePR(std::string detector_filename, - std::string finemapping_prototxt, std::string finemapping_caffemodel, - std::string segmentation_prototxt, std::string segmentation_caffemodel, - std::string charRecognization_proto, std::string charRecognization_caffemodel - ) { - plateDetection = new PlateDetection(detector_filename); - fineMapping = new FineMapping(finemapping_prototxt, finemapping_caffemodel); - plateSegmentation = new PlateSegmentation(segmentation_prototxt, segmentation_caffemodel); - generalRecognizer = new CNNRecognizer(charRecognization_proto, charRecognization_caffemodel); - } - - ~PipelinePR() { - - delete plateDetection; - delete fineMapping; - delete plateSegmentation; - delete generalRecognizer; - } - - std::vector chars_code{ "京","沪","津","渝","冀","晋","蒙","辽","吉","黑","苏","浙","皖","闽","赣","鲁","豫","鄂","湘","粤","桂","琼","川","贵","云","藏","陕","甘","青","宁","新","0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","J","K","L","M","N","P","Q","R","S","T","U","V","W","X","Y","Z" }; - - std::vector plateRes; - std::vector RunPiplineAsImage(cv::Mat plateImage) { - std::vector results; - std::vector plates; - plateDetection->plateDetectionRough(plateImage, plates); - - for (pr::PlateInfo plateinfo : plates) { - - cv::Mat image_finemapping = plateinfo.getPlateImage(); - image_finemapping = fineMapping->FineMappingVertical(image_finemapping); - image_finemapping = pr::fastdeskew(image_finemapping, 5); - image_finemapping = fineMapping->FineMappingHorizon(image_finemapping, 2, 5); - cv::resize(image_finemapping, image_finemapping, cv::Size(136, 36)); - plateinfo.setPlateImage(image_finemapping); - std::vector rects; - plateSegmentation->segmentPlatePipline(plateinfo, 1, rects); - plateSegmentation->ExtractRegions(plateinfo, rects); - cv::copyMakeBorder(image_finemapping, image_finemapping, 0, 0, 0, 20, cv::BORDER_REPLICATE); - - plateinfo.setPlateImage(image_finemapping); - generalRecognizer->SegmentBasedSequenceRecognition(plateinfo); - plateinfo.decodePlateNormal(chars_code); - results.push_back(plateinfo); - std::cout << plateinfo.getPlateName() << std::endl; - - - } - - // for (auto str:results) { - // std::cout << str << std::endl; - // } - return results; - } - - - - - }; - - -} -#endif //SWIFTPR_PIPLINE_H +// +// Created by ׽ on 22/10/2017. +// + +#ifndef SWIFTPR_PIPLINE_H +#define SWIFTPR_PIPLINE_H + +#include "PlateDetection.h" +#include "PlateSegmentation.h" +#include "CNNRecognizer.h" +#include "PlateInfo.h" +#include "FastDeskew.h" +#include "FineMapping.h" +#include "Recognizer.h" +#include "SegmentationFreeRecognizer.h" + +namespace pr{ + + const std::vector CH_PLATE_CODE{"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "³", "ԥ", "", "", "", "", + "", "", "", "", "", "", "", "", "", "", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", + "B", "C", "D", "E", "F", "G", "H", "J", "K", "L", "M", "N", "P", "Q", "R", "S", "T", "U", "V", "W", "X", + "Y", "Z","","ѧ","ʹ","","","","","","","","","","","","","","",""}; + + + + const int SEGMENTATION_FREE_METHOD = 0; + const int SEGMENTATION_BASED_METHOD = 1; + + class PipelinePR{ + public: + GeneralRecognizer *generalRecognizer; + PlateDetection *plateDetection; + PlateSegmentation *plateSegmentation; + FineMapping *fineMapping; + SegmentationFreeRecognizer *segmentationFreeRecognizer; + + PipelinePR(std::string detector_filename, + std::string finemapping_prototxt,std::string finemapping_caffemodel, + std::string segmentation_prototxt,std::string segmentation_caffemodel, + std::string charRecognization_proto,std::string charRecognization_caffemodel, + std::string segmentationfree_proto,std::string segmentationfree_caffemodel + ); + ~PipelinePR(); + + + + std::vector plateRes; + std::vector RunPiplineAsImage(cv::Mat plateImage,int method); + + + + + + + + }; + + +} +#endif //SWIFTPR_PIPLINE_H diff --git a/Prj-Win/lpr/include/PlateDetection.h b/Prj-Win/lpr/include/PlateDetection.h index f4cf567..71ad9af 100644 --- a/Prj-Win/lpr/include/PlateDetection.h +++ b/Prj-Win/lpr/include/PlateDetection.h @@ -6,7 +6,7 @@ #define SWIFTPR_PLATEDETECTION_H #include -#include "PlateInfo.h" +#include #include namespace pr{ class PlateDetection{ diff --git a/Prj-Win/lpr/include/PlateInfo.h b/Prj-Win/lpr/include/PlateInfo.h index ee1e5da..f500bb5 100644 --- a/Prj-Win/lpr/include/PlateInfo.h +++ b/Prj-Win/lpr/include/PlateInfo.h @@ -10,17 +10,14 @@ namespace pr { typedef std::vector Character; enum PlateColor { BLUE, YELLOW, WHITE, GREEN, BLACK,UNKNOWN}; - enum CharType {CHINESE,LETTER,LETTER_NUMS}; + enum CharType {CHINESE,LETTER,LETTER_NUMS,INVALID}; class PlateInfo { public: - std::vector> plateChars; + std::vector> plateChars; std::vector> plateCoding; - float confidence = 0; - - PlateInfo(const cv::Mat &plateData, std::string plateName, cv::Rect plateRect, PlateColor plateType) { licensePlate = plateData; name = plateName; @@ -93,17 +90,21 @@ namespace pr { } - if(plate.first == LETTER) { + else if(plate.first == LETTER) { decode += mappingTable[std::max_element(prob+41,prob+65)- prob]; confidence+=*std::max_element(prob+41,prob+65); } - if(plate.first == LETTER_NUMS) { + else if(plate.first == LETTER_NUMS) { decode += mappingTable[std::max_element(prob+31,prob+65)- prob]; confidence+=*std::max_element(prob+31,prob+65); // std::cout<<*std::max_element(prob+31,prob+65)< SegmentationFreeForSinglePlate(cv::Mat plate,std::vector mapping_table); + + + private: + cv::dnn::Net net; + + }; + +} +#endif //SWIFTPR_SEGMENTATIONFREERECOGNIZER_H diff --git a/Prj-Win/lpr/include/niBlackThreshold.h b/Prj-Win/lpr/include/niBlackThreshold.h index 5ad7e14..54ac06d 100644 --- a/Prj-Win/lpr/include/niBlackThreshold.h +++ b/Prj-Win/lpr/include/niBlackThreshold.h @@ -62,8 +62,9 @@ void niBlackThreshold( InputArray _src, OutputArray _dst, double maxValue, thresh = mean + static_cast(k) * sqrtVarianceMeanSum; break; default: - CV_Error( CV_StsBadArg, "Unknown binarization method" ); - break; +// CV_Error( CV_StsBadArg, "Unknown binarization method" ); + CV_Error(-5, "Unknown binarization method"); + break; } thresh.convertTo(thresh, src.depth()); @@ -99,7 +100,8 @@ void niBlackThreshold( InputArray _src, OutputArray _dst, double maxValue, src.copyTo(dst, mask); break; default: - CV_Error( CV_StsBadArg, "Unknown threshold type" ); +// CV_Error( CV_StsBadArg, "Unknown threshold type" ); + CV_Error(-5, "Unknown threshold type"); break; } } diff --git a/Prj-Win/lpr/model/README.md b/Prj-Win/lpr/model/README.md index 2c524eb..3085edf 100644 --- a/Prj-Win/lpr/model/README.md +++ b/Prj-Win/lpr/model/README.md @@ -10,8 +10,8 @@ HorizonalFinemapping.caffemodel HorizonalFinemapping.prototxt -Segmentation.caffemodel +SegmentationFree.caffemodel -Segmentation.prototxt +SegmentationFree.prototxt 放置在该目录 \ No newline at end of file diff --git a/Prj-Win/lpr/res/test.jpg b/Prj-Win/lpr/res/test.jpg new file mode 100644 index 0000000..342a5d9 Binary files /dev/null and b/Prj-Win/lpr/res/test.jpg differ diff --git a/Prj-Win/lpr/src/CNNRecognizer.cpp b/Prj-Win/lpr/src/CNNRecognizer.cpp index 909753c..252bd7c 100644 --- a/Prj-Win/lpr/src/CNNRecognizer.cpp +++ b/Prj-Win/lpr/src/CNNRecognizer.cpp @@ -1,5 +1,5 @@ // -// Created by 庾金科 on 21/10/2017. +// Created by Jack Yu on 21/10/2017. // #include "../include/CNNRecognizer.h" @@ -16,4 +16,4 @@ namespace pr{ net.setInput(inputBlob,"data"); return net.forward(); } -} \ No newline at end of file +} diff --git a/Prj-Win/lpr/src/FastDeskew.cpp b/Prj-Win/lpr/src/FastDeskew.cpp index bf46f29..58c8eb3 100644 --- a/Prj-Win/lpr/src/FastDeskew.cpp +++ b/Prj-Win/lpr/src/FastDeskew.cpp @@ -1,19 +1,17 @@ // -// Created by 庾金科 on 02/10/2017. +// Created by Jack Yu on 02/10/2017. // -#include "FastDeskew.h" +#include <../include/FastDeskew.h> namespace pr{ - - - const int ANGLE_MIN = 30 ; + const int ANGLE_MIN = 30 ; const int ANGLE_MAX = 150 ; const int PLATE_H = 36; const int PLATE_W = 136; - + int angle(float x,float y) { return atan2(x,y)*180/3.1415; @@ -51,59 +49,38 @@ namespace pr{ cv::Mat correctPlateImage(cv::Mat skewPlate,float angle,float maxAngle) { - cv::Mat dst; - cv::Size size_o(skewPlate.cols,skewPlate.rows); - - int extend_padding = 0; -// if(angle<0) - extend_padding = static_cast(skewPlate.rows*tan(cv::abs(angle)/180* 3.14) ); -// else -// extend_padding = static_cast(skewPlate.rows/tan(cv::abs(angle)/180* 3.14) ); - -// std::cout<<"extend:"<(skewPlate.rows*tan(cv::abs(angle)/180* 3.14) ); cv::Size size(skewPlate.cols + extend_padding ,skewPlate.rows); - float interval = abs(sin((angle /180) * 3.14)* skewPlate.rows); -// std::cout<0) { cv::Point2f pts2[4] = {cv::Point2f(interval, 0), cv::Point2f(0, size_o.height), cv::Point2f(size_o.width, 0), cv::Point2f(size_o.width - interval, size_o.height)}; cv::Mat M = cv::getPerspectiveTransform(pts1,pts2); cv::warpPerspective(skewPlate,dst,M,size); - - } else { cv::Point2f pts2[4] = {cv::Point2f(0, 0), cv::Point2f(interval, size_o.height), cv::Point2f(size_o.width-interval, 0), cv::Point2f(size_o.width, size_o.height)}; cv::Mat M = cv::getPerspectiveTransform(pts1,pts2); cv::warpPerspective(skewPlate,dst,M,size,cv::INTER_CUBIC); - } return dst; } cv::Mat fastdeskew(cv::Mat skewImage,int blockSize){ - - const int FILTER_WINDOWS_SIZE = 5; std::vector angle_list(180); memset(angle_list.data(),0,angle_list.size()*sizeof(int)); - cv::Mat bak; skewImage.copyTo(bak); if(skewImage.channels() == 3) cv::cvtColor(skewImage,skewImage,cv::COLOR_RGB2GRAY); - if(skewImage.channels() == 1) { cv::Mat eigen; - cv::cornerEigenValsAndVecs(skewImage,eigen,blockSize,5); for( int j = 0; j < skewImage.rows; j+=blockSize ) { for( int i = 0; i < skewImage.cols; i+=blockSize ) @@ -112,12 +89,10 @@ namespace pr{ float y2 = eigen.at(j, i)[5]; int angle_cell = angle(x2,y2); angle_list[(angle_cell + 180)%180]+=1.0; - } } } std::vector filtered = avgfilter(angle_list,5); - int maxPos = std::max_element(filtered.begin(),filtered.end()) - filtered.begin() + FILTER_WINDOWS_SIZE/2; if(maxPos>ANGLE_MAX) maxPos = (-maxPos+90+180)%180; diff --git a/Prj-Win/lpr/src/FineMapping.cpp b/Prj-Win/lpr/src/FineMapping.cpp index 6890eff..2032b15 100644 --- a/Prj-Win/lpr/src/FineMapping.cpp +++ b/Prj-Win/lpr/src/FineMapping.cpp @@ -1,12 +1,8 @@ -// -// Created by 庾金科 on 22/09/2017. -// - #include "FineMapping.h" namespace pr{ - const int FINEMAPPING_H = 50; - const int FINEMAPPING_W = 120; + const int FINEMAPPING_H = 60 ; + const int FINEMAPPING_W = 140; const int PADDING_UP_DOWN = 30; void drawRect(cv::Mat image,cv::Rect rect) { @@ -65,31 +61,21 @@ namespace pr{ } cv::Mat FineMapping::FineMappingVertical(cv::Mat InputProposal,int sliceNum,int upper,int lower,int windows_size){ - - cv::Mat PreInputProposal; cv::Mat proposal; - cv::resize(InputProposal,PreInputProposal,cv::Size(FINEMAPPING_W,FINEMAPPING_H)); if(InputProposal.channels() == 3) cv::cvtColor(PreInputProposal,proposal,cv::COLOR_BGR2GRAY); else PreInputProposal.copyTo(proposal); - -// proposal = PreInputProposal; - // this will improve some sen cv::Mat kernal = cv::getStructuringElement(cv::MORPH_ELLIPSE,cv::Size(1,3)); -// cv::erode(proposal,proposal,kernal); - - float diff = static_cast(upper-lower); diff/=static_cast(sliceNum-1); cv::Mat binary_adaptive; std::vector line_upper; std::vector line_lower; int contours_nums=0; - for(int i = 0 ; i < sliceNum ; i++) { std::vector > contours; @@ -106,7 +92,6 @@ namespace pr{ if (( lwRatio>0.7&&bdbox.width*bdbox.height>100 && bdboxAera<300) || (lwRatio>3.0 && bdboxAera<100 && bdboxAera>10)) { - cv::Point p1(bdbox.x, bdbox.y); cv::Point p2(bdbox.x + bdbox.width, bdbox.y + bdbox.height); line_upper.push_back(p1); @@ -115,9 +100,6 @@ namespace pr{ } } } - - std:: cout<<"contours_nums "< > contours; float k =lower + i*diff; cv::adaptiveThreshold(proposal,binary_adaptive,255,cv::ADAPTIVE_THRESH_MEAN_C,cv::THRESH_BINARY,windows_size,k); -// cv::imshow("image",binary_adaptive); -// cv::waitKey(0); cv::Mat draw; binary_adaptive.copyTo(draw); cv::findContours(binary_adaptive,contours,cv::RETR_EXTERNAL,cv::CHAIN_APPROX_SIMPLE); @@ -158,30 +137,19 @@ namespace pr{ } } } -// std:: cout<<"contours_nums "< A; std::pair B; - A = FitLineRansac(line_upper, -2); - B = FitLineRansac(line_lower, 2); + A = FitLineRansac(line_upper, -1); + B = FitLineRansac(line_lower, 1); int leftyB = A.first; int rightyB = A.second; int leftyA = B.first; int rightyA = B.second; int cols = rgb.cols; int rows = rgb.rows; -// pts_map1 = np.float32([[cols - 1, rightyA], [0, leftyA],[cols - 1, rightyB], [0, leftyB]]) -// pts_map2 = np.float32([[136,36],[0,36],[136,0],[0,0]]) -// mat = cv2.getPerspectiveTransform(pts_map1,pts_map2) -// image = cv2.warpPerspective(rgb,mat,(136,36),flags=cv2.INTER_CUBIC) std::vector corners(4); corners[0] = cv::Point2f(cols - 1, rightyA); corners[1] = cv::Point2f(0, leftyA); @@ -196,10 +164,7 @@ namespace pr{ cv::Mat quad = cv::Mat::zeros(36, 136, CV_8UC3); cv::warpPerspective(rgb, quad, transform, quad.size()); return quad; - } - - } diff --git a/Prj-Win/lpr/src/Pipeline.cpp b/Prj-Win/lpr/src/Pipeline.cpp index fad9e19..95ae658 100644 --- a/Prj-Win/lpr/src/Pipeline.cpp +++ b/Prj-Win/lpr/src/Pipeline.cpp @@ -1 +1,85 @@ -//// //// Created by �׽�� on 23/10/2017. //// // //#include "../include/Pipeline.h" // // //namespace pr { // // //std::vector chars_code{ "��","��","��","��","��","��","��","��","��","��","��","��","��","��","��","³","ԥ","��","��","��","��","��","��","��","��","��","��","��","��","��","��","0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","J","K","L","M","N","P","Q","R","S","T","U","V","W","X","Y","Z" }; // // std::vector chars_code{ "京","沪","津","渝","冀","晋","蒙","辽","吉","黑","苏","浙","皖","闽","赣","鲁","豫","鄂","湘","粤","桂","琼","川","贵","云","藏","陕","甘","青","宁","新","0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","J","K","L","M","N","P","Q","R","S","T","U","V","W","X","Y","Z" }; // // // PipelinePR::PipelinePR(std::string detector_filename, // std::string finemapping_prototxt, std::string finemapping_caffemodel, // std::string segmentation_prototxt, std::string segmentation_caffemodel, // std::string charRecognization_proto, std::string charRecognization_caffemodel) { // plateDetection = new PlateDetection(detector_filename); // fineMapping = new FineMapping(finemapping_prototxt, finemapping_caffemodel); // plateSegmentation = new PlateSegmentation(segmentation_prototxt, segmentation_caffemodel); // generalRecognizer = new CNNRecognizer(charRecognization_proto, charRecognization_caffemodel); // } // // PipelinePR::~PipelinePR() { // // delete plateDetection; // delete fineMapping; // delete plateSegmentation; // delete generalRecognizer; // // } // // std::vector PipelinePR:: RunPiplineAsImage(cv::Mat plateImage) { // std::vector results; // std::vector plates; // plateDetection->plateDetectionRough(plateImage,plates); // // for (pr::PlateInfo plateinfo:plates) { // // cv::Mat image_finemapping = plateinfo.getPlateImage(); // image_finemapping = fineMapping->FineMappingVertical(image_finemapping); // image_finemapping = pr::fastdeskew(image_finemapping, 5); // image_finemapping = fineMapping->FineMappingHorizon(image_finemapping, 2, 5); // cv::resize(image_finemapping, image_finemapping, cv::Size(136, 36)); // plateinfo.setPlateImage(image_finemapping); // std::vector rects; // plateSegmentation->segmentPlatePipline(plateinfo, 1, rects); // plateSegmentation->ExtractRegions(plateinfo, rects); // cv::copyMakeBorder(image_finemapping, image_finemapping, 0, 0, 0, 20, cv::BORDER_REPLICATE); // // plateinfo.setPlateImage(image_finemapping); // generalRecognizer->SegmentBasedSequenceRecognition(plateinfo); // plateinfo.decodePlateNormal(chars_code); // results.push_back(plateinfo); // std::cout << plateinfo.getPlateName() << std::endl; // // // } // //// for (auto str:results) { //// std::cout << str << std::endl; //// } // return results; // // }//namespace pr // // // //} \ No newline at end of file +// +// Created by Jack Yu on 23/10/2017. +// + +#include "../include/Pipeline.h" + + +namespace pr { + + + + const int HorizontalPadding = 4; + PipelinePR::PipelinePR(std::string detector_filename, + std::string finemapping_prototxt, std::string finemapping_caffemodel, + std::string segmentation_prototxt, std::string segmentation_caffemodel, + std::string charRecognization_proto, std::string charRecognization_caffemodel, + std::string segmentationfree_proto,std::string segmentationfree_caffemodel) { + plateDetection = new PlateDetection(detector_filename); + fineMapping = new FineMapping(finemapping_prototxt, finemapping_caffemodel); + plateSegmentation = new PlateSegmentation(segmentation_prototxt, segmentation_caffemodel); + generalRecognizer = new CNNRecognizer(charRecognization_proto, charRecognization_caffemodel); + segmentationFreeRecognizer = new SegmentationFreeRecognizer(segmentationfree_proto,segmentationfree_caffemodel); + + } + + PipelinePR::~PipelinePR() { + + delete plateDetection; + delete fineMapping; + delete plateSegmentation; + delete generalRecognizer; + delete segmentationFreeRecognizer; + + + } + + std::vector PipelinePR:: RunPiplineAsImage(cv::Mat plateImage,int method) { + std::vector results; + std::vector plates; + plateDetection->plateDetectionRough(plateImage,plates,36,700); + + for (pr::PlateInfo plateinfo:plates) { + + cv::Mat image_finemapping = plateinfo.getPlateImage(); + image_finemapping = fineMapping->FineMappingVertical(image_finemapping); + image_finemapping = pr::fastdeskew(image_finemapping, 5); + + + + //Segmentation-based + + if(method==SEGMENTATION_BASED_METHOD) + { + image_finemapping = fineMapping->FineMappingHorizon(image_finemapping, 2, HorizontalPadding); + cv::resize(image_finemapping, image_finemapping, cv::Size(136+HorizontalPadding, 36)); + plateinfo.setPlateImage(image_finemapping); + std::vector rects; + plateSegmentation->segmentPlatePipline(plateinfo, 1, rects); + plateSegmentation->ExtractRegions(plateinfo, rects); + cv::copyMakeBorder(image_finemapping, image_finemapping, 0, 0, 0, 20, cv::BORDER_REPLICATE); + plateinfo.setPlateImage(image_finemapping); + generalRecognizer->SegmentBasedSequenceRecognition(plateinfo); + plateinfo.decodePlateNormal(pr::CH_PLATE_CODE); + + } + //Segmentation-free + else if(method==SEGMENTATION_FREE_METHOD) + { + image_finemapping = fineMapping->FineMappingHorizon(image_finemapping, 4, HorizontalPadding+3); + cv::resize(image_finemapping, image_finemapping, cv::Size(136+HorizontalPadding, 36)); + plateinfo.setPlateImage(image_finemapping); + std::pair res = segmentationFreeRecognizer->SegmentationFreeForSinglePlate(plateinfo.getPlateImage(),pr::CH_PLATE_CODE); + plateinfo.confidence = res.second; + plateinfo.setPlateName(res.first); + } + results.push_back(plateinfo); + } + + return results; + + }//namespace pr + + + +} diff --git a/Prj-Win/lpr/src/PlateDetection.cpp b/Prj-Win/lpr/src/PlateDetection.cpp index b207190..a25f4db 100644 --- a/Prj-Win/lpr/src/PlateDetection.cpp +++ b/Prj-Win/lpr/src/PlateDetection.cpp @@ -1,45 +1,24 @@ -// -// Created by 庾金科 on 20/09/2017. -// #include "../include/PlateDetection.h" - #include "util.h" - namespace pr{ - - PlateDetection::PlateDetection(std::string filename_cascade){ cascade.load(filename_cascade); }; - - void PlateDetection::plateDetectionRough(cv::Mat InputImage,std::vector &plateInfos,int min_w,int max_w){ - cv::Mat processImage; - - cv::cvtColor(InputImage,processImage,cv::COLOR_BGR2GRAY); - - + cv::cvtColor(InputImage,processImage,cv::COLOR_BGR2GRAY); std::vector platesRegions; -// std::vector plates; cv::Size minSize(min_w,min_w/4); cv::Size maxSize(max_w,max_w/4); -// cv::imshow("input",InputImage); -// cv::waitKey(0); cascade.detectMultiScale( processImage, platesRegions, 1.1, 3, cv::CASCADE_SCALE_IMAGE,minSize,maxSize); for(auto plate:platesRegions) { - // extend rects -// x -= w * 0.14 -// w += w * 0.28 -// y -= h * 0.6 -// h += h * 1.1; - int zeroadd_w = static_cast(plate.width*0.28); - int zeroadd_h = static_cast(plate.height*1.2); - int zeroadd_x = static_cast(plate.width*0.14); - int zeroadd_y = static_cast(plate.height*0.6); + int zeroadd_w = static_cast(plate.width*0.30); + int zeroadd_h = static_cast(plate.height*2); + int zeroadd_x = static_cast(plate.width*0.15); + int zeroadd_y = static_cast(plate.height*1); plate.x-=zeroadd_x; plate.y-=zeroadd_y; plate.height += zeroadd_h; @@ -50,12 +29,4 @@ namespace pr{ } } -// std::vector PlateDetection::plateDetectionRough(cv::Mat InputImage,cv::Rect roi,int min_w,int max_w){ -// cv::Mat roi_region = util::cropFromImage(InputImage,roi); -// return plateDetectionRough(roi_region,min_w,max_w); -// } - - - - }//namespace pr diff --git a/Prj-Win/lpr/src/PlateSegmentation.cpp b/Prj-Win/lpr/src/PlateSegmentation.cpp index 94da476..6c454e5 100644 --- a/Prj-Win/lpr/src/PlateSegmentation.cpp +++ b/Prj-Win/lpr/src/PlateSegmentation.cpp @@ -1,5 +1,5 @@ // -// Created by 庾金科 on 16/10/2017. +// Created by Jack Yu on 16/10/2017. // #include "../include/PlateSegmentation.h" @@ -94,7 +94,7 @@ namespace pr{ cv::Mat roi_thres; // cv::threshold(roiImage,roi_thres,0,255,cv::THRESH_OTSU|cv::THRESH_BINARY); - niBlackThreshold(roiImage,roi_thres,255,cv::THRESH_BINARY,15,0.3,BINARIZATION_NIBLACK); + niBlackThreshold(roiImage,roi_thres,255,cv::THRESH_BINARY,15,0.27,BINARIZATION_NIBLACK); std::vector> contours; cv::findContours(roi_thres,contours,cv::RETR_LIST,cv::CHAIN_APPROX_SIMPLE); @@ -110,7 +110,7 @@ namespace pr{ cv::Rect bdbox = cv::boundingRect(contour); cv::Point center(bdbox.x+(bdbox.width>>1),bdbox.y + (bdbox.height>>1)); int dist = (center.x - boxCenter.x)*(center.x - boxCenter.x); - if(dist rows>>1) + if(dist rows>>1) { final_dist =dist; final_center = center; final_bdbox = bdbox; @@ -220,7 +220,7 @@ namespace pr{ int cp_list[7]; - float loss_selected = -1; + float loss_selected = -10; for(int start = 0 ; start < 20 ; start+=2) for(int width = windowsWidth-5; width < windowsWidth+5 ; width++ ){ @@ -246,13 +246,10 @@ namespace pr{ if(cp7_p5>=cols) continue; - float loss = ch_prob[cp1_ch]+ - engNum_prob[cp2_p0] +engNum_prob[cp3_p1]+engNum_prob[cp4_p2]+engNum_prob[cp5_p3]+engNum_prob[cp6_p4] +engNum_prob[cp7_p5] - + (false_prob[md2]+false_prob[md3]+false_prob[md4]+false_prob[md5]+false_prob[md5] + false_prob[md6]); -// float loss = ch_prob[cp1_ch]*3 -(false_prob[cp3_p1]+false_prob[cp4_p2]+false_prob[cp5_p3]+false_prob[cp6_p4]+false_prob[cp7_p5]); - - - +// float loss = ch_prob[cp1_ch]+ +// engNum_prob[cp2_p0] +engNum_prob[cp3_p1]+engNum_prob[cp4_p2]+engNum_prob[cp5_p3]+engNum_prob[cp6_p4] +engNum_prob[cp7_p5] +// + (false_prob[md2]+false_prob[md3]+false_prob[md4]+false_prob[md5]+false_prob[md5] + false_prob[md6]); + float loss = ch_prob[cp1_ch]*3 -(false_prob[cp3_p1]+false_prob[cp4_p2]+false_prob[cp5_p3]+false_prob[cp6_p4]+false_prob[cp7_p5]); if(loss>loss_selected) { @@ -284,15 +281,15 @@ namespace pr{ void PlateSegmentation::segmentPlateBySlidingWindows(cv::Mat &plateImage,int windowsWidth,int stride,cv::Mat &respones){ - cv::resize(plateImage,plateImage,cv::Size(136,36)); +// cv::resize(plateImage,plateImage,cv::Size(136,36)); cv::Mat plateImageGray; cv::cvtColor(plateImage,plateImageGray,cv::COLOR_BGR2GRAY); - + int padding = plateImage.cols-136 ; +// int padding = 0 ; int height = plateImage.rows - 1; - int width = plateImage.cols - 1; - - for(int i = 0 ; i < plateImage.cols - windowsWidth +1 ; i +=stride) + int width = plateImage.cols - 1 - padding; + for(int i = 0 ; i < width - windowsWidth +1 ; i +=stride) { cv::Rect roi(i,0,windowsWidth,height); cv::Mat roiImage = plateImageGray(roi); @@ -348,6 +345,11 @@ namespace pr{ cv::Mat respones; //three response of every sub region from origin image . segmentPlateBySlidingWindows(plateImage,DEFAULT_WIDTH,1,respones); templateMatchFinding(respones,DEFAULT_WIDTH/stride,sections); + for(int i = 0; i < sections.second.size() ; i++) + { + sections.second[i]*=stride; + + } // std::cout< res; - cv::Mat code_table= recognizeCharacter(char_instance.second); - res.first = char_instance.first; - code_table.copyTo(res.second); - plateinfo.appendPlateCoding(res); - + if(char_instance.second.rows*char_instance.second.cols>40) { + label code_table = recognizeCharacter(char_instance.second); + res.first = char_instance.first; + code_table.copyTo(res.second); + plateinfo.appendPlateCoding(res); + } else{ + res.first = INVALID; + plateinfo.appendPlateCoding(res); + } } - - - } -} \ No newline at end of file +} diff --git a/Prj-Win/lpr/src/SegmentationFreeRecognizer.cpp b/Prj-Win/lpr/src/SegmentationFreeRecognizer.cpp new file mode 100644 index 0000000..369f63b --- /dev/null +++ b/Prj-Win/lpr/src/SegmentationFreeRecognizer.cpp @@ -0,0 +1,89 @@ +// +// Created by Jack Yu on 28/11/2017. +// +#include "../include/SegmentationFreeRecognizer.h" + +namespace pr { + SegmentationFreeRecognizer::SegmentationFreeRecognizer(std::string prototxt, std::string caffemodel) { + net = cv::dnn::readNetFromCaffe(prototxt, caffemodel); + } + inline int judgeCharRange(int id) + {return id<31 || id>63; + } + std::pair decodeResults(cv::Mat code_table,std::vector mapping_table,float thres) + { + cv::MatSize mtsize = code_table.size; + int sequencelength = mtsize[2]; + int labellength = mtsize[1]; + cv::transpose(code_table.reshape(1,1).reshape(1,labellength),code_table); + std::string name = ""; + std::vector seq(sequencelength); + std::vector> seq_decode_res; + for(int i = 0 ; i < sequencelength; i++) { + float *fstart = ((float *) (code_table.data) + i * labellength ); + int id = std::max_element(fstart,fstart+labellength) - fstart; + seq[i] =id; + } + + float sum_confidence = 0; + int plate_lenghth = 0 ; + for(int i = 0 ; i< sequencelength ; i++) + { + if(seq[i]!=labellength-1 && (i==0 || seq[i]!=seq[i-1])) + { + float *fstart = ((float *) (code_table.data) + i * labellength ); + float confidence = *(fstart+seq[i]); + std::pair pair_(seq[i],confidence); + seq_decode_res.push_back(pair_); + } + } + int i = 0; + if (seq_decode_res.size()>1 && judgeCharRange(seq_decode_res[0].first) && judgeCharRange(seq_decode_res[1].first)) + { + i=2; + int c = seq_decode_res[0].second res; + res.second = sum_confidence/plate_lenghth; + res.first = name; + return res; + + } + std::string decodeResults(cv::Mat code_table,std::vector mapping_table) + { + cv::MatSize mtsize = code_table.size; + int sequencelength = mtsize[2]; + int labellength = mtsize[1]; + cv::transpose(code_table.reshape(1,1).reshape(1,labellength),code_table); + std::string name = ""; + std::vector seq(sequencelength); + for(int i = 0 ; i < sequencelength; i++) { + float *fstart = ((float *) (code_table.data) + i * labellength ); + int id = std::max_element(fstart,fstart+labellength) - fstart; + seq[i] =id; + } + for(int i = 0 ; i< sequencelength ; i++) + { + if(seq[i]!=labellength-1 && (i==0 || seq[i]!=seq[i-1])) + name+=mapping_table[seq[i]]; + } + return name; + } + std::pair SegmentationFreeRecognizer::SegmentationFreeForSinglePlate(cv::Mat Image,std::vector mapping_table) { + cv::transpose(Image,Image); + cv::Mat inputBlob = cv::dnn::blobFromImage(Image, 1 / 255.0, cv::Size(40,160)); + net.setInput(inputBlob, "data"); + cv::Mat char_prob_mat = net.forward(); + return decodeResults(char_prob_mat,mapping_table,0.00); + } +} diff --git a/Prj-Win/lpr/src/util.h b/Prj-Win/lpr/src/util.h index 76a452f..67e21ce 100644 --- a/Prj-Win/lpr/src/util.h +++ b/Prj-Win/lpr/src/util.h @@ -1,20 +1,16 @@ // -// Created by 庾金科 on 04/04/2017. +// Created by Jack Yu on 04/04/2017. // #include - namespace util{ - template void swap ( T& a, T& b ) { T c(a); a=b; b=c; } - template T min(T& a,T& b ) { return a>b?b:a; - } cv::Mat cropFromImage(const cv::Mat &image,cv::Rect rect){ @@ -57,23 +53,16 @@ namespace util{ int histSize = 256; float range[] = {0,255}; const float* histRange = {range}; - cv::calcHist( &hsv_planes[0], 1, 0, cv::Mat(), hist, 1, &histSize, &histRange,true, true); return hist; - } + float computeSimilir(const cv::Mat &A,const cv::Mat &B) { - cv::Mat histA,histB; histA = calcHist(A); histB = calcHist(B); - return cv::compareHist(histA,histB,CV_COMP_CORREL); - +// return cv::compareHist(histA,histB,CV_COMP_CORREL); + return cv::compareHist(histA, histB, 0); } - - - - - }//namespace util diff --git a/Prj-Win/lpr/tests/test_fastdeskew.cpp b/Prj-Win/lpr/tests/test_fastdeskew.cpp index faa0d93..3740147 100644 --- a/Prj-Win/lpr/tests/test_fastdeskew.cpp +++ b/Prj-Win/lpr/tests/test_fastdeskew.cpp @@ -1,5 +1,5 @@ // -// Created by 庾金科 on 02/10/2017. +// Created by Jack Yu on 02/10/2017. // diff --git a/Prj-Win/lpr/tests/test_finemapping.cpp b/Prj-Win/lpr/tests/test_finemapping.cpp index 4893efd..dd7e26f 100644 --- a/Prj-Win/lpr/tests/test_finemapping.cpp +++ b/Prj-Win/lpr/tests/test_finemapping.cpp @@ -1,5 +1,5 @@ // -// Created by 庾金科 on 24/09/2017. +// Created by Jack Yu on 24/09/2017. // #include "FineMapping.h" @@ -22,4 +22,4 @@ int main() return 0 ; -} \ No newline at end of file +} diff --git a/Prj-Win/lpr/tests/test_pipeline.cpp b/Prj-Win/lpr/tests/test_pipeline.cpp index fccdf1f..647cbe7 100644 --- a/Prj-Win/lpr/tests/test_pipeline.cpp +++ b/Prj-Win/lpr/tests/test_pipeline.cpp @@ -1,42 +1,229 @@ -// -// Created by 庾金科 on 23/10/2017. +// +// Created by Jack Yu on 23/10/2017. // #include "../include/Pipeline.h" -using namespace pr; +#include +#include -void TEST_PIPELINE(){ - pr::PipelinePR prc("../lpr/model/cascade.xml", - "../lpr/model/HorizonalFinemapping.prototxt","../lpr/model/HorizonalFinemapping.caffemodel", - "../lpr/model/Segmentation.prototxt","../lpr/model/Segmentation.caffemodel", - "../lpr/model/CharacterRecognization.prototxt","../lpr/model/CharacterRecognization.caffemodel" - ); - cv::Mat image = cv::imread("../6.jpg"); - cv::imshow("image",image); - cv::waitKey(0); +using namespace std; - std::vector res = prc.RunPiplineAsImage(image); - float conf = 0 ; - for(auto st:res) { - if(st.confidence>0.1) { - std::cout << st.getPlateName() << " " << st.confidence << std::endl; - conf += st.confidence; - } - } - std::cout< +static unsigned int levenshtein_distance(const T &s1, const T &s2) { + const size_t len1 = s1.size(), len2 = s2.size(); + std::vector col(len2 + 1), prevCol(len2 + 1); + for (unsigned int i = 0; i < prevCol.size(); i++) prevCol[i] = i; + for (unsigned int i = 0; i < len1; i++) { + col[0] = i + 1; + for (unsigned int j = 0; j < len2; j++) + col[j + 1] = min( + min(prevCol[1 + j] + 1, col[j] + 1), + prevCol[j] + (s1[i] == s2[j] ? 0 : 1)); + col.swap(prevCol); + } + return prevCol[len2]; } -int main() + + +void TEST_CAM() { + cv::VideoCapture capture("test1.mp4"); + cv::Mat frame; + pr::PipelinePR prc("../lpr/model/cascade.xml", + "../lpr/model/HorizonalFinemapping.prototxt", "../lpr/model/HorizonalFinemapping.caffemodel", + "../lpr/model/Segmentation.prototxt", "../lpr/model/Segmentation.caffemodel", + "../lpr/model/CharacterRecognization.prototxt", "../lpr/model/CharacterRecognization.caffemodel", + "../lpr/model/SegmentationFree.prototxt", "../lpr/model/SegmentationFree.caffemodel" + ); + while (1) { + //读取下一帧 + if (!capture.read(frame)) { + std::cout << "读取视频失败" << std::endl; + exit(1); + } + // + // cv::transpose(frame,frame); + // cv::flip(frame,frame,2); + + // cv::resize(frame,frame,cv::Size(frame.cols/2,frame.rows/2)); + + + std::vector res = prc.RunPiplineAsImage(frame, pr::SEGMENTATION_FREE_METHOD); + + for (auto st : res) { + if (st.confidence > 0.75) { + std::cout << st.getPlateName() << " " << st.confidence << std::endl; + cv::Rect region = st.getPlateRect(); + + cv::rectangle(frame, cv::Point(region.x, region.y), cv::Point(region.x + region.width, region.y + region.height), cv::Scalar(255, 255, 0), 2); + } + } + + cv::imshow("image", frame); + cv::waitKey(1); + } +} + + +void TEST_ACC() { + + pr::PipelinePR prc("../lpr/model/cascade.xml", + "../lpr/model/HorizonalFinemapping.prototxt", "../lpr/model/HorizonalFinemapping.caffemodel", + "../lpr/model/Segmentation.prototxt", "../lpr/model/Segmentation.caffemodel", + "../lpr/model/CharacterRecognization.prototxt", "../lpr/model/CharacterRecognization.caffemodel", + "../lpr/model/SegmentationFree.prototxt", "../lpr/model/SegmentationFree.caffemodel" + ); + + ifstream file; + string imagename; + int n = 0, correct = 0, j = 0, sum = 0; + char filename[] = "/Users/yujinke/Downloads/general_test/1.txt"; + string pathh = "/Users/yujinke/Downloads/general_test/"; + file.open(filename, ios::in); + while (!file.eof()) + { + file >> imagename; + string imgpath = pathh + imagename; + std::cout << "------------------------------------------------" << endl; + cout << "图片名：" << imagename << endl; + cv::Mat image = cv::imread(imgpath); + // cv::imshow("image", image); + // cv::waitKey(0); + + std::vector res = prc.RunPiplineAsImage(image, pr::SEGMENTATION_FREE_METHOD); + + float conf = 0; + vector con; + vector name; + for (auto st : res) { + if (st.confidence > 0.1) { + //std::cout << st.getPlateName() << " " << st.confidence << std::endl; + con.push_back(st.confidence); + name.push_back(st.getPlateName()); + //conf += st.confidence; + } + else + cout << "no string" << endl; + } + // std::cout << conf << std::endl; + int num = con.size(); + float max = 0; + string platestr, chpr, ch; + int diff = 0, dif = 0; + for (int i = 0; i < num; i++) { + + if (con.at(i) > max) + { + max = con.at(i); + platestr = name.at(i); + } + + } + // cout << "max:"< res = prc.RunPiplineAsImage(image, pr::SEGMENTATION_FREE_METHOD); + for (auto st : res) { + if (st.confidence > 0.75) { + std::cout << st.getPlateName() << " " << st.confidence << std::endl; + cv::Rect region = st.getPlateRect(); -} \ No newline at end of file + cv::rectangle(image, cv::Point(region.x, region.y), cv::Point(region.x + region.width, region.y + region.height), cv::Scalar(255, 255, 0), 2); + } + } + + cv::imshow("image", image); + cv::waitKey(0); + +} + + + + +/*void TEST_CAM() +{ + + cv::VideoCapture capture("test1.mp4"); + cv::Mat frame; + + pr::PipelinePR prc("../lpr/model/cascade.xml", + "../lpr/model/HorizonalFinemapping.prototxt", "../lpr/model/HorizonalFinemapping.caffemodel", + "../lpr/model/Segmentation.prototxt", "../lpr/model/Segmentation.caffemodel", + "../lpr/model/CharacterRecognization.prototxt", "../lpr/model/CharacterRecognization.caffemodel", + "../lpr/model/SegmentationFree.prototxt", "../lpr/model/SegmentationFree.caffemodel" + ); + while (1) { + //读取下一帧 + if (!capture.read(frame)) { + std::cout << "读取视频失败" << std::endl; + exit(1); + } + // + // cv::transpose(frame,frame); + // cv::flip(frame,frame,2); + + // cv::resize(frame,frame,cv::Size(frame.cols/2,frame.rows/2)); + + + + std::vector res = prc.RunPiplineAsImage(frame, pr::SEGMENTATION_FREE_METHOD); + + for (auto st : res) { + if (st.confidence > 0.75) { + std::cout << st.getPlateName() << " " << st.confidence << std::endl; + cv::Rect region = st.getPlateRect(); + + cv::rectangle(frame, cv::Point(region.x, region.y), cv::Point(region.x + region.width, region.y + region.height), cv::Scalar(255, 255, 0), 2); + } + } + + cv::imshow("image", frame); + cv::waitKey(1); + } +}*/ + + +int main() +{ + // TEST_ACC(); + + // TEST_CAM(); + TEST_PIPELINE(); + return 0; +} diff --git a/Prj-Win/lpr/tests/test_recognization.cpp b/Prj-Win/lpr/tests/test_recognization.cpp index 6d6fe84..9244f95 100644 --- a/Prj-Win/lpr/tests/test_recognization.cpp +++ b/Prj-Win/lpr/tests/test_recognization.cpp @@ -1,5 +1,5 @@ // -// Created by 庾金科 on 23/10/2017. +// Created by Jack Yu on 23/10/2017. // #include "../include/CNNRecognizer.h" @@ -16,6 +16,7 @@ void getMaxClass(cv::Mat &probBlob, int *classId, double *classProb) cv::Point classNumber; cv::minMaxLoc(probBlob, NULL, classProb, NULL, &classNumber); + *classId = classNumber.x; } diff --git a/Prj-Win/lpr/tests/test_segmentation.cpp b/Prj-Win/lpr/tests/test_segmentation.cpp index e903636..b379406 100644 --- a/Prj-Win/lpr/tests/test_segmentation.cpp +++ b/Prj-Win/lpr/tests/test_segmentation.cpp @@ -1,5 +1,5 @@ // -// Created by 庾金科 on 16/10/2017. +// Created by Jack Yu on 16/10/2017. // @@ -40,4 +40,4 @@ int main(){ TEST_SLIDINGWINDOWS_EVAL(); return 0; -} \ No newline at end of file +} diff --git a/Prj-Win/lpr/tests/test_segmentationFree.cpp b/Prj-Win/lpr/tests/test_segmentationFree.cpp new file mode 100644 index 0000000..4f948d2 --- /dev/null +++ b/Prj-Win/lpr/tests/test_segmentationFree.cpp @@ -0,0 +1,54 @@ +// +// Created by Jack Yu on 29/11/2017. +// +#include "../include/SegmentationFreeRecognizer.h" +#include "../include/Pipeline.h" + +#include "../include/PlateInfo.h" + + + +std::string decodeResults(cv::Mat code_table,std::vector mapping_table) +{ + cv::MatSize mtsize = code_table.size; + int sequencelength = mtsize[2]; + int labellength = mtsize[1]; + cv::transpose(code_table.reshape(1,1).reshape(1,labellength),code_table); + std::string name = ""; + std::vector seq(sequencelength); + for(int i = 0 ; i < sequencelength; i++) { + float *fstart = ((float *) (code_table.data) + i * labellength ); + int id = std::max_element(fstart,fstart+labellength) - fstart; + seq[i] =id; + } + for(int i = 0 ; i< sequencelength ; i++) + { + if(seq[i]!=labellength-1 && (i==0 || seq[i]!=seq[i-1])) + name+=mapping_table[seq[i]]; + } + std::cout< res = recognizr.SegmentationFreeForSinglePlate(image,pr::CH_PLATE_CODE); + std::cout<