23 bool handPoseEstimationModule::configure(ResourceFinder &rf)
27 moduleName = rf.check(
"name", Value(
"hpe")).asString();
28 setName(moduleName.c_str());
29 arm = rf.check(
"arm", Value(
"right")).asString();
31 initialMean = rf.check(
"initialMean", Value(0.0)).asDouble();
32 initialStdDev= rf.check(
"initialStdDev", Value(3.5)).asDouble();
33 initialArtificialNoiseStdDev = rf.check(
"artificialNoiseStdDev", Value(3.0)).asDouble();
35 lowerBoundNoise = rf.check(
"lowerBound", Value(0.04)).asDouble();
36 upperBoundNoise = rf.check(
"upperBound", Value(3.5)).asDouble();
37 increasedMultiplier = rf.check(
"increaseMultiplier", Value(1.2)).asDouble();
38 decreasedMultiplier = rf.check(
"decreaseMultiplier", Value(0.85)).asDouble();
39 minimumLikelihood = rf.check(
"minimumLikelihood", Value(0.55)).asDouble();
40 minimumIteration = rf.check(
"minIteration", Value(35)).asInt();
41 KDEStdDev = rf.check(
"KDEStdDev", Value(1.0)).asDouble();
43 handlerPortName =
"/" + moduleName +
"/rpc:i";
44 handlerPort.open(handlerPortName.c_str());
50 imageInputPortRName =
"/" + moduleName +
"/rightCam:i";
51 imageInputPortR.open(imageInputPortRName.c_str());
53 imageInputPortLName =
"/" + moduleName +
"/leftCam:i";
54 imageInputPortL.open(imageInputPortLName.c_str());
56 armPortName =
"/" + moduleName +
"/" + arm +
"Arm:i";
57 armPort.open(armPortName.c_str());
59 headPortName =
"/" + moduleName +
"/head:i";
60 headPort.open(headPortName.c_str());
62 likelihoodPortName =
"/" + moduleName +
"/likelihood:i";
63 likelihoodPort.open(likelihoodPortName.c_str());
68 LRimageOutputPortName =
"/" + moduleName +
"/LRimage:o";
69 LRimageOutputPort.open(LRimageOutputPortName.c_str());
71 headOutPortName =
"/" + moduleName +
"/head:o";
72 headOutPort.open(headOutPortName.c_str());
74 particlesOutPortName =
"/" + moduleName +
"/particles:o";
75 particlesOutPort.open(particlesOutPortName.c_str());
76 offsetsPortName =
"/" + moduleName +
"/bestOffsets:o";
77 offsetsPort.open(offsetsPortName.c_str());
82 encodersArm.resize(16);
83 encodersHead.resize(6);
84 initializeSMCVariables();
87 fingers_port.open(
"/hpe/fingerPosition:i");
88 outputPortImage.open(
"/" + moduleName +
"/imageCorrected:o");
89 outputPortImage2.open(
"/" + moduleName +
"/imageCanonical:o");
97 particles=cvCreateMat(8,nParticles,CV_32FC1);
103 particles1 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
104 cvInitMatHeader( particles1, 1, nParticles, CV_32FC1, particles->data.ptr, particles->step );
106 particles2 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
107 cvInitMatHeader( particles2, 1, nParticles, CV_32FC1, particles->data.ptr + particles->step*1, particles->step );
109 particles3 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
110 cvInitMatHeader( particles3, 1, nParticles, CV_32FC1, particles->data.ptr + particles->step*2, particles->step );
112 particles4 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
113 cvInitMatHeader( particles4, 1, nParticles, CV_32FC1, particles->data.ptr + particles->step*3, particles->step );
115 particles5 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
116 cvInitMatHeader( particles5, 1, nParticles, CV_32FC1, particles->data.ptr + particles->step*4, particles->step );
118 particles6 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
119 cvInitMatHeader( particles6, 1, nParticles, CV_32FC1, particles->data.ptr + particles->step*5, particles->step );
121 particles7 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
122 cvInitMatHeader( particles7, 1, nParticles, CV_32FC1, particles->data.ptr + particles->step*6, particles->step );
124 particles8 = cvCreateMatHeader( 1,nParticles, CV_32FC1);
125 cvInitMatHeader( particles8, 1, nParticles, CV_32FC1, particles->data.ptr + particles->step*7, particles->step );
128 particles1to7 = cvCreateMatHeader( 7,nParticles, CV_32FC1);
129 cvInitMatHeader( particles1to7, 7, nParticles, CV_32FC1, particles->data.ptr, particles->step );
132 newParticles=cvCreateMat(8,nParticles,CV_32FC1);
133 newParticles1to7 = cvCreateMatHeader( 7,nParticles, CV_32FC1);
134 cvInitMatHeader( newParticles1to7, 7, nParticles, CV_32FC1, newParticles->data.ptr, newParticles->step );
136 cumWeight =cvCreateMat(1,nParticles+1,CV_32FC1);
137 noise=cvCreateMat(7,nParticles,CV_32FC1);
146 srand((
unsigned int)time(0));
147 rngState = cvRNG(rand());
149 cvRandArr( &rngState, particles1, CV_RAND_NORMAL, cvScalar(initialMean), cvScalar(initialStdDev));
151 cvRandArr( &rngState, particles2, CV_RAND_NORMAL, cvScalar(initialMean), cvScalar(initialStdDev));
153 cvRandArr( &rngState, particles3, CV_RAND_NORMAL, cvScalar(initialMean), cvScalar(initialStdDev));
155 cvRandArr( &rngState, particles4, CV_RAND_NORMAL, cvScalar(initialMean), cvScalar(initialStdDev));
157 cvRandArr( &rngState, particles5, CV_RAND_NORMAL, cvScalar(initialMean), cvScalar(initialStdDev));
159 cvRandArr( &rngState, particles6, CV_RAND_NORMAL, cvScalar(initialMean), cvScalar(initialStdDev));
161 cvRandArr( &rngState, particles7, CV_RAND_NORMAL, cvScalar(initialMean), cvScalar(initialStdDev));
164 artifNoiseStdDev = initialArtificialNoiseStdDev;
166 for (
unsigned int joint=0;joint<8;joint++)
168 cvmSet(particles,joint,0,0.0);
176 double maxLikelihood=0.0;
177 double sumLikelihood=0.0;
178 double likelihood=0.0;
181 ImageOf<PixelBgr> &yarpReturnImage = LRimageOutputPort.prepare();
182 Bottle &outputParticles = particlesOutPort.prepare();
183 Bottle &outputHead= headOutPort.prepare();
185 concatenatedImage.convertTo(concatenatedImage,CV_8UC3);
186 yarpReturnImage.resize(concatenatedImage.cols,concatenatedImage.rows);
187 concatenatedImage.copyTo( cvarrToMat( static_cast<IplImage*> ( yarpReturnImage.getIplImage() ) ) );
189 for(
int index=0;index < nParticles;index++)
192 for (
unsigned int joint=0;joint<7;joint++)
194 outputParticles.addDouble(encodersArm[joint]+cvmGet(particles,joint,index));
197 yInfo() << cvmGet(particles,joint,index);
201 for(
unsigned int joint=7;joint<16;joint++)
203 outputParticles.addDouble(encodersArm[joint]);
206 outputParticles.addInt(nParticles);
211 outputHead.addDouble(encodersHead[k]);
215 particlesOutPort.write();
217 LRimageOutputPort.write();
220 yInfo(
"Waiting for Hypotheses generation and evaluation");
221 Bottle *receivedLikelihood = likelihoodPort.read();
225 for(
int index=0;index<nParticles;index++)
227 likelihood = receivedLikelihood->pop().asDouble();
228 cvmSet(particles,7,index,likelihood);
229 sumLikelihood+=likelihood;
230 if(likelihood>maxLikelihood)
232 maxLikelihood=likelihood;
235 cvmSet(particles,7,0,0.0);
236 kernelDensityEstimation();
239 yInfo(
"Best likelihood: %f", (
float) cvmGet(particles,7,maxWeightIndex));
241 if(iteration>minimumIteration)
244 Bottle &bestOffset = offsetsPort.prepare();
248 bestOffset.addDouble(cvmGet(particles,i,maxWeightIndex));
250 lastBestOffset.clear();
251 lastBestOffset = bestOffset;
252 bestOffset.addDouble(iteration);
257 if(maxLikelihood>minimumLikelihood)
259 systematic_resampling(particles1to7,particles8,newParticles,cumWeight, sumLikelihood);
260 artifNoiseStdDev=artifNoiseStdDev*decreasedMultiplier;
264 cvCopy(particles,newParticles);
265 artifNoiseStdDev=artifNoiseStdDev*increasedMultiplier;
267 if(artifNoiseStdDev > upperBoundNoise)
269 artifNoiseStdDev = upperBoundNoise;
273 CvMat* A = cvCreateMat(8,8,CV_32FC1);
275 cvMatMul(A,newParticles,particles);
279 noiseSingle = cvCreateMat(1,nParticles,CV_32FC1);
280 cvSetZero(noiseSingle);
281 if(artifNoiseStdDev < lowerBoundNoise)
283 artifNoiseStdDev = lowerBoundNoise;
285 cvRandArr( &rngState, noise, CV_RAND_NORMAL, cvScalar(mean), cvScalar(artifNoiseStdDev));
286 cvAdd(particles1to7,noise,particles1to7);
287 yInfo() <<
"ArtNoise: " << artifNoiseStdDev;
289 for (
unsigned int joint=0;joint<8;joint++)
291 cvmSet(particles,joint,0,0.0);
300 double maxWeight = 0.0;
301 for(
int iParticle=0;iParticle<nParticles;iParticle++)
305 for(
int jParticle=0;jParticle<nParticles;jParticle++)
308 if( (
float) cvmGet(particles,7,jParticle) > 0 )
311 for(
int joint=0;joint<7;joint++)
314 sum2+= pow( ((
float)cvmGet(particles,joint,jParticle)-(
float)cvmGet(particles,joint,iParticle)) ,2)/pow(KDEStdDev,2);
316 sum1 += std::exp(-sum2/( 2) )*cvmGet(particles,7,jParticle);
319 sum1 = sum1/(nParticles*sqrt(pow(2*M_PI,1)*pow(KDEStdDev,7)));
320 double weight = 500*sum1 + cvmGet(particles,7,iParticle);
323 maxWeightIndex=iParticle;
347 for(c1=0;c1<nParticles;c1++)
349 ((
float*)(oldParticlesWeights->data.ptr + oldParticlesWeights->step*0))[c1] = (((
float*)(oldParticlesWeights->data.ptr + oldParticlesWeights->step*0))[c1])/(
float)sum2;
351 for(c1=0;c1<nParticles;c1++)
353 sum+=((
float*)(oldParticlesWeights->data.ptr + oldParticlesWeights->step*0))[c1];
358 u=1/(double)nParticles*((
double)rand()/(double)RAND_MAX);
364 ((
float*)(cumWeight->data.ptr))[0]=0.0;
365 for(c1=0;c1<nParticles;c1++)
367 ((
float*)(cumWeight->data.ptr))[c1+1]=((
float*)(cumWeight->data.ptr))[c1]+((
float*)(oldParticlesWeights->data.ptr + oldParticlesWeights->step*0))[c1];
370 if(((
float*)(cumWeight->data.ptr))[nParticles]!=1)
372 ((
float*)(cumWeight->data.ptr))[nParticles]=1;
380 while(npIndex < nParticles)
382 if(((
float*)(cumWeight->data.ptr))[cIndex]>=(double)rIndex/(
double)nParticles+u) {
383 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*0))[npIndex]=((
float*)(oldParticlesState->data.ptr + oldParticlesState->step*0))[cIndex-1];
384 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*1))[npIndex]=((
float*)(oldParticlesState->data.ptr + oldParticlesState->step*1))[cIndex-1];
385 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*2))[npIndex]=((
float*)(oldParticlesState->data.ptr + oldParticlesState->step*2))[cIndex-1];
386 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*3))[npIndex]=((
float*)(oldParticlesState->data.ptr + oldParticlesState->step*3))[cIndex-1];
387 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*4))[npIndex]=((
float*)(oldParticlesState->data.ptr + oldParticlesState->step*4))[cIndex-1];
388 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*5))[npIndex]=((
float*)(oldParticlesState->data.ptr + oldParticlesState->step*5))[cIndex-1];
389 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*6))[npIndex]=((
float*)(oldParticlesState->data.ptr + oldParticlesState->step*6))[cIndex-1];
390 ((
float*)(newParticlesState->data.ptr + newParticlesState->step*7))[npIndex]=0;
404 cv::hconcat(imageProcL, imageProcR,concatenatedImage);
405 cv::flip(concatenatedImage,concatenatedImage,0);
406 cvtColor(concatenatedImage,concatenatedImage,CV_GRAY2BGR);
409 bool handPoseEstimationModule::updateModule()
412 if(imageInputPortR.getInputCount()<=0 || imageInputPortL.getInputCount()<=0 || armPort.getInputCount()<=0 || headPort.getInputCount()<=0)
414 yInfo(
" Waiting for external connections...");
420 yInfo(
"Module Stopped");
425 yInfo(
"Module paused");
429 double timing2=time.now();
431 yarp::sig::ImageOf<yarp::sig::PixelBgr> *iR = imageInputPortR.read(
false);
432 yarp::sig::ImageOf<yarp::sig::PixelBgr> *iL = imageInputPortL.read(
false);
434 if (iR==NULL || iL ==NULL)
439 yInfo(
"Iteration: %d", iteration);
441 imageR=cvarrToMat(static_cast<IplImage*> (iR->getIplImage()));
442 imageL=cvarrToMat(static_cast<IplImage*> (iL->getIplImage()));
448 imageProcR = processImages(imageR);
449 imageProcL = processImages(imageL);
450 mergeAndFlipImages();
451 yInfo() << encodersArm.toString().c_str();
455 Scalar green(0,255,0);
456 Scalar blue(255,0,0);
457 Scalar yellow(0,200,255);
458 Scalar color(100,200,200);
460 Bottle* fingers = fingers_port.read(
false);
465 if(fingers->size() ==0)
467 cout <<
"empty fingers" << endl;
469 int init = (nParticles-1-maxWeightIndex)*12;
470 deque<cv::Point> point_f;
471 for(
int i = init;i< init + 12; i++)
475 fingerP.x = (int) fingers->get(i).asDouble();
479 fingerP.y = (int) (240- fingers->get(i).asDouble());
480 cout <<
"fingerTips: x:" << fingerP.x <<
" y: "<< fingerP.y << endl;
481 point_f.push_front(fingerP);
483 circle(imageL,fingerP,5,red,-1);
485 circle(imageL,fingerP,5,color,-1);
487 circle(imageL,fingerP,5,yellow,-1);
489 circle(imageL,fingerP,5,color,-1);
491 circle(imageL,fingerP,5,green,-1);
493 circle(imageL,fingerP,5,blue,-1);
496 for(
int k = 0; k<6;k++)
498 cv::line(imageL,point_f.front(),point_f.back(),cv::Scalar(255,255,224),2);
502 init = (nParticles-1)*12;
503 for(
int i = init;i< init + 12; i++)
507 fingerP.x = (int) fingers->get(i).asDouble();
511 fingerP.y = (int) (240- fingers->get(i).asDouble());
512 cout <<
"fingerTips: x:" << fingerP.x <<
" y: "<< fingerP.y << endl;
513 point_f.push_front(fingerP);
514 if(i!=init+9 && i!=init+11)
515 circle(tmp,fingerP,5,color,-1);
517 circle(tmp,fingerP,5,red,-1);
519 circle(tmp,fingerP,5,color,-1);
521 circle(tmp,fingerP,5,yellow,-1);
523 circle(tmp,fingerP,5,color,-1);
525 circle(tmp,fingerP,5,green,-1);
527 circle(tmp,fingerP,5,blue,-1);
530 for(
int k = 0; k<6;k++)
532 cv::line(tmp,point_f.front(),point_f.back(),cv::Scalar(255,255,224),2);
535 yarp::sig::ImageOf< yarp::sig::PixelBgr> &resultImage = outputPortImage.prepare();
536 IplImage outIpl = imageL;
537 resultImage.resize(outIpl.width,outIpl.height);
538 cvCopy( &outIpl,static_cast<IplImage*>(resultImage.getIplImage()) );
540 outputPortImage.write();
542 yarp::sig::ImageOf< yarp::sig::PixelBgr> &resultImage2 = outputPortImage2.prepare();
543 IplImage outIpl2 = tmp;
544 resultImage2.resize(outIpl2.width,outIpl2.height);
545 cvCopy( &outIpl2,static_cast<IplImage*>(resultImage2.getIplImage()) );
547 outputPortImage2.write();
548 yInfo() <<
"maxWeightIndex" << maxWeightIndex;
555 Mat dtImage2,dtImage2_8;
556 cvtColor(inputImage,edges,CV_RGB2GRAY);
559 blur( edges, edges, Size(3,3) );
560 Canny(edges,edges,65,3*65,3);
561 threshold(edges,edges,100,255,THRESH_BINARY_INV);
562 distanceTransform(edges,dtImage,CV_DIST_L2,CV_DIST_MASK_5);
563 cvtColor(dtImage,dtImage2,CV_GRAY2BGR);
564 dtImage.convertTo(dtImage2_8,CV_8UC3);
571 Bottle *receive = armPort.read();
573 string s1 = receive->toString();
574 char *str =
new char[s1.size()+1];
575 strcpy(str, s1.c_str());
577 pch = strtok ( (
char*) str,
" ");
581 encodersArm[i]=atof(pch);
582 pch = strtok (NULL,
" ");
590 Bottle *receive = headPort.read();
591 string s2 = receive->toString();
592 char *str =
new char[s2.size()+1];
593 strcpy(str, s2.c_str());
595 pch = strtok ( (
char*) str,
" ");
599 encodersHead[i]=atof(pch);
600 pch = strtok (NULL,
" ");
605 double handPoseEstimationModule::getPeriod()
610 bool handPoseEstimationModule::attach(RpcServer &source)
612 return this->yarp().attachAsServer(source);
615 bool handPoseEstimationModule::interruptModule()
618 handlerPort.interrupt();
620 headPort.interrupt();
621 imageInputPortR.interrupt();
622 imageInputPortL.interrupt();
626 bool handPoseEstimationModule::close()
628 yInfo(
"starting shutdown procedure");
630 yInfo(
"closing ports");
634 imageInputPortR.close();
635 imageInputPortL.close();
649 yInfo(
"start command received");
656 yInfo(
"stop command received");
664 yInfo(
"pause command received");
671 yInfo(
"resume command received");
677 yInfo(
"lastOffsets command received");
679 reply = lastBestOffset;
685 yInfo(
"quit command received");
bool stop()
Stop the hand pose estimation.
void mergeAndFlipImages()
merge the left and right images.
bool resume()
Resume the hand pose estimation.
bool systematic_resampling(CvMat *oldParticlesState, CvMat *oldParticlesWeights, CvMat *newParticlesState, CvMat *cumWeight, float sum2)
perform the systematic resampling step of the SMC algorithm
bool runSMCIteration()
Run one iteration of the Sequential Monte Carlo Parameters estimation.
bool readArmJoints()
Read the arm joints.
bool initializeSMCVariables()
Initialize the variables structs needed for the SMC to run.
bool readHeadJoints()
Read the head joints.
bool initSMC()
Initialize the variables values for the SMC.
bool start()
Start (re-start) the hand pose estimation.
cv::Mat processImages(cv::Mat inputImage)
The function applies a canny edge detector and a distance transform.
bool pause()
Pause the hand pose estimation.
yarp::os::Bottle lastOffsets()
Ask for the last estimated angular Offsets on the arm (7DoF)
bool quit()
Quit the module.
void kernelDensityEstimation()
Perform the Kernel Density estimation with a multivariate gaussian kernel.