CustusX  2023.01.05-dev+develop.0da12
An IGT application
cxImageStreamerOpenCV.cpp
Go to the documentation of this file.
1 /*=========================================================================
2 This file is part of CustusX, an Image Guided Therapy Application.
3 
4 Copyright (c) SINTEF Department of Medical Technology.
5 All rights reserved.
6 
7 CustusX is released under a BSD 3-Clause license.
8 
9 See Lisence.txt (https://github.com/SINTEFMedtek/CustusX/blob/master/License.txt) for details.
10 =========================================================================*/
11 #include "cxImageStreamerOpenCV.h"
12 #include "cxConfig.h"
13 
14 #include <QCoreApplication>
15 #include <QTimer>
16 #include <QTime>
17 #include <QHostAddress>
18 #include "vtkImageData.h"
19 #include "vtkSmartPointer.h"
20 #include "vtkMetaImageReader.h"
21 #include "vtkImageImport.h"
22 #include "vtkLookupTable.h"
23 #include "vtkImageMapToColors.h"
24 #include "vtkMetaImageWriter.h"
25 
26 #include "cxTypeConversions.h"
28 #include "cxStringHelpers.h"
29 #include "cxDoubleProperty.h"
30 #include "cxBoolProperty.h"
31 #include "cxLogger.h"
32 #include "cxVideoServerConfig.h"
33 
34 #ifdef CX_USE_OpenCV
35 #include <opencv2/highgui/highgui.hpp>
36 #include <opencv2/imgproc/imgproc.hpp>
37 #endif
38 
39 namespace cx
40 {
41 
42 std::vector<PropertyPtr> ImageStreamerOpenCVArguments::getSettings(QDomElement root)
43 {
44  std::vector<PropertyPtr> retval;
45  retval.push_back(this->getVideoPortOption(root));
46  retval.push_back(this->getPrintPropertiesOption(root));
47  return retval;
48 }
49 
51 {
52  DoublePropertyPtr retval;
53  retval = DoubleProperty::initialize("videoport", "Video Port", "Select video source as an integer from 0 and up.", 0, DoubleRange(0, 10, 1), 0, root);
54  retval->setGuiRepresentation(DoublePropertyBase::grSPINBOX);
55  retval->setGroup("OpenCV");
56  return retval;
57 }
58 
60 {
61  BoolPropertyPtr retval;
62  bool defaultValue = false;
63  retval = BoolProperty::initialize("properties", "Print Properties",
64  "When starting OpenCV, print properties to console",
65  defaultValue, root);
66  retval->setAdvanced(true);
67  retval->setGroup("OpenCV");
68  return retval;
69 }
70 
72 {
73  StringMap retval;
74  retval["--type"] = "OpenCV";
75  retval["--videoport"] = this->getVideoPortOption(root)->getValueAsVariant().toString();
76  if (this->getPrintPropertiesOption(root)->getValue())
77  retval["--properties"] = "1";
78  return retval;
79 }
80 
82 {
83  QStringList retval;
84 #ifdef CX_USE_OpenCV
85  retval << "--videoport: video id, default=0";
86  retval << "--in_width: width of incoming image, default=camera";
87  retval << "--in_height: height of incoming image, default=camera";
88  retval << "--out_width: width of outgoing image, default=camera";
89  retval << "--out_height: width of outgoing image, default=camera";
90  retval << "--properties: dump image properties";
91 #endif
92  return retval;
93 }
94 
95 } // namespace cx
96 
97 //------------------------------------------------------------
98 //------------------------------------------------------------
99 //------------------------------------------------------------
100 
101 namespace
102 {
103 
104 //------------------------------------------------------------
105 // Function to generate random matrix.
106 void GetRandomTestMatrix(igtl::Matrix4x4& matrix)
107 {
108  //float position[3];
109  //float orientation[4];
110 
111  matrix[0][0] = 1.0;
112  matrix[1][0] = 0.0;
113  matrix[2][0] = 0.0;
114  matrix[3][0] = 0.0;
115  matrix[0][1] = 0.0;
116  matrix[1][1] = -1.0;
117  matrix[2][1] = 0.0;
118  matrix[3][1] = 0.0;
119  matrix[0][2] = 0.0;
120  matrix[1][2] = 0.0;
121  matrix[2][2] = 1.0;
122  matrix[3][2] = 0.0;
123  matrix[0][3] = 0.0;
124  matrix[1][3] = 0.0;
125  matrix[2][3] = 0.0;
126  matrix[3][3] = 1.0;
127 }
128 
129 }
130 
131 namespace cx
132 {
134 {
135  mGrabbing = false;
136  mAvailableImage = false;
137  setSendInterval(40);
138 
139 #ifdef CX_USE_OpenCV
140  mVideoCapture.reset(new cv::VideoCapture());
141 #endif
142  mSendTimer = new QTimer(this);
143  connect(mSendTimer, SIGNAL(timeout()), this, SLOT(send())); // this signal will be executed in the thread of THIS, i.e. the main thread.
144 }
145 
147 {
148  this->deinitialize_local();
149 }
150 
152 {
153  return "OpenCV";
154 }
155 
157 {
159 }
160 
161 
163 {
165 }
166 
167 void ImageStreamerOpenCV::deinitialize_local()
168 {
169 #ifdef CX_USE_OpenCV
170  while (mGrabbing) // grab() method seems to call processmessages itself...
171  qApp->processEvents();
172  mVideoCapture->release();
173  mVideoCapture.reset(new cv::VideoCapture());
174 #endif
175 }
176 
177 void ImageStreamerOpenCV::initialize_local()
178 {
179 #ifdef CX_USE_OpenCV
180 
181  if (!mArguments.count("videoport"))
182  mArguments["videoport"] = "0";
183  if (!mArguments.count("out_width"))
184  mArguments["out_width"] = "";
185  if (!mArguments.count("out_height"))
186  mArguments["out_height"] = "";
187  if (!mArguments.count("in_width"))
188  mArguments["in_width"] = "";
189  if (!mArguments.count("in_height"))
190  mArguments["in_height"] = "";
191 
192  QString videoSource = mArguments["videoport"];
193  int videoport = convertStringWithDefault(mArguments["videoport"], 0);
194 
195  bool sourceIsInt = false;
196  videoSource.toInt(&sourceIsInt);
197 
198  if (sourceIsInt){
199  // open device (camera)
200  mVideoCapture->open(videoport);
201  }
202  else{
203  // open file
204  mVideoCapture->open(videoSource.toStdString().c_str());
205  }
206 
207  if (!mVideoCapture->isOpened())
208  {
209  cerr << "ImageStreamerOpenCV: Failed to open a video device or video file!\n" << endl;
210  return;
211  }
212 
213  // try one grab before accepting the streamer
214  // - this fails if no camera is attached
215  try
216  {
217  mVideoCapture->grab();
218  }
219  catch(cv::Exception& e)
220  {
221  CX_LOG_ERROR() << "OpenCV failed with message: " << e.what();
222  mVideoCapture->release();
223  return;
224  }
225 
226 
227  {
228  //determine default values
229  int default_width = mVideoCapture->get(cv::CAP_PROP_FRAME_WIDTH);
230  int default_height = mVideoCapture->get(cv::CAP_PROP_FRAME_HEIGHT);
231 
232  //set input size
233  int in_width = convertStringWithDefault(mArguments["in_width"], default_width);
234  int in_height = convertStringWithDefault(mArguments["in_height"], default_height);
235  mVideoCapture->set(cv::CAP_PROP_FRAME_WIDTH, in_width);
236  mVideoCapture->set(cv::CAP_PROP_FRAME_HEIGHT, in_height);
237 
238  //set output size (resize)
239  int out_width = convertStringWithDefault(mArguments["out_width"], in_width);
240  int out_height = convertStringWithDefault(mArguments["out_height"], in_height);
241  mRescaleSize.setWidth(out_width);
242  mRescaleSize.setHeight(out_height);
243 
244  if (mArguments.count("properties"))
245  this->dumpProperties();
246 
247  std::cout << "ImageStreamerOpenCV: Started streaming from openCV device "
248  << videoSource.toStdString()
249  << ", size=(" << in_width << "," << in_height << ")";
250  if (( in_width!=mRescaleSize.width() )|| (in_height!=mRescaleSize.height()))
251  std::cout << ". Scaled to (" << mRescaleSize.width() << "," << mRescaleSize.height() << ")";
252 
253  std::cout << std::endl;
254  }
255 #endif
256 }
257 
259 {
260 #ifdef CX_USE_OpenCV
261  this->initialize_local();
262 
263  if (!mSendTimer || !mVideoCapture->isOpened())
264  {
265  reportError("ImageStreamerOpenCV: Failed to start streaming: Not initialized.");
266  return;
267  }
268 
269  mSender = sender;
270  mSendTimer->start(getSendInterval());
271  this->continousGrabEvent(); // instead of grabtimer
272 
273 #else
274  reportWarning("ImageStreamerOpenCV: Failed to start streaming: CX_USE_OpenCV not defined.");
275 #endif //CX_USE_OpenCV
276 }
277 
279 {
280  if (!mSendTimer)
281  return;
282  mSendTimer->stop();
283  mSender.reset();
284 
285  this->deinitialize_local();
286 }
287 
289 {
290  return (mSendTimer && mVideoCapture->isOpened());
291 }
292 
293 void ImageStreamerOpenCV::dumpProperties()
294 {
295 #ifdef CX_USE_OpenCV
296  this->dumpProperty(cv::CAP_PROP_POS_MSEC, "cv::CAP_PROP_POS_MSEC");
297  this->dumpProperty(cv::CAP_PROP_POS_FRAMES, "cv::CAP_PROP_POS_FRAMES");
298  this->dumpProperty(cv::CAP_PROP_POS_AVI_RATIO, "cv::CAP_PROP_POS_AVI_RATIO");
299  this->dumpProperty(cv::CAP_PROP_FRAME_WIDTH, "cv::CAP_PROP_FRAME_WIDTH");
300  this->dumpProperty(cv::CAP_PROP_FRAME_HEIGHT, "cv::CAP_PROP_FRAME_HEIGHT");
301  this->dumpProperty(cv::CAP_PROP_FPS, "cv::CAP_PROP_FPS");
302  this->dumpProperty(cv::CAP_PROP_FOURCC, "cv::CAP_PROP_FOURCC");
303  this->dumpProperty(cv::CAP_PROP_FRAME_COUNT, "cv::CAP_PROP_FRAME_COUNT");
304  this->dumpProperty(cv::CAP_PROP_FORMAT, "cv::CAP_PROP_FORMAT");
305  this->dumpProperty(cv::CAP_PROP_MODE, "cv::CAP_PROP_MODE");
306  this->dumpProperty(cv::CAP_PROP_BRIGHTNESS, "cv::CAP_PROP_BRIGHTNESS");
307  this->dumpProperty(cv::CAP_PROP_CONTRAST, "cv::CAP_PROP_CONTRAST");
308  this->dumpProperty(cv::CAP_PROP_SATURATION, "cv::CAP_PROP_SATURATION");
309  this->dumpProperty(cv::CAP_PROP_HUE, "cv::CAP_PROP_HUE");
310  this->dumpProperty(cv::CAP_PROP_GAIN, "cv::CAP_PROP_GAIN");
311  this->dumpProperty(cv::CAP_PROP_EXPOSURE, "cv::CAP_PROP_EXPOSURE");
312  this->dumpProperty(cv::CAP_PROP_CONVERT_RGB, "cv::CAP_PROP_CONVERT_RGB");
313  // this->dumpProperty(cv::CAP_PROP_WHITE_BALANCE, "cv::CAP_PROP_WHITE_BALANCE");
314  this->dumpProperty(cv::CAP_PROP_RECTIFICATION, "cv::CAP_PROP_RECTIFICATION");
315 #endif
316 }
317 
318 void ImageStreamerOpenCV::dumpProperty(int val, QString name)
319 {
320 #ifdef CX_USE_OpenCV
321  double value = mVideoCapture->get(val);
322  if (value != -1)
323  std::cout << "Property " << name.toStdString() << " : " << mVideoCapture->get(val) << std::endl;
324 #endif
325 }
326 
332 void ImageStreamerOpenCV::continousGrabEvent()
333 {
334  if (!mSendTimer->isActive())
335  return;
336  this->grab();
337  QMetaObject::invokeMethod(this, "continousGrabEvent", Qt::QueuedConnection);
338 }
339 
340 void ImageStreamerOpenCV::grab()
341 {
342 #ifdef CX_USE_OpenCV
343  if (!mVideoCapture->isOpened())
344  {
345  return;
346  }
347 
348  mGrabbing = true;
349  // grab images from camera to opencv internal buffer, do not process
350  bool success = mVideoCapture->grab();
351  if (success)
352  mLastGrabTime = QDateTime::currentDateTime();
353  mAvailableImage = success;
354  mGrabbing = false;
355 #endif
356 }
357 
358 void ImageStreamerOpenCV::send()
359 {
360  if (!mSender || !mSender->isReady())
361  return;
362  if (!mAvailableImage)
363  {
364 // reportDebug("dropped resend of frame");
365  return;
366  }
367  PackagePtr package(new Package());
368  package->mImage = this->getImageMessage();
369  mSender->send(package);
370  mAvailableImage = false;
371 
372 // static int counter=0;
373 // if (++counter%50==0)
374 // std::cout << "=== ImageStreamerOpenCV send: " << start.msecsTo(QTime::currentTime()) << " ms" << std::endl;
375 }
376 
377 ImagePtr ImageStreamerOpenCV::getImageMessage()
378 {
379 #ifdef CX_USE_OpenCV
380  if (!mVideoCapture->isOpened())
381  return ImagePtr();
382 
383  QTime start = QTime::currentTime();
384 
385  cv::Mat frame_source;
386  // mVideoCapture >> frame_source;
387  if (!mVideoCapture->retrieve(frame_source, 0))
388  return ImagePtr();
389 
390  if (this->thread() == QCoreApplication::instance()->thread() && !mSender)
391  {
392  cv::imshow("ImageStreamerOpenCV", frame_source);
393  }
394 
395  cv::Mat frame = frame_source;
396  if (( frame.cols!=mRescaleSize.width() )|| (frame.rows!=mRescaleSize.height()))
397  {
398  cv::resize(frame_source, frame, cv::Size(mRescaleSize.width(), mRescaleSize.height()), 0, 0, cv::INTER_LINEAR);
399  }
400 
401  vtkImageDataPtr raw = this->convertTovtkImageData(frame);
402  ImagePtr image(new Image("openCV", raw));
403  image->setAcquisitionTime(mLastGrabTime);
404  return image;
405 #else
406  return ImagePtr();
407 #endif
408 }
409 
410 vtkImageDataPtr ImageStreamerOpenCV::convertTovtkImageData(cv::Mat& frame)
411 {
412  vtkImageDataPtr retval = vtkImageDataPtr::New();
413 
414  Eigen::Array3i dim(frame.cols, frame.rows, 1);
415 // Eigen::Array3f spacing(1,1);
416  retval->SetDimensions(dim.data());
417  retval->SetSpacing(1,1,1);
418 
419  int dataType = -1;
420 
421 // if (frame.channels() == 3 || frame.channels() == 4) // dropped support for alpha - dont think openCV uses this.
422  if (frame.channels() == 3)
423  {
424  dataType = VTK_UNSIGNED_CHAR;
425  }
426  else if (frame.channels() == 1)
427  {
428  // dropped support: must iterate over using shorts later on.
429 // if (frame.depth() == 16)
430 // {
431 // dataType = VTK_UNSIGNED_SHORT;
432 // }
433  if (frame.depth() == 8)
434  {
435  dataType = VTK_UNSIGNED_CHAR;
436  }
437  }
438 
439  if (dataType == -1)
440  {
441  std::cerr << "unknown image type" << std::endl;
442  return vtkImageDataPtr();
443  }
444 
445  retval->AllocateScalars(dataType, frame.channels());
446 
447  //------------------------------------------------------------
448  // Create a new IMAGE type message
449 // IGTLinkImageMessage::Pointer imgMsg = IGTLinkImageMessage::New();
450 // imgMsg->SetDimensions(size);
451 // imgMsg->SetSpacing(spacingF);
452 // imgMsg->SetScalarType(scalarType);
453 // imgMsg->SetSubVolume(svsize, svoffset);
454 // imgMsg->AllocateScalars();
455 // imgMsg->SetTimeStamp(timestamp);
456 
457  unsigned char* dest = reinterpret_cast<unsigned char*> (retval->GetScalarPointer());
458  uchar* src = frame.data;
459  int N = frame.rows * frame.cols;
460 // QString colorFormat;
461 
462  if (frame.channels() == 3)
463  {
464  if (frame.isContinuous())
465  {
466  // 3-channel continous colors
467  for (int i = 0; i < N; ++i)
468  {
469 // *destPtr++ = 255;
470  *dest++ = src[2]; // R
471  *dest++ = src[1]; // G
472  *dest++ = src[0]; // B
473  src += 3;
474  }
475  }
476  else
477  {
478 // std::cout << "noncontinous conversion, rows=" << size[1] << std::endl;
479  for (int i=0; i<dim[1]; ++i)
480  {
481  const uchar* src = frame.ptr<uchar>(i);
482  for (int j=0; j<dim[0]; ++j)
483  {
484 // *destPtr++ = 255;
485  *dest++ = src[2];
486  *dest++ = src[1];
487  *dest++ = src[0];
488  src += 3;
489  }
490  }
491  }
492 // colorFormat = "ARGB";
493  }
494  if (frame.channels() == 1)
495  {
496  if (!frame.isContinuous())
497  {
498  std::cout << "Error: Non-continous frame data." << std::endl;
499  return vtkImageDataPtr();
500  }
501 
502  // BW camera
503  for (int i = 0; i < N; ++i)
504  {
505  *dest++ = *src++;
506  }
507 // colorFormat = "R";
508  }
509 
510  return retval;
511 
512 }
513 
514 //------------------------------------------------------------
515 //------------------------------------------------------------
516 //------------------------------------------------------------
517 
518 }
static BoolPropertyPtr initialize(const QString &uid, QString name, QString help, bool value, QDomNode root=QDomNode())
void reportError(QString msg)
Definition: cxLogger.cpp:71
DoublePropertyBasePtr getVideoPortOption(QDomElement root)
virtual void initialize(StringMap arguments)
Utility class for describing a bounded numeric range.
Definition: cxDoubleRange.h:32
boost::shared_ptr< class Image > ImagePtr
Definition: cxDicomWidget.h:27
virtual QStringList getArgumentDescription()
BoolPropertyBasePtr getPrintPropertiesOption(QDomElement root)
StringMap convertToCommandLineArguments(QDomElement root)
std::map< QString, QString > StringMap
void reportWarning(QString msg)
Definition: cxLogger.cpp:70
#define CX_LOG_ERROR
Definition: cxLogger.h:99
A volumetric data set.
Definition: cxImage.h:45
boost::shared_ptr< class BoolPropertyBase > BoolPropertyBasePtr
boost::shared_ptr< class DoublePropertyBase > DoublePropertyBasePtr
boost::shared_ptr< class DoubleProperty > DoublePropertyPtr
boost::shared_ptr< struct Package > PackagePtr
static DoublePropertyPtr initialize(const QString &uid, QString name, QString help, double value, DoubleRange range, int decimals, QDomNode root=QDomNode())
virtual void startStreaming(SenderPtr sender)
unsigned char uchar
int convertStringWithDefault(QString text, int def)
boost::shared_ptr< class BoolProperty > BoolPropertyPtr
boost::shared_ptr< Sender > SenderPtr
Definition: cxSender.h:64
vtkSmartPointer< class vtkImageData > vtkImageDataPtr
virtual void initialize(StringMap arguments)
Definition: cxStreamer.cpp:59
virtual std::vector< PropertyPtr > getSettings(QDomElement root)
Namespace for all CustusX production code.