SLProject  4.2.000
A platform independent 3D computer graphics framework for desktop OS, Android, iOS and online in web browsers
AppDemoVideo.cpp File Reference

All video capturing and video tracking functions are in here. More...

#include <AppDemoSceneView.h>
#include <AppCommon.h>
#include <AppDemoSceneID.h>
#include <SLScene.h>
#include <SLSceneView.h>
#include <CVImage.h>
#include <CVCapture.h>
#include <CVTracked.h>
#include <CVTrackedAruco.h>
#include <SLGLTexture.h>
#include <CVCalibrationEstimator.h>
#include <GlobalTimer.h>
#include <Profiler.h>
#include <FtpUtils.h>
Include dependency graph for AppDemoVideo.cpp:

Go to the source code of this file.

Functions

void updateTrackingSceneCamera (CVCamera *ac)
 
void runCalibrationEstimator (CVCamera *ac, SLScene *s, SLSceneView *sv)
 
void ensureValidCalibration (CVCamera *ac, SLSceneView *sv)
 logic that ensures that we have a valid calibration state More...
 
bool onUpdateVideo ()
 Implements the update per frame for video update and feature tracking. More...
 

Variables

SLGLTexturegVideoTexture = nullptr
 
CVTrackedgVideoTracker = nullptr
 
SLNodegVideoTrackedNode = nullptr
 

Detailed Description

All video capturing and video tracking functions are in here.

Date
August 2019
Authors
Marcus Hudritsch
Remarks
Please use clangformat to format the code. See more code style on https://github.com/cpvrlab/SLProject4/wiki/SLProject-Coding-Style

Definition in file AppDemoVideo.cpp.

Function Documentation

◆ ensureValidCalibration()

void ensureValidCalibration ( CVCamera ac,
SLSceneView sv 
)

logic that ensures that we have a valid calibration state

Definition at line 174 of file AppDemoVideo.cpp.

175 {
177 
178  // we have to make sure calibration process is stopped if someone stops calibrating
180  {
183  }
184 
185  if (ac->calibration.state() == CS_uncalibrated)
186  {
187  // Try to read device lens and sensor information
188  string strF = AppCommon::deviceParameter["DeviceLensFocalLength"];
189  string strW = AppCommon::deviceParameter["DeviceSensorPhysicalSizeW"];
190  string strH = AppCommon::deviceParameter["DeviceSensorPhysicalSizeH"];
191  if (!strF.empty() && !strW.empty() && !strH.empty())
192  {
193  float devF = strF.empty() ? 0.0f : stof(strF);
194  float devW = strW.empty() ? 0.0f : stof(strW);
195  float devH = strH.empty() ? 0.0f : stof(strH);
196 
197  // Changes the state to CS_guessed
198  ac->calibration = CVCalibration(devW,
199  devH,
200  devF,
201  cv::Size(CVCapture::instance()->lastFrame.cols,
203  ac->mirrorH(),
204  ac->mirrorV(),
205  ac->type(),
207  }
208  else
209  {
210  // make a guess using frame size and a guessed field of view
211  ac->calibration = CVCalibration(cv::Size(CVCapture::instance()->lastFrame.cols,
213  60.0,
214  ac->mirrorH(),
215  ac->mirrorV(),
216  ac->type(),
218  }
219  }
220 }
@ CS_uncalibrated
The camera is not calibrated (no calibration found)
Definition: CVCalibration.h:32
#define PROFILE_FUNCTION()
Definition: Instrumentor.h:41
static CVCalibrationEstimator * calibrationEstimator
Definition: AppCommon.h:107
static map< string, string > deviceParameter
Generic device parameter.
Definition: AppCommon.h:101
Live video camera calibration class with OpenCV an OpenCV calibration.
Definition: CVCalibration.h:71
CVCalibState state() const
bool mirrorH()
Definition: CVCamera.h:22
CVCalibration calibration
Definition: CVCamera.h:36
CVCameraType type()
Definition: CVCamera.h:24
bool mirrorV()
Definition: CVCamera.h:23
CVMat lastFrame
last frame grabbed in BGR
Definition: CVCapture.h:119
static CVCapture * instance()
Public static instance getter for singleton pattern.
Definition: CVCapture.h:65
static std::string get()
Definition: Utils.cpp:1261

◆ onUpdateVideo()

bool onUpdateVideo ( )

Implements the update per frame for video update and feature tracking.

This routine is called once per frame before any other update within the the main rendering loop (see: AppDemoMainGLFW::onPaint or GLES3View::onDrawFrame). See the documentation within SLCVTracked and in all of its inheritants.

Definition at line 227 of file AppDemoVideo.cpp.

228 {
230 
231  if (AppCommon::sceneViews.empty())
232  return false;
233 
236 
237  if (CVCapture::instance()->videoType() != VT_NONE &&
238  !CVCapture::instance()->lastFrame.empty())
239  {
240  SLfloat trackingTimeStartMS = GlobalTimer::timeMS();
241 
243 
246  {
248  }
249  else
250  {
252  // Attention: Always update scene camera fovV from calibration because the calibration may have
253  // been adapted in adjustForSL after a change of aspect ratio!
254  // The active scene view camera may be a different one that the tracking camera
255  // but we have to update the tracking camera only!
257 
259  {
260  bool foundPose = gVideoTracker->track(CVCapture::instance()->lastFrameGray,
261  CVCapture::instance()->lastFrame,
262  &ac->calibration);
263  if (foundPose)
264  {
265  // clang-format off
266  // convert matrix type CVMatx44f to SLMat4f
268  SLMat4f glOVM(cvOVM.val[0], cvOVM.val[1], cvOVM.val[2], cvOVM.val[3],
269  cvOVM.val[4], cvOVM.val[5], cvOVM.val[6], cvOVM.val[7],
270  cvOVM.val[8], cvOVM.val[9], cvOVM.val[10],cvOVM.val[11],
271  cvOVM.val[12],cvOVM.val[13],cvOVM.val[14],cvOVM.val[15]);
272  // clang-format on
273 
274  // set the object matrix depending if the
275  // tracked node is attached to a camera or not
276  if (typeid(*gVideoTrackedNode) == typeid(SLCamera))
277  {
278  gVideoTrackedNode->om(glOVM.inverted());
280  }
281  else
282  {
283  // see comments in CVTracked::calcObjectMatrix
284  gVideoTrackedNode->om(sv->camera()->om() * glOVM);
286  }
287  }
288  else
290  }
291 
292  // Update info text only for chessboard scene
297  {
298  SLfloat fovH = ac->calibration.cameraFovHDeg();
300  stringstream ss; // info line text
301  ss << "Tracking Chessboard on " << (CVCapture::instance()->videoType() == VT_MAIN ? "main " : "scnd. ") << "camera. ";
302  if (ac->calibration.state() == CS_calibrated)
303  ss << "FOVH: " << fovH << ", error: " << err;
304  else
305  ss << "Not calibrated. FOVH guessed: " << fovH << " degrees.";
306  s->info(ss.str());
307  }
308  }
309 
310  //...................................................................
311  // copy image to video texture
312  if (gVideoTexture)
313  {
314  //SL_LOG("glFormat: %s\n", CVImage::formatString(CVCapture::instance()->format).c_str());
315 
316  if (ac->calibration.state() == CS_calibrated && ac->showUndistorted())
317  {
318  CVMat undistorted;
319  ac->calibration.remap(CVCapture::instance()->lastFrame, undistorted);
320  gVideoTexture->copyVideoImage(undistorted.cols,
321  undistorted.rows,
323  undistorted.data,
324  undistorted.isContinuous(),
325  true);
326  }
327  else
328  {
333  CVCapture::instance()->lastFrame.isContinuous(),
334  true);
335  }
336  }
337  else
338  SL_WARN_MSG("No video texture to copy to.");
339 
340 #ifndef SL_EMSCRIPTEN
341  CVTracked::trackingTimesMS.set(GlobalTimer::timeMS() - trackingTimeStartMS);
342 #endif
343 
344  return true;
345  }
346 
347  return false;
348 }
@ SID_VideoTrackChessScnd
@ SID_VideoCalibrateScnd
@ SID_VideoCalibrateMain
@ SID_VideoTrackChessMain
SLNode * gVideoTrackedNode
SLGLTexture * gVideoTexture
void ensureValidCalibration(CVCamera *ac, SLSceneView *sv)
logic that ensures that we have a valid calibration state
void updateTrackingSceneCamera(CVCamera *ac)
CVTracked * gVideoTracker
void runCalibrationEstimator(CVCamera *ac, SLScene *s, SLSceneView *sv)
@ CS_calibrated
The camera is calibrated.
Definition: CVCalibration.h:33
@ VT_NONE
No camera needed.
Definition: CVCapture.h:41
@ VT_MAIN
Main camera on all on all all devices.
Definition: CVCapture.h:42
cv::Matx44f CVMatx44f
Definition: CVTypedefs.h:59
cv::Mat CVMat
Definition: CVTypedefs.h:38
float SLfloat
Definition: SL.h:173
#define SL_WARN_MSG(message)
Definition: SL.h:241
#define SL_DB_HIDDEN
Flags an object as hidden.
Definition: SLDrawBits.h:20
static SLVSceneView sceneViews
Vector of sceneview pointers.
Definition: AppCommon.h:62
static SLSceneID sceneID
ID of currently loaded scene.
Definition: AppCommon.h:89
static SLScene * scene
Pointer to the one and only SLScene instance.
Definition: AppCommon.h:61
void remap(CVMat &inDistorted, CVMat &outUndistorted)
Undistorts the inDistorted image into the outUndistorted.
float reprojectionError() const
float cameraFovHDeg() const
void showUndistorted(bool su)
Definition: CVCamera.h:25
CVCamera * activeCamera
Pointer to the active camera.
Definition: CVCapture.h:136
CVPixelFormatGL format
GL pixel format.
Definition: CVCapture.h:122
void videoType(CVVideoType vt)
Setter for video type also sets the active calibration.
Definition: CVCapture.cpp:866
static AvgFloat trackingTimesMS
Averaged time for video tracking in ms.
Definition: CVTracked.h:82
virtual bool track(CVMat imageGray, CVMat imageBgr, CVCalibration *calib)=0
CVMatx44f objectViewMat()
Definition: CVTracked.h:65
static float timeMS()
Definition: GlobalTimer.cpp:25
Active or visible camera node class.
Definition: SLCamera.h:54
SLbool copyVideoImage(SLint camWidth, SLint camHeight, CVPixelFormatGL glFormat, SLuchar *data, SLbool isContinuous, SLbool isTopLeft)
Copies the image data from a video camera into the current video image.
void om(const SLMat4f &mat)
Definition: SLNode.h:277
void setDrawBitsRec(SLuint bit, SLbool state)
Definition: SLNode.cpp:804
The SLScene class represents the top level instance holding the scene structure.
Definition: SLScene.h:47
void info(SLstring i)
Definition: SLScene.h:93
SceneView class represents a dynamic real time 3D view onto the scene.
Definition: SLSceneView.h:69
void camera(SLCamera *camera)
Definition: SLSceneView.h:145
void set(T value)
Sets the current value in the value array and builds the average.
Definition: Averaged.h:53

◆ runCalibrationEstimator()

void runCalibrationEstimator ( CVCamera ac,
SLScene s,
SLSceneView sv 
)

Definition at line 63 of file AppDemoVideo.cpp.

64 {
66 
67  AppDemoSceneView* adSv = dynamic_cast<AppDemoSceneView*>(sv);
68  static bool processedCalibResult = false;
69  try
70  {
72  {
74  CVCapture::instance()->activeCamSizeIndex,
75  ac->mirrorH(),
76  ac->mirrorV(),
77  ac->type(),
82 
83  // clear grab request from sceneview
84  adSv->grab = false;
85  processedCalibResult = false;
86  }
87 
88  if (AppCommon::calibrationEstimator->isStreaming())
89  {
91  CVCapture::instance()->lastFrameGray,
92  adSv->grab);
93  // reset grabbing switch
94  adSv->grab = false;
95 
96  stringstream ss;
97  ss << "Click on the screen to create a calibration photo. Created "
100  s->info(ss.str());
101  }
102  else if (AppCommon::calibrationEstimator->isBusyExtracting())
103  {
104  // also reset grabbing, user has to click again
105  adSv->grab = false;
107  CVCapture::instance()->lastFrameGray,
108  false);
109  s->info("Busy extracting corners, please wait with grabbing ...");
110  }
111  else if (AppCommon::calibrationEstimator->isCalculating())
112  {
114  CVCapture::instance()->lastFrameGray,
115  false);
116  s->info("Calculating calibration, please wait ...");
117  }
118  else if (AppCommon::calibrationEstimator->isDone())
119  {
120  if (!processedCalibResult)
121  {
122  if (AppCommon::calibrationEstimator->calibrationSuccessful())
123  {
124  processedCalibResult = true;
126 
127  std::string computerInfo = Utils::ComputerInfos::get();
128  string mainCalibFilename = "camCalib_" + computerInfo + "_main.xml";
129  string scndCalibFilename = "camCalib_" + computerInfo + "_scnd.xml";
130  std::string errorMsg;
131  if (ac->calibration.save(AppCommon::calibFilePath, mainCalibFilename))
132  {
133 
134 #ifndef SL_EMSCRIPTEN
136  mainCalibFilename,
141  errorMsg))
142  {
143  Utils::log("WAIApp", errorMsg.c_str());
144  }
145 #endif
146  }
147  else
148  {
149  errorMsg += " Saving calibration failed!";
150  }
151 
152  s->info("Calibration successful." + errorMsg);
153  }
154  else
155  {
156  s->info(("Calibration failed!"));
157  }
158  }
159  }
160  else if (AppCommon::calibrationEstimator->isDoneCaptureAndSave())
161  {
162  s->info(("Capturing done!"));
163  }
164  }
166  {
167  log("SLProject", e.what());
168  s->info("Exception during calibration! Please restart!");
169  }
170 }
SLSceneView * sv
Definition: SLGLImGui.h:28
static const string CALIB_FTP_HOST
ftp host for calibration up and download
Definition: AppCommon.h:114
static const string CALIB_FTP_USER
ftp login user for calibration up and download
Definition: AppCommon.h:115
static const string CALIB_FTP_DIR
ftp directory for calibration up and download
Definition: AppCommon.h:117
static SLstring calibIniPath
That's where data/calibrations folder is located.
Definition: AppCommon.h:108
static SLstring exePath
executable root path
Definition: AppCommon.h:80
static const string CALIB_FTP_PWD
ftp login pwd for calibration up and download
Definition: AppCommon.h:116
static CVCalibrationEstimatorParams calibrationEstimatorParams
Definition: AppCommon.h:106
static SLstring externalPath
Default path for external file storage.
Definition: AppCommon.h:82
static SLstring calibFilePath
That's where calibrations are stored and loaded from.
Definition: AppCommon.h:109
special exception that informs about errors during calibration process
CVCalibration getCalibration()
Get resulting calibration.
bool updateAndDecorate(CVMat imageColor, const CVMat &imageGray, bool grabFrame, bool drawCorners=true)
< Finds the inner chessboard corners in the given image
bool save(const string &calibDir, const string &calibFileName)
Saves the camera calibration parameters to the config file.
bool uploadFile(const string &fileDir, const string &fileName, const string &ftpHost, const string &ftpUser, const string &ftpPwd, const string &ftpDir, string &errorMsg)
Uploads file to the ftp server.
Definition: FtpUtils.cpp:198
void errorMsg(const char *tag, const char *msg, const int line, const char *file)
Platform independent error message output.
Definition: Utils.cpp:1168
void log(const char *tag, const char *format,...)
logs a formatted string platform independently
Definition: Utils.cpp:1103

◆ updateTrackingSceneCamera()

void updateTrackingSceneCamera ( CVCamera ac)

always update scene camera fovV from calibration because the calibration may have been adapted in adjustForSL after a change of aspect ratio! Attention: The active scene view camera may be a different one that the tracking camera but we have to update the tracking camera only!

Definition at line 51 of file AppDemoVideo.cpp.

52 {
54 
55  if (gVideoTrackedNode && typeid(*gVideoTrackedNode) == typeid(SLCamera))
56  {
57  SLCamera* trackingCam = dynamic_cast<SLCamera*>(gVideoTrackedNode);
58  trackingCam->fov(ac->calibration.cameraFovVDeg());
59  }
60 }
float cameraFovVDeg() const
void fov(const SLfloat fov)
vertical field of view
Definition: SLCamera.h:98

Variable Documentation

◆ gVideoTexture

SLGLTexture* gVideoTexture = nullptr

Global pointer for the video texture defined in AppDemoLoad for video scenes It gets updated in the following onUpdateTracking routine

Definition at line 33 of file AppDemoVideo.cpp.

◆ gVideoTrackedNode

SLNode* gVideoTrackedNode = nullptr

Global pointer to a node that from witch the gVideoTracker changes the pose. it gets updated in the following onUpdateTracking routine

Definition at line 43 of file AppDemoVideo.cpp.

◆ gVideoTracker

CVTracked* gVideoTracker = nullptr

Global pointer for a gVideoTracker that is set in AppDemoLoad for video scenes It gets updated in the following onUpdateTracking routine

Definition at line 38 of file AppDemoVideo.cpp.