Martin Lambers

Add conversion of PMD range to cartesian coordinates to simulation library.

This is for convenience. See the changes to the PMD example program.
......@@ -144,6 +144,7 @@ int main(int argc, char* argv[])
output.rgb = true;
output.srgb = true;
output.pmd = true;
output.pmdCoordinates = true;
output.depthAndRange = true;
CamSim::Simulator simulator;
......@@ -183,32 +184,13 @@ int main(int argc, char* argv[])
{ simulator.getSRGB(0), simulator.getSRGB(1), simulator.getSRGB(2), simulator.getSRGB(3) });
// PMD simulation result: range, amplitude, intensity
exporter.asyncExportData("pmd-result.pfs", simulator.getPMD());
// PMD simulation results: cartesian coordinates (computed from range)
exporter.asyncExportData("pmd-coordinates.pfs", simulator.getPMDCoordinates());
// RGB simulation result
exporter.asyncExportData("rgb-result.ppm", simulator.getSRGB());
// Grount Truth depth and range values
exporter.asyncExportData("groundtruth-depthrange.pfs", simulator.getDepthAndRange());
// PMD simulation produces range data, i.e. the distance to the sensor center.
// You can compute 3D cartesian coordinates from range data by using the camera
// intrinsic parameters:
CamSim::TexData pmdResult = simulator.getPMD();
const float* pmdResultData = static_cast<const float*>(pmdResult.packedData());
QVector<QVector3D> pmdCoordData(pmdResult.width() * pmdResult.height());
for (int y = 0; y < pmdResult.height(); y++) {
float v = (y - projection.centerPixel().y()) / projection.focalLengths().y();
for (int x = 0; x < pmdResult.width(); x++) {
float u = (x - projection.centerPixel().x()) / projection.focalLengths().x();
float w = 1.0f;
float range = pmdResultData[3 * (y * pmdResult.width() + x) + 0];
QVector3D xyz = QVector3D(u, v, w).normalized() * range;
pmdCoordData[y * pmdResult.width() + x] = xyz;
}
}
exporter.exportData("pmd-coordinates.pfs", CamSim::TexData(
pmdResult.width(), pmdResult.height(), 3, GL_FLOAT, QByteArray::fromRawData(
reinterpret_cast<const char*>(pmdCoordData.constData()),
pmdCoordData.size() * sizeof(QVector3D))));
frameCounter++;
}
exporter.waitForAsyncExports();
......
......@@ -15,6 +15,8 @@
<file>simulation-rgb-result-fs.glsl</file>
<file>simulation-pmd-result-vs.glsl</file>
<file>simulation-pmd-result-fs.glsl</file>
<file>simulation-pmd-coords-vs.glsl</file>
<file>simulation-pmd-coords-fs.glsl</file>
<file>convert-to-srgb-vs.glsl</file>
<file>convert-to-srgb-fs.glsl</file>
<file>simulation-postproc-lensdistortion-vs.glsl</file>
......
/*
* Copyright (C) 2018
* Computer Graphics Group, University of Siegen
* Written by Martin Lambers <martin.lambers@uni-siegen.de>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#version 450
const float pi = 3.14159265358979323846;
uniform sampler2D pmd_result_tex;
uniform float w, h; // width and height of image
uniform float fx, fy; // focal lengths
uniform float cx, cy; // center pixel
smooth in vec2 vtexcoord;
layout(location = 0) out vec3 fcoords;
void main(void)
{
float px = vtexcoord.x * w - 0.5; // pixel coordinate x
float py = (1.0 - vtexcoord.y) * h - 0.5; // pixel coordinate y
float range = texture(pmd_result_tex, vtexcoord).r;
vec3 uvw = vec3((px - cx) / fx, (py - cy) / fy, 1.0);
vec3 xyz = normalize(uvw) * range;
fcoords = xyz;
}
/*
* Copyright (C) 2018
* Computer Graphics Group, University of Siegen
* Written by Martin Lambers <martin.lambers@uni-siegen.de>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#version 450
layout(location = 0) in vec4 position;
layout(location = 1) in vec2 texcoord;
smooth out vec2 vtexcoord;
void main(void)
{
vtexcoord = texcoord;
gl_Position = position;
}
......@@ -183,6 +183,7 @@ Output::Output() :
rgb(true),
srgb(false),
pmd(false),
pmdCoordinates(false),
eyeSpacePositions(false),
customSpacePositions(false),
eyeSpaceNormals(false),
......@@ -205,6 +206,7 @@ Simulator::Simulator() :
_rgbTexOversampled(0),
_pmdEnergyTexOversampled(0),
_pmdEnergyTex(0),
_pmdCoordinatesTex(0),
_gaussianNoiseTex(0),
_postProcessingTex(0),
_fbo(0),
......@@ -399,6 +401,7 @@ void Simulator::recreateShadersIfNecessary()
_zeroPrg.removeAllShaders();
_rgbResultPrg.removeAllShaders();
_pmdResultPrg.removeAllShaders();
_pmdCoordinatesPrg.removeAllShaders();
_geomPrg.removeAllShaders();
_flowPrg.removeAllShaders();
_convertToSRGBPrg.removeAllShaders();
......@@ -654,6 +657,15 @@ void Simulator::recreateShadersIfNecessary()
_convertToSRGBPrg.addShaderFromSourceCode(QOpenGLShader::Fragment, convFs);
_convertToSRGBPrg.link();
}
// conversion from PMD range to coordinates
if (_output.pmdCoordinates) {
QString pmdCoordinatesVs = readFile(":/libcamsim/simulation-pmd-coords-vs.glsl");
QString pmdCoordinatesFs = readFile(":/libcamsim/simulation-pmd-coords-fs.glsl");
_pmdCoordinatesPrg.addShaderFromSourceCode(QOpenGLShader::Vertex, pmdCoordinatesVs);
_pmdCoordinatesPrg.addShaderFromSourceCode(QOpenGLShader::Fragment, pmdCoordinatesFs);
_pmdCoordinatesPrg.link();
_pmdCoordinatesPrg.bind();
}
}
// Geometry simulation program
......@@ -864,6 +876,8 @@ void Simulator::recreateOutputIfNecessary()
_pmdEnergyTexOversampled = 0;
gl->glDeleteTextures(1, &_pmdEnergyTex);
_pmdEnergyTex = 0;
gl->glDeleteTextures(1, &_pmdCoordinatesTex);
_pmdCoordinatesTex = 0;
gl->glDeleteTextures(1, &_gaussianNoiseTex);
_gaussianNoiseTex = 0;
_gaussianWhiteNoiseBuf.clear();
......@@ -994,6 +1008,10 @@ void Simulator::recreateOutputIfNecessary()
prepareOutputTexs(spatialOversamplingSize(), { _pmdEnergyTexOversampled }, GL_RG32F, false);
gl->glGenTextures(1, &_pmdEnergyTex);
prepareOutputTexs(_projection.imageSize(), { _pmdEnergyTex }, GL_RG32F, false);
if (_output.pmdCoordinates) {
gl->glGenTextures(1, &_pmdCoordinatesTex);
prepareOutputTexs(_projection.imageSize(), { _pmdCoordinatesTex }, GL_RGBA32F, false);
}
}
int extra = (subFrames() > 1 ? 1 : 0);
if (_output.rgb && _pipeline.gaussianWhiteNoise) {
......@@ -1871,6 +1889,24 @@ void Simulator::simulatePMDResult()
ASSERT_GLCHECK();
}
void Simulator::simulatePMDCoordinates()
{
auto gl = getGlFunctionsFromCurrentContext(Q_FUNC_INFO);
ASSERT_GLCHECK();
_pmdCoordinatesPrg.bind();
_pmdCoordinatesPrg.setUniformValue("w", static_cast<float>(_projection.imageSize().width()));
_pmdCoordinatesPrg.setUniformValue("h", static_cast<float>(_projection.imageSize().height()));
_pmdCoordinatesPrg.setUniformValue("fx", _projection.focalLengths().x());
_pmdCoordinatesPrg.setUniformValue("fy", _projection.focalLengths().y());
_pmdCoordinatesPrg.setUniformValue("cx", _projection.centerPixel().x());
_pmdCoordinatesPrg.setUniformValue("cy", _projection.centerPixel().y());
gl->glActiveTexture(GL_TEXTURE0);
gl->glBindTexture(GL_TEXTURE_2D, _pmdDigNumTexs[subFrames()]);
gl->glBindVertexArray(_fullScreenQuadVao);
gl->glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
ASSERT_GLCHECK();
}
void Simulator::simulateGeometry(int subFrame)
{
simulate(_geomPrg, subFrame,
......@@ -2060,6 +2096,10 @@ void Simulator::simulate(long long frameTimestamp)
if (_output.pmd) {
prepareFBO(_projection.imageSize(), 0, false, { _pmdDigNumTexs[subFrames()] });
simulatePMDResult();
if (_output.pmdCoordinates) {
prepareFBO(_projection.imageSize(), 0, false, { _pmdCoordinatesTex });
simulatePMDCoordinates();
}
}
if (_output.forwardFlow3D || _output.forwardFlow2D || _output.backwardFlow3D || _output.backwardFlow2D) {
long long lastFrameTimestamp = frameTimestamp - frameDuration();
......@@ -2143,6 +2183,11 @@ unsigned int Simulator::getPMDTex(int i) const
return ((_output.pmd && haveValidOutput(i)) ? (i == -1 ? _pmdDigNumTexs.last() : _pmdDigNumTexs[i]) : 0);
}
unsigned int Simulator::getPMDCoordinatesTex() const
{
return ((_output.pmd && _output.pmdCoordinates && haveValidOutput(-1)) ? _pmdCoordinatesTex : 0);
}
unsigned int Simulator::getEyeSpacePositionsTex(int i) const
{
return ((_output.eyeSpacePositions && haveValidOutput(i)) ? (i == -1 ? _eyeSpacePosTexs[0] : _eyeSpacePosTexs[i]) : 0);
......
......@@ -278,8 +278,10 @@ public:
bool rgb;
/*! \brief Flag: enable output of sRGB colors (only if \a rgb is also true)? */
bool srgb;
/*! \brief Flag: enable output of PMD phase image (energy, A tap, B tap)? */
/*! \brief Flag: enable output of PMD results (subframes/phase images and final result)? */
bool pmd;
/*! \brief Flag: enable output of PMD cartesian coordinates (only if \a pmd is also true)? */
bool pmdCoordinates;
/*@}*/
......@@ -362,6 +364,7 @@ private:
QOpenGLShaderProgram _zeroPrg; // produce all-zero output
QOpenGLShaderProgram _rgbResultPrg; // combine rgb subframes to final result
QOpenGLShaderProgram _pmdResultPrg; // combine pmd phase subframes to final result
QOpenGLShaderProgram _pmdCoordinatesPrg; // compute cartesian coordinates from pmd range
QOpenGLShaderProgram _geomPrg; // simulate geometry (pos, normals, depth, range, indices) information
QOpenGLShaderProgram _flowPrg; // simulate 2D/3D flow information
QOpenGLShaderProgram _convertToSRGBPrg; // convert linear RGB to sRGB
......@@ -381,6 +384,7 @@ private:
unsigned int _rgbTexOversampled;
unsigned int _pmdEnergyTexOversampled;
unsigned int _pmdEnergyTex;
unsigned int _pmdCoordinatesTex;
unsigned int _gaussianNoiseTex; // for shot noise generation, dynamically generated
QVector<float> _gaussianWhiteNoiseBuf; // reused vector to generate gaussian white noise
QVector<unsigned int> _gaussianWhiteNoiseTexs;// subFrames
......@@ -466,6 +470,7 @@ private:
void simulateGaussianWhiteNoise(int subFrame);
void simulateRGBResult();
void simulatePMDResult();
void simulatePMDCoordinates();
void simulateGeometry(int subFrame);
void simulateFlow(int subFrame, long long lastT, long long nextT, unsigned int lastDepthBuf);
void simulatePostprocLensDistortion(const QList<unsigned int>& textures);
......@@ -706,6 +711,17 @@ public:
return TexData(getPMDTex(i), -1, -1, GL_RGBA32F, { "a_minus_b", "a_plus_b", "a", "b" }, _pbo);
}
/*! \brief Get the output texture containing PMD simulated cartesian
* coordinates. These are computed from the simulated PMD range value, taking the camera intrinsic parameters into
* account (image size, center pixel and focal lengths). Note that lens distortion is currently ignored. */
unsigned int getPMDCoordinatesTex() const;
/*! \brief Convenience wrapper for \a getPMDCoordinatesTex() */
TexData getPMDCoordinates() const
{
return TexData(getPMDCoordinatesTex(), -1, -1, GL_RGB32F, { "x", "y", "z" }, _pbo);
}
/*! \brief Get the output texture containing eye space positions
* for the given subframe \a i or the final result (if \a i is -1).
* Positions of the final result are always the same as those for the first subframe. */
......