Cr8-xyz/blender
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fluid: Updated Mantaflow source files

Browse Source 
Includes improvements for the file IO. Namely, more meta data will be written from now on.

This change is required to prevent IO issues (e.g. T84649) that arised through the use of sparse grids caching (introduced in 2.92).
This commit is contained in:
Sebastián Barschkis
2021-02-02 17:46:48 +01:00
parent f8359b5f52
commit 9ad828dbad
10 changed files with 212 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
extern/mantaflow/preprocessed/fileio/iogrids.cpp vendored
View File

@@ -628,13 +628,24 @@ template<class T> int readGridUni(const string &name, Grid<T> *grid)
// current file format
UniHeader head;
assertMsg(gzread(gzf, &head, sizeof(UniHeader)) == sizeof(UniHeader),
"can't read file, no header present");
assertMsg(head.dimX == grid->getSizeX() && head.dimY == grid->getSizeY() &&
head.dimZ == grid->getSizeZ(),
"grid dim doesn't match, " << Vec3(head.dimX, head.dimY, head.dimZ) << " vs "
<< grid->getSize());
"readGridUni: Can't read file, no header present");
assertMsg(unifyGridType(head.gridType) == unifyGridType(grid->getType()),
"grid type doesn't match " << head.gridType << " vs " << grid->getType());
"readGridUni: Grid type doesn't match " << head.gridType << " vs "
<< grid->getType());
const Vec3i curGridSize = grid->getParent()->getGridSize();
const Vec3i headGridSize(head.dimX, head.dimY, head.dimZ);
# if BLENDER
// Correct grid size is only a soft requirement in Blender
if (headGridSize != curGridSize) {
debMsg("readGridUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize, 1);
return 0;
}
# else
assertMsg(headGridSize == curGridSize,
"readGridUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize);
# endif
# if FLOATINGPOINT_PRECISION != 1
// convert float to double
Grid<T> temp(grid->getParent());

26
extern/mantaflow/preprocessed/fileio/ioparticles.cpp vendored
View File

@@ -230,6 +230,19 @@ int readParticlesUni(const std::string &name, BasicParticleSystem *parts)
assertMsg(((head.bytesPerElement == PartSysSize) && (head.elementType == 0)),
"particle type doesn't match");
const Vec3i curGridSize = parts->getParent()->getGridSize();
const Vec3i headGridSize(head.dimX, head.dimY, head.dimZ);
# if BLENDER
// Correct grid size is only a soft requirement in Blender
if (headGridSize != curGridSize) {
debMsg("readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize, 1);
return 0;
}
# else
assertMsg(headGridSize == curGridSize,
"readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize);
# endif
// re-allocate all data
parts->resizeAll(head.dim);
@@ -325,6 +338,19 @@ template<class T> int readPdataUni(const std::string &name, ParticleDataImpl<T>
pdata->getParticleSys()->resize(head.dim); // ensure that parent particle system has same size
pdata->resize(head.dim);
const Vec3i curGridSize = pdata->getParent()->getGridSize();
const Vec3i headGridSize(head.dimX, head.dimY, head.dimZ);
# if BLENDER
// Correct grid size is only a soft requirement in Blender
if (headGridSize != curGridSize) {
debMsg("readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize, 1);
return 0;
}
# else
assertMsg(headGridSize == curGridSize,
"readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize);
# endif
assertMsg(head.dim == pdata->size(), "pdata size doesn't match");
# if FLOATINGPOINT_PRECISION != 1
ParticleDataImpl<T> temp(pdata->getParent());

7
extern/mantaflow/preprocessed/fileio/ioutil.cpp vendored
View File

@@ -90,6 +90,13 @@ template<> void convertFrom(openvdb::Vec3s &in, Vec3 *out)
(*out).z = in.z();
}
template<> void convertFrom(openvdb::Vec3i &in, Vec3i *out)
{
(*out).x = in.x();
(*out).y = in.y();
(*out).z = in.z();
}
// Convert to OpenVDB value from Manta value.
template<class S, class T> void convertTo(S *out, T &in)
{

153
extern/mantaflow/preprocessed/fileio/iovdb.cpp vendored
View File

@@ -38,6 +38,11 @@
#define POSITION_NAME "P"
#define FLAG_NAME "U"
#define META_BASE_RES "file_base_resolution"
#define META_VOXEL_SIZE "file_voxel_size"
#define META_BBOX_MAX "file_bbox_max"
#define META_BBOX_MIN "file_bbox_min"
using namespace std;
namespace Manta {
@@ -388,7 +393,8 @@ int writeObjectsVDB(const string &filename,
int compression,
int precision,
float clip,
const Grid<Real> *clipGrid)
const Grid<Real> *clipGrid,
const bool meta)
{
openvdb::initialize();
openvdb::io::File file(filename);
@@ -489,6 +495,16 @@ int writeObjectsVDB(const string &filename,
// Set additional grid attributes, e.g. name, grid class, compression level, etc.
if (vdbGrid) {
setGridOptions<openvdb::GridBase>(vdbGrid, objectName, gClass, voxelSize, precision);
// Optional metadata: Save additional simulation information per vdb object
if (meta) {
const Vec3i size = object->getParent()->getGridSize();
// The (dense) resolution of this grid
vdbGrid->insertMeta(META_BASE_RES,
openvdb::Vec3IMetadata(openvdb::Vec3i(size.x, size.y, size.z)));
// Length of one voxel side
vdbGrid->insertMeta(META_VOXEL_SIZE, openvdb::FloatMetadata(voxelSize));
}
}
}
@@ -533,6 +549,44 @@ int writeObjectsVDB(const string &filename,
return 1;
}
static void clearAll(std::vector<PbClass *> *objects, std::vector<ParticleDataBase *> pdbBuffer)
{
// Clear all data loaded into manta objects (e.g. during IO error)
for (std::vector<PbClass *>::iterator iter = objects->begin(); iter != objects->end(); ++iter) {
if (GridBase *mantaGrid = dynamic_cast<GridBase *>(*iter)) {
if (mantaGrid->getType() & GridBase::TypeInt) {
Grid<int> *mantaIntGrid = (Grid<int> *)mantaGrid;
mantaIntGrid->clear();
}
else if (mantaGrid->getType() & GridBase::TypeReal) {
Grid<Real> *mantaRealGrid = (Grid<Real> *)mantaGrid;
mantaRealGrid->clear();
}
else if (mantaGrid->getType() & GridBase::TypeVec3) {
Grid<Vec3> *mantaVec3Grid = (Grid<Vec3> *)mantaGrid;
mantaVec3Grid->clear();
}
}
else if (BasicParticleSystem *mantaPP = dynamic_cast<BasicParticleSystem *>(*iter)) {
mantaPP->clear();
}
}
for (ParticleDataBase *pdb : pdbBuffer) {
if (pdb->getType() == ParticleDataBase::TypeInt) {
ParticleDataImpl<int> *mantaPDataInt = (ParticleDataImpl<int> *)pdb;
mantaPDataInt->clear();
}
else if (pdb->getType() == ParticleDataBase::TypeReal) {
ParticleDataImpl<Real> *mantaPDataReal = (ParticleDataImpl<Real> *)pdb;
mantaPDataReal->clear();
}
else if (pdb->getType() == ParticleDataBase::TypeVec3) {
ParticleDataImpl<Vec3> *mantaPDataVec3 = (ParticleDataImpl<Vec3> *)pdb;
mantaPDataVec3->clear();
}
}
}
int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, float worldSize)
{
@@ -561,6 +615,9 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
// A buffer to store a handle to pData objects. These will be read alongside a particle system.
std::vector<ParticleDataBase *> pdbBuffer;
// Count how many objects could not be read correctly
int readFailure = 0;
for (std::vector<PbClass *>::iterator iter = objects->begin(); iter != objects->end(); ++iter) {
if (gridsVDB.empty()) {
@@ -568,11 +625,12 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
}
// If there is just one grid in this file, load it regardless of name match (to vdb caches per
// grid).
bool onlyGrid = (gridsVDB.size() == 1);
const bool onlyGrid = (gridsVDB.size() == 1);
PbClass *object = dynamic_cast<PbClass *>(*iter);
const Real dx = object->getParent()->getDx();
const Real voxelSize = worldSize * dx;
const Vec3i origRes = object->getParent()->getGridSize();
Real voxelSize = worldSize * dx;
// Particle data objects are treated separately - buffered and inserted when reading the
// particle system
@@ -596,6 +654,81 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
if (!nameMatch && !onlyGrid) {
continue;
}
// Metadata: If present in the file, meta data will be parsed into these fields
Real metaVoxelSize(0);
Vec3i metaRes(0), metaBBoxMax(0), metaBBoxMin(0);
// Loop to load all meta data that we care about
for (openvdb::MetaMap::MetaIterator iter = vdbGrid->beginMeta(); iter != vdbGrid->endMeta();
++iter) {
const std::string &name = iter->first;
const openvdb::Metadata::Ptr value = iter->second;
if (name.compare(META_BASE_RES) == 0) {
openvdb::Vec3i tmp = static_cast<openvdb::Vec3IMetadata &>(*value).value();
convertFrom(tmp, &metaRes);
}
else if (name.compare(META_VOXEL_SIZE) == 0) {
float tmp = static_cast<openvdb::FloatMetadata &>(*value).value();
convertFrom(tmp, &metaVoxelSize);
voxelSize = metaVoxelSize; // Make sure to update voxel size variable (used in
// pointgrid's importVDB())
if (worldSize != 1.0)
debMsg(
"readObjectsVDB: Found voxel size in meta data. worldSize parameter will be "
"ignored!",
1);
}
else if (name.compare(META_BBOX_MAX) == 0) {
openvdb::Vec3i tmp = static_cast<openvdb::Vec3IMetadata &>(*value).value();
convertFrom(tmp, &metaBBoxMax);
}
else if (name.compare(META_BBOX_MIN) == 0) {
openvdb::Vec3i tmp = static_cast<openvdb::Vec3IMetadata &>(*value).value();
convertFrom(tmp, &metaBBoxMin);
}
else {
debMsg("readObjectsVDB: Skipping unknown meta information '" << name << "'", 1);
}
}
// Compare metadata with allocated grid setup. This prevents invalid index access.
if (notZero(metaRes) && metaRes != origRes) {
debMsg("readObjectsVDB Warning: Grid '" << vdbGrid->getName()
<< "' has not been read. Meta grid res " << metaRes
<< " vs " << origRes << " current grid size",
1);
readFailure++;
break;
}
if (notZero(metaVoxelSize) && metaVoxelSize != voxelSize) {
debMsg("readObjectsVDB Warning: Grid '"
<< vdbGrid->getName() << "' has not been read. Meta voxel size "
<< metaVoxelSize << " vs " << voxelSize << " current voxel size",
1);
readFailure++;
break;
}
if (metaBBoxMax.x > origRes.x || metaBBoxMax.y > origRes.y || metaBBoxMax.z > origRes.z) {
debMsg("readObjectsVDB Warning: Grid '"
<< vdbGrid->getName() << "' has not been read. Vdb bbox max " << metaBBoxMax
<< " vs " << origRes << " current grid size",
1);
readFailure++;
break;
}
const Vec3i origOrigin(0);
if (metaBBoxMin.x < origOrigin.x || metaBBoxMin.y < origOrigin.y ||
metaBBoxMin.z < origOrigin.z) {
debMsg("readObjectsVDB Warning: Grid '"
<< vdbGrid->getName() << "' has not been read. Vdb bbox min " << metaBBoxMin
<< " vs " << origOrigin << " current grid origin",
1);
readFailure++;
break;
}
if (GridBase *mantaGrid = dynamic_cast<GridBase *>(*iter)) {
if (mantaGrid->getType() & GridBase::TypeInt) {
@@ -655,6 +788,17 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
return 0;
}
}
// Do not continue loading objects in this loop if there was a read error
if (readFailure > 0) {
break;
}
}
if (readFailure > 0) {
// Clear all data that has already been loaded into simulation objects
clearAll(objects, pdbBuffer);
pdbBuffer.clear();
return 0;
}
// Give out a warning if pData items were present but could not be read due to missing particle
@@ -729,7 +873,8 @@ int writeObjectsVDB(const string &filename,
int compression,
int precision,
float clip,
const Grid<Real> *clipGrid)
const Grid<Real> *clipGrid,
const bool meta)
{
errMsg("Cannot save to .vdb file. Mantaflow has not been built with OpenVDB support.");
return 0;

9
extern/mantaflow/preprocessed/fileio/mantaio.cpp vendored
View File

@@ -85,7 +85,8 @@ int save(const string &name,
bool precisionHalf = true,
int precision = PRECISION_HALF,
float clip = 1e-4,
const Grid<Real> *clipGrid = nullptr)
const Grid<Real> *clipGrid = nullptr,
const bool meta = false)
{
if (!precisionHalf) {
@@ -105,7 +106,7 @@ int save(const string &name,
return writeGridsVol(name, &objects);
if (ext == ".vdb")
return writeObjectsVDB(
name, &objects, worldSize, skipDeletedParts, compression, precision, clip, clipGrid);
name, &objects, worldSize, skipDeletedParts, compression, precision, clip, clipGrid, meta);
else if (ext == ".npz")
return writeGridsNumpy(name, &objects);
else if (ext == ".txt")
@@ -134,6 +135,7 @@ static PyObject *_W_1(PyObject *_self, PyObject *_linargs, PyObject *_kwds)
int precision = _args.getOpt<int>("precision", 6, PRECISION_HALF, &_lock);
float clip = _args.getOpt<float>("clip", 7, 1e-4, &_lock);
const Grid<Real> *clipGrid = _args.getPtrOpt<Grid<Real>>("clipGrid", 8, nullptr, &_lock);
const bool meta = _args.getOpt<bool>("meta", 9, false, &_lock);
_retval = toPy(save(name,
objects,
worldSize,
@@ -142,7 +144,8 @@ static PyObject *_W_1(PyObject *_self, PyObject *_linargs, PyObject *_kwds)
precisionHalf,
precision,
clip,
clipGrid));
clipGrid,
meta));
_args.check();
}
pbFinalizePlugin(parent, "save", !noTiming);

3
extern/mantaflow/preprocessed/fileio/mantaio.h vendored
View File

@@ -77,7 +77,8 @@ int writeObjectsVDB(const std::string &filename,
int compression = COMPRESSION_ZIP,
int precision = PRECISION_HALF,
float clip = 1e-4,
const Grid<Real> *clipGrid = nullptr);
const Grid<Real> *clipGrid = nullptr,
const bool meta = false);
int readObjectsVDB(const std::string &filename,
std::vector<PbClass *> *objects,
float scale = 1.0);

2
extern/mantaflow/preprocessed/gitinfo.h vendored
View File

@@ -1,3 +1,3 @@
#define MANTA_GIT_VERSION "commit e2285cb9bc492987f728123be6cfc1fe11fe73d6"
#define MANTA_GIT_VERSION "commit 1c86d86496e7f7473c36248d12ef07bf4d9d2840"

6
extern/mantaflow/preprocessed/grid.cpp vendored
View File

@@ -508,8 +508,7 @@ struct CompMaxVec : public KernelBase {
template<class T> Grid<T> &Grid<T>::copyFrom(const Grid<T> &a, bool copyType)
{
assertMsg(a.mSize.x == mSize.x && a.mSize.y == mSize.y && a.mSize.z == mSize.z,
"different grid resolutions " << a.mSize << " vs " << this->mSize);
assertMsg(a.mSize == mSize, "different grid resolutions " << a.mSize << " vs " << this->mSize);
memcpy(mData, a.mData, sizeof(T) * mSize.x * mSize.y * mSize.z);
if (copyType)
mType = a.mType; // copy type marker
@@ -3402,8 +3401,7 @@ void PbRegister_markIsolatedFluidCell()
void copyMACData(
const MACGrid &source, MACGrid &target, const FlagGrid &flags, const int flag, const int bnd)
{
assertMsg(source.getSize().x == target.getSize().x && source.getSize().y == target.getSize().y &&
source.getSize().z == target.getSize().z,
assertMsg(source.getSize() == target.getSize(),
"different grid resolutions " << source.getSize() << " vs " << target.getSize());
// Grid<Real> divGrid(target.getParent());

1
extern/mantaflow/preprocessed/grid.h vendored
View File

@@ -596,6 +596,7 @@ template<class T> class Grid : public GridBase {
//! set data
inline void set(int i, int j, int k, T &val)
{
DEBUG_ONLY(checkIndex(i, j, k));
mData[index(i, j, k)] = val;
}

4
extern/mantaflow/preprocessed/grid4d.cpp vendored
View File

@@ -491,9 +491,7 @@ template<class T> Grid4d<T> &Grid4d<T>::safeDivide(const Grid4d<T> &a)
}
template<class T> Grid4d<T> &Grid4d<T>::copyFrom(const Grid4d<T> &a, bool copyType)
{
assertMsg(a.mSize.x == mSize.x && a.mSize.y == mSize.y && a.mSize.z == mSize.z &&
a.mSize.t == mSize.t,
"different Grid4d resolutions " << a.mSize << " vs " << this->mSize);
assertMsg(a.mSize == mSize, "different Grid4d resolutions " << a.mSize << " vs " << this->mSize);
memcpy(mData, a.mData, sizeof(T) * mSize.x * mSize.y * mSize.z * mSize.t);
if (copyType)
mType = a.mType; // copy type marker