00001 //# TSMCube.h: Tiled hypercube in a table 00002 //# Copyright (C) 1995,1996,1997,1999,2000,2001,2002 00003 //# Associated Universities, Inc. Washington DC, USA. 00004 //# 00005 //# This library is free software; you can redistribute it and/or modify it 00006 //# under the terms of the GNU Library General Public License as published by 00007 //# the Free Software Foundation; either version 2 of the License, or (at your 00008 //# option) any later version. 00009 //# 00010 //# This library is distributed in the hope that it will be useful, but WITHOUT 00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 00012 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public 00013 //# License for more details. 00014 //# 00015 //# You should have received a copy of the GNU Library General Public License 00016 //# along with this library; if not, write to the Free Software Foundation, 00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. 00018 //# 00019 //# Correspondence concerning AIPS++ should be addressed as follows: 00020 //# Internet email: aips2-request@nrao.edu. 00021 //# Postal address: AIPS++ Project Office 00022 //# National Radio Astronomy Observatory 00023 //# 520 Edgemont Road 00024 //# Charlottesville, VA 22903-2475 USA 00025 //# 00026 //# $Id: TSMCube.h 21521 2014-12-10 08:06:42Z gervandiepen $ 00027 00028 #ifndef TABLES_TSMCUBE_H 00029 #define TABLES_TSMCUBE_H 00030 00031 00032 //# Includes 00033 #include <casacore/casa/aips.h> 00034 #include <casacore/tables/DataMan/TSMShape.h> 00035 #include <casacore/casa/Containers/Record.h> 00036 #include <casacore/casa/Arrays/IPosition.h> 00037 #include <casacore/casa/OS/Conversion.h> 00038 #include <casacore/casa/iosfwd.h> 00039 00040 namespace casacore { //# NAMESPACE CASACORE - BEGIN 00041 00042 //# Forward declarations 00043 class TiledStMan; 00044 class TSMFile; 00045 class TSMColumn; 00046 class BucketCache; 00047 template<class T> class Block; 00048 00049 // <summary> 00050 // Tiled hypercube in a table 00051 // </summary> 00052 00053 // <use visibility=local> 00054 00055 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests=""> 00056 // </reviewed> 00057 00058 // <prerequisite> 00059 //# Classes you should understand before using this one. 00060 // <li> <linkto class=TiledStMan>TiledStMan</linkto> 00061 // <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto> 00062 // for a discussion of the maximum cache size 00063 // <li> <linkto class=TSMFile>TSMFile</linkto> 00064 // <li> <linkto class=BucketCache>BucketCache</linkto> 00065 // </prerequisite> 00066 00067 // <etymology> 00068 // TSMCube represents a hypercube in the Tiled Storage Manager. 00069 // </etymology> 00070 00071 // <synopsis> 00072 // TSMCube defines a tiled hypercube. The data is stored in a TSMFile 00073 // object and accessed using a BucketCache object. The hypercube can 00074 // be extensible in its last dimension to support tables with a size 00075 // which is not known in advance. 00076 // <br> 00077 // Normally hypercubes share the same TSMFile object, but extensible 00078 // hypercubes have their own TSMFile object (to be extensible). 00079 // If the hypercolumn has multiple data columns, their cells share the same 00080 // tiles. Per tile data column A appears first, thereafter B, etc.. 00081 // <br> 00082 // The data in the cache is held in external format and is converted 00083 // when accessed. The alternative would be to hold it in the cache in 00084 // local format and convert it when read/written from the file. It was 00085 // felt that the latter approach would generate more needless conversions. 00086 // <p> 00087 // The possible id and coordinate values are stored in a Record 00088 // object. They are written in the main hypercube AipsIO file. 00089 // <p> 00090 // TSMCube uses the maximum cache size set for a Tiled Storage manager. 00091 // The description of class 00092 // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto> 00093 // contains a discussion about the effect of setting the maximum cache size. 00094 // </synopsis> 00095 00096 // <motivation> 00097 // TSMCube encapsulates all operations on a hypercube. 00098 // </motivation> 00099 00100 //# <todo asof="$DATE:$"> 00101 //# A List of bugs, limitations, extensions or planned refinements. 00102 //# </todo> 00103 00104 00105 class TSMCube 00106 { 00107 public: 00108 // Define the possible access types for TSMDataColumn. 00109 enum AccessType { 00110 NoAccess, 00111 CellAccess, 00112 SliceAccess, 00113 ColumnAccess, 00114 ColumnSliceAccess 00115 }; 00116 00117 // Construct the hypercube using the given file with the given shape. 00118 // The record contains the id and possible coordinate values. 00119 // <br>If the cubeshape is empty, the hypercube is still undefined and 00120 // can be added later with setShape. That is only used by TiledCellStMan. 00121 // <br> The fileOffset argument is meant for class TiledFileAccess. 00122 TSMCube (TiledStMan* stman, TSMFile* file, 00123 const IPosition& cubeShape, 00124 const IPosition& tileShape, 00125 const Record& values, 00126 Int64 fileOffset, 00127 Bool useDerived = False); 00128 00129 // Reconstruct the hypercube by reading its data from the AipsIO stream. 00130 // It will link itself to the correct TSMFile. The TSMFile objects 00131 // must have been reconstructed in advance. 00132 TSMCube (TiledStMan* stman, AipsIO& ios, 00133 Bool useDerived = False); 00134 00135 virtual ~TSMCube(); 00136 00137 // Flush the data in the cache. 00138 virtual void flushCache(); 00139 00140 // Clear the cache, so data will be reread. 00141 // If wanted, the data is flushed before the cache is cleared. 00142 void clearCache (Bool doFlush = True); 00143 00144 // Empty the cache. 00145 // It will flush the cache as needed and remove all buckets from it 00146 // resulting in a possibly large drop in memory used. 00147 // It'll also clear the <src>userSetCache_p</src> flag. 00148 void emptyCache(); 00149 00150 // Show the cache statistics. 00151 virtual void showCacheStatistics (ostream& os) const; 00152 00153 // Put the data of the object into the AipsIO stream. 00154 void putObject (AipsIO& ios); 00155 00156 // Get the data of the object from the AipsIO stream. 00157 // It returns the data manager sequence number, which is -1 if 00158 // no file is attached to the cube (for cells without a value). 00159 Int getObject (AipsIO& ios); 00160 00161 // Resync the object with the data file. 00162 // It reads the object, and adjusts the cache. 00163 virtual void resync (AipsIO& ios); 00164 00165 // Is the hypercube extensible? 00166 Bool isExtensible() const; 00167 00168 // Get the bucket size (which is the length of a tile in external format). 00169 uInt bucketSize() const; 00170 00171 // Get the length of a tile in local format. 00172 uInt localTileLength() const; 00173 00174 // Set the hypercube shape. 00175 // This is only possible if the shape was not defined yet. 00176 virtual void setShape (const IPosition& cubeShape, 00177 const IPosition& tileShape); 00178 00179 // Get the shape of the hypercube. 00180 const IPosition& cubeShape() const; 00181 00182 // Get the shape of the tiles. 00183 const IPosition& tileShape() const; 00184 00185 // Get the shape of the data cells in the cube. 00186 IPosition cellShape() const; 00187 00188 // Get the size of a coordinate (i.e. the number of values in it). 00189 // If not defined, it returns zero. 00190 uInt coordinateSize (const String& coordinateName) const; 00191 00192 // Get the record containing the id and coordinate values. 00193 // It is used by TSMIdColumn and TSMCoordColumn. 00194 // <group> 00195 const Record& valueRecord() const; 00196 Record& rwValueRecord(); 00197 // </group> 00198 00199 // Test if the id values match. 00200 Bool matches (const PtrBlock<TSMColumn*>& idColSet, 00201 const Record& idValues); 00202 00203 // Extend the last dimension of the cube with the given number. 00204 // The record can contain the coordinates of the elements added. 00205 virtual void extend (uInt nr, const Record& coordValues, 00206 const TSMColumn* lastCoordColumn); 00207 00208 // Extend the coordinates vector for the given coordinate 00209 // to the given length with the given coordValues. 00210 // It will be initialized to zero if no coordValues are given. 00211 // If the coordinate vector does not exist yet, it will be created. 00212 void extendCoordinates (const Record& coordValues, 00213 const String& coordName, uInt length); 00214 00215 // Read or write a section in the cube. 00216 // It is assumed that the section buffer is long enough. 00217 virtual void accessSection (const IPosition& start, const IPosition& end, 00218 char* section, uInt colnr, 00219 uInt localPixelSize, uInt externalPixelSize, 00220 Bool writeFlag); 00221 00222 // Read or write a section in a strided way. 00223 // It is assumed that the section buffer is long enough. 00224 virtual void accessStrided (const IPosition& start, const IPosition& end, 00225 const IPosition& stride, 00226 char* section, uInt colnr, 00227 uInt localPixelSize, uInt externalPixelSize, 00228 Bool writeFlag); 00229 00230 // Get the current cache size (in buckets). 00231 uInt cacheSize() const; 00232 00233 // Calculate the cache size (in buckets) for the given slice 00234 // and access path. 00235 // <group> 00236 uInt calcCacheSize (const IPosition& sliceShape, 00237 const IPosition& windowStart, 00238 const IPosition& windowLength, 00239 const IPosition& axisPath) const; 00240 static uInt calcCacheSize (const IPosition& cubeShape, 00241 const IPosition& tileShape, 00242 Bool extensible, 00243 const IPosition& sliceShape, 00244 const IPosition& windowStart, 00245 const IPosition& windowLength, 00246 const IPosition& axisPath, 00247 uInt maxCacheSize, uInt bucketSize); 00248 // </group> 00249 00250 // Set the cache size for the given slice and access path. 00251 virtual void setCacheSize (const IPosition& sliceShape, 00252 const IPosition& windowStart, 00253 const IPosition& windowLength, 00254 const IPosition& axisPath, 00255 Bool forceSmaller, Bool userSet); 00256 00257 // Resize the cache object. 00258 // If forceSmaller is False, the cache will only be resized when it grows. 00259 // If the given size exceeds the maximum size with more 00260 // than 10%, the maximum size will be used. 00261 // The cacheSize has to be given in buckets. 00262 // <br>The flag <src>userSet</src> inidicates if the cache size is set by 00263 // the user (by an Accessor object) or automatically (by TSMDataColumn). 00264 virtual void setCacheSize (uInt cacheSize, Bool forceSmaller, Bool userSet); 00265 00266 // Validate the cache size (in buckets). 00267 // This means it will return the given cache size if smaller 00268 // than the maximum cache size. Otherwise the maximum is returned. 00269 // <group> 00270 uInt validateCacheSize (uInt cacheSize) const; 00271 static uInt validateCacheSize (uInt cacheSize, uInt maxSize, 00272 uInt bucketSize); 00273 // </group> 00274 00275 // Determine if the user set the cache size (using setCacheSize). 00276 Bool userSetCache() const; 00277 00278 // Functions for TSMDataColumn to keep track of the last type of 00279 // access to a hypercube. It uses it to determine if the cache 00280 // has to be reset. 00281 // <group> 00282 AccessType getLastColAccess() const; 00283 const IPosition& getLastColSlice() const; 00284 void setLastColAccess (AccessType type); 00285 void setLastColSlice (const IPosition& slice); 00286 // </group> 00287 00288 protected: 00289 // Initialize the various variables. 00290 // <group> 00291 void setup(); 00292 void setupNrTiles(); 00293 // </group> 00294 00295 // Adjust the tile shape to the hypercube shape. 00296 // A size of 0 gets set to 1. 00297 // A tile size > cube size gets set to the cube size. 00298 IPosition adjustTileShape (const IPosition& cubeShape, 00299 const IPosition& tileShape) const; 00300 00301 // Resize the IPosition member variables used in accessSection() 00302 // if nrdim_p changes value. 00303 void resizeTileSections(); 00304 00305 private: 00306 // Forbid copy constructor. 00307 TSMCube (const TSMCube&); 00308 00309 // Forbid assignment. 00310 TSMCube& operator= (const TSMCube&); 00311 00312 // Get the cache object. 00313 // This will construct the cache object if not present yet. 00314 BucketCache* getCache(); 00315 00316 // Construct the cache object (if not constructed yet). 00317 virtual void makeCache(); 00318 00319 // Resync the cache object. 00320 virtual void resyncCache(); 00321 00322 // Delete the cache object. 00323 virtual void deleteCache(); 00324 00325 // Access a line in a more optimized way. 00326 void accessLine (char* section, uInt pixelOffset, 00327 uInt localPixelSize, 00328 Bool writeFlag, BucketCache* cachePtr, 00329 const IPosition& startTile, uInt endTile, 00330 const IPosition& startPixelInFirstTile, 00331 uInt endPixelInLastTile, 00332 uInt lineIndex); 00333 00334 // Define the callback functions for the BucketCache. 00335 // <group> 00336 static char* readCallBack (void* owner, const char* external); 00337 static void writeCallBack (void* owner, char* external, 00338 const char* local); 00339 static char* initCallBack (void* owner); 00340 static void deleteCallBack (void* owner, char* buffer); 00341 // </group> 00342 00343 // Define the functions doing the actual read and write of the 00344 // data in the tile and converting it to/from local format. 00345 // <group> 00346 char* readTile (const char* external); 00347 void writeTile (char* external, const char* local); 00348 // </group> 00349 00350 protected: 00351 //# Declare member variables. 00352 00353 char * cachedTile_p; // optimization to hold one tile chunk 00354 00355 // Pointer to the parent storage manager. 00356 TiledStMan* stmanPtr_p; 00357 // Is the class used directly or only by a derived class only? 00358 Bool useDerived_p; 00359 // The values of the possible id and coordinate columns. 00360 Record values_p; 00361 // Is the hypercube extensible? 00362 Bool extensible_p; 00363 // Dimensionality of the hypercube. 00364 uInt nrdim_p; 00365 // Number of tiles in the hypercube. 00366 uInt nrTiles_p; 00367 // The shape of the hypercube. 00368 IPosition cubeShape_p; 00369 // The shape of the tiles in the hypercube. 00370 IPosition tileShape_p; 00371 // The number of tiles in each hypercube dimension. 00372 IPosition tilesPerDim_p; 00373 // Precomputed tileShape information. 00374 TSMShape expandedTileShape_p; 00375 // Precomputed tilesPerDim information. 00376 TSMShape expandedTilesPerDim_p; 00377 // Number of tiles in all but last dimension (used when extending). 00378 uInt nrTilesSubCube_p; 00379 // The tilesize in pixels. 00380 uInt tileSize_p; 00381 // Pointer to the TSMFile object holding the data. 00382 TSMFile* filePtr_p; 00383 // Offset in the TSMFile object where the data of this hypercube starts. 00384 Int64 fileOffset_p; 00385 // Offset for each data column in a tile (in external format). 00386 Block<uInt> externalOffset_p; 00387 // Offset for each data column in a tile (in local format). 00388 Block<uInt> localOffset_p; 00389 // The bucket size in bytes (is equal to tile size in bytes). 00390 uInt bucketSize_p; 00391 // The tile size in bytes in local format. 00392 uInt localTileLength_p; 00393 // The bucket cache. 00394 BucketCache* cache_p; 00395 // Did the user set the cache size? 00396 Bool userSetCache_p; 00397 // Was the last column access to a cell, slice, or column? 00398 AccessType lastColAccess_p; 00399 // The slice shape of the last column access to a slice. 00400 IPosition lastColSlice_p; 00401 00402 // IPosition variables used in accessSection(); declared here 00403 // as member variables to avoid significant construction and 00404 // desctruction overhead if they are local to accessSection() 00405 // #tiles needed for the section 00406 IPosition nrTileSection_p; 00407 // First tile needed 00408 IPosition startTile_p; 00409 // Last tile needed 00410 IPosition endTile_p; 00411 // First pixel in first tile 00412 IPosition startPixelInFirstTile_p; 00413 // Last pixel in first tile 00414 IPosition endPixelInFirstTile_p; 00415 // Last pixel in last tile 00416 IPosition endPixelInLastTile_p; 00417 }; 00418 00419 00420 00421 inline BucketCache* TSMCube::getCache() 00422 { 00423 if (cache_p == 0) { 00424 makeCache(); 00425 } 00426 return cache_p; 00427 } 00428 inline uInt TSMCube::bucketSize() const 00429 { 00430 return bucketSize_p; 00431 } 00432 inline uInt TSMCube::localTileLength() const 00433 { 00434 return localTileLength_p; 00435 } 00436 inline const IPosition& TSMCube::cubeShape() const 00437 { 00438 return cubeShape_p; 00439 } 00440 inline const IPosition& TSMCube::tileShape() const 00441 { 00442 return tileShape_p; 00443 } 00444 inline const Record& TSMCube::valueRecord() const 00445 { 00446 return values_p; 00447 } 00448 inline Record& TSMCube::rwValueRecord() 00449 { 00450 return values_p; 00451 } 00452 inline Bool TSMCube::userSetCache() const 00453 { 00454 return userSetCache_p; 00455 } 00456 inline TSMCube::AccessType TSMCube::getLastColAccess() const 00457 { 00458 return lastColAccess_p; 00459 } 00460 inline const IPosition& TSMCube::getLastColSlice() const 00461 { 00462 return lastColSlice_p; 00463 } 00464 inline void TSMCube::setLastColAccess (TSMCube::AccessType type) 00465 { 00466 lastColAccess_p = type; 00467 } 00468 inline void TSMCube::setLastColSlice (const IPosition& slice) 00469 { 00470 lastColSlice_p.resize (slice.nelements()); 00471 lastColSlice_p = slice; 00472 } 00473 00474 00475 00476 00477 } //# NAMESPACE CASACORE - END 00478 00479 #endif