00001 //# TiledStMan.h: Base class for Tiled Storage Managers 00002 //# Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003 00003 //# Associated Universities, Inc. Washington DC, USA. 00004 //# 00005 //# This library is free software; you can redistribute it and/or modify it 00006 //# under the terms of the GNU Library General Public License as published by 00007 //# the Free Software Foundation; either version 2 of the License, or (at your 00008 //# option) any later version. 00009 //# 00010 //# This library is distributed in the hope that it will be useful, but WITHOUT 00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 00012 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public 00013 //# License for more details. 00014 //# 00015 //# You should have received a copy of the GNU Library General Public License 00016 //# along with this library; if not, write to the Free Software Foundation, 00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. 00018 //# 00019 //# Correspondence concerning AIPS++ should be addressed as follows: 00020 //# Internet email: aips2-request@nrao.edu. 00021 //# Postal address: AIPS++ Project Office 00022 //# National Radio Astronomy Observatory 00023 //# 520 Edgemont Road 00024 //# Charlottesville, VA 22903-2475 USA 00025 //# 00026 //# $Id$ 00027 00028 #ifndef TABLES_TILEDSTMAN_H 00029 #define TABLES_TILEDSTMAN_H 00030 00031 00032 //# Includes 00033 #include <casacore/casa/aips.h> 00034 #include <casacore/tables/DataMan/DataManager.h> 00035 #include <casacore/casa/Containers/Block.h> 00036 #include <casacore/casa/Arrays/IPosition.h> 00037 #include <casacore/casa/OS/Conversion.h> 00038 #include <casacore/casa/BasicSL/String.h> 00039 00040 namespace casacore { //# NAMESPACE CASACORE - BEGIN 00041 00042 //# Forward Declarations 00043 class TSMColumn; 00044 class TSMDataColumn; 00045 class TSMCube; 00046 class TSMFile; 00047 class TableDesc; 00048 class Record; 00049 template<class T> class Vector; 00050 00051 00052 // <summary> 00053 // Base class for Tiled Storage Manager classes 00054 // </summary> 00055 00056 // <use visibility=export> 00057 00058 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests=""> 00059 // </reviewed> 00060 00061 // <prerequisite> 00062 //# Classes you should understand before using this one. 00063 // <li> Description of Tiled Storage Manager in module file 00064 // <linkto module=Tables:TiledStMan>Tables.h</linkto> 00065 // <li> <linkto class=DataManager>DataManager</linkto> 00066 // <li> <linkto class=TSMColumn>TSMColumn</linkto> 00067 // <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto> 00068 // for a discussion of the maximum cache size 00069 // </prerequisite> 00070 00071 // <synopsis> 00072 // TiledStMan is the base class for Tiled Storage Managers. 00073 // A tiled storage manager is capable of storing a hypercolumn 00074 // (as defined by <linkto file="TableDesc.h#defineHypercolumn"> 00075 // TableDesc::defineHypercolumn</linkto>) 00076 // in one or more hypercubes. 00077 // <br>It is not necessary to define a hypercolumn. If not defined, 00078 // it is assumed that all columns bound to this storage manager are 00079 // data columns. At least one of the columns must have a fixed 00080 // dimensionality and is used to determine the hypercube dimnensionality. 00081 // <br>The general concept of these storage managers is explained in the 00082 // <linkto module="Tables:TiledStMan">Tables module description</linkto>. 00083 // <p> 00084 // TiledStMan contains all common functions for the different tiled 00085 // storage managers. In particular, it contains functions 00086 // to check if the definitions of the shapes of hypercubes, coordinates, and 00087 // data cells are consistent. 00088 // It also contains various data members and functions to make them 00089 // persistent by writing them into an AipsIO stream. 00090 // </synopsis> 00091 00092 // <motivation> 00093 // This base class contains the common functionality of all 00094 // tiled storage managers. The base class is still abstract. 00095 // Only concrete tiled storage managers derived from it can 00096 // be instantiated. 00097 // <p> 00098 // Tiled storage managers make access to array data possible with 00099 // more or less the same efficiency for access along different axes. 00100 // </motivation> 00101 00102 //# <todo asof="$DATE:$"> 00103 //# A List of bugs, limitations, extensions or planned refinements. 00104 //# </todo> 00105 00106 00107 class TiledStMan : public DataManager 00108 { 00109 public: 00110 // Create a TiledStMan. 00111 TiledStMan(); 00112 00113 // Create a TiledStMan storage manager. 00114 // The given maximum cache size is persistent, 00115 // thus will be reused when the table is read back. Note that the class 00116 // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto> 00117 // allows one to overwrite the maximum cache size temporarily. 00118 // Its description contains a discussion about the effects of 00119 // setting a maximum cache. 00120 TiledStMan (const String& hypercolumnName, uInt maximumCacheSize); 00121 00122 virtual ~TiledStMan(); 00123 00124 // Get the name given to the storage manager. 00125 // This is the name of the hypercolumn. 00126 virtual String dataManagerName() const; 00127 00128 void setDataManagerName (const String& newHypercolumnName); 00129 00130 // Return a record containing data manager specifications. 00131 virtual Record dataManagerSpec() const; 00132 00133 // Get data manager properties that can be modified. 00134 // It is only ActualCacheSize (the actual cache size in buckets). 00135 // It is a subset of the data manager specification. 00136 virtual Record getProperties() const; 00137 00138 // Modify data manager properties. 00139 // Only ActualCacheSize can be used. It is similar to function setCacheSize 00140 // with <src>canExceedNrBuckets=False</src>. 00141 virtual void setProperties (const Record& spec); 00142 00143 // Set the flag to "data has changed since last flush". 00144 void setDataChanged(); 00145 00146 // Derive the tile shape from the hypercube shape for the given 00147 // number of pixels per tile. It is tried to get the same number 00148 // of tiles for each dimension. 00149 // When a weight vector is given, the number of tiles for a dimension 00150 // is proportional to the weight. 00151 // <br>After the initial guess it tries to optimize it by trying 00152 // to waste as little space as possible, while trying to keep as close 00153 // to the initial guess. The given tolerance (possibly per axis) 00154 // gives the minimum and maximum possible length of a tile axis 00155 // (minimum = initial_guess*tolerance; maximum = initial_guess/tolerance). 00156 // The heuristic is such that a tile axis length dividing the cube length 00157 // exactly is always favoured. 00158 // The test program <src>tTiledStMan</src> can be used to see how 00159 // the algorithm works out for a given tile size and cube shape. 00160 // <group> 00161 static IPosition makeTileShape (const IPosition& hypercubeShape, 00162 Double tolerance = 0.5, 00163 uInt maxNrPixelsPerTile = 32768); 00164 static IPosition makeTileShape (const IPosition& hypercubeShape, 00165 const Vector<double>& weight, 00166 const Vector<double>& tolerance, 00167 uInt maxNrPixelsPerTile = 32768); 00168 // </group> 00169 00170 // Set the maximum cache size (in bytes) in a non-persistent way. 00171 virtual void setMaximumCacheSize (uInt nbytes); 00172 00173 // Get the current maximum cache size (in bytes). 00174 uInt maximumCacheSize() const; 00175 00176 // Get the current cache size (in buckets) for the hypercube in 00177 // the given row. 00178 uInt cacheSize (uInt rownr) const; 00179 00180 // Get the hypercube shape of the data in the given row. 00181 const IPosition& hypercubeShape (uInt rownr) const; 00182 00183 // Get the tile shape of the data in the given row. 00184 const IPosition& tileShape (uInt rownr) const; 00185 00186 // Get the bucket size (in bytes) of the hypercube in the given row. 00187 uInt bucketSize (uInt rownr) const; 00188 00189 // Can the tiled storage manager handle changing array shapes? 00190 // The default is no (but TiledCellStMan can). 00191 virtual Bool canChangeShape() const; 00192 00193 // Can the tiled storage manager access an entire column. 00194 // TiledColumnStMan can always do that. 00195 // The others might be able to do it (for this time). 00196 // The default implementation returns True if there is only 1 hypercube. 00197 // reask is set to True (because next time things might be different). 00198 virtual Bool canAccessColumn (Bool& reask) const; 00199 00200 // The data manager supports use of MultiFile. 00201 virtual Bool hasMultiFileSupport() const; 00202 00203 // Calculate the cache size (in buckets) for accessing the hypercube 00204 // containing the given row. It takes the maximum cache size into 00205 // account (allowing an overdraft of 10%). 00206 // It uses the given axisPath (i.e. traversal order) to determine 00207 // the optimum size. A window can be specified to indicate that only 00208 // the given subset of the hypercube will be accessed. 00209 // <br> 00210 // The length of the slice and window arguments and <src>axisPath</src> 00211 // must be less or equal to the dimensionality of the hypercube. 00212 // The non-specified <src>windowStart</src> parts default to 0. 00213 // The non-specified <src>windowLength</src> parts default to 00214 // the hypercube shape. 00215 // The non-specified <src>sliceShape</src> parts default to 1. 00216 // <br> 00217 // Axispath = [2,0,1] indicates that the z-axis changes most rapidly, 00218 // thereafter x and y. An axis can occur only once in the axisPath. 00219 // The non-specified <src>axisPath</src> parts get the natural order. 00220 // E.g. in the previous example axisPath=[2] defines the same path. 00221 // <br>When forceSmaller is False, the cache is not resized when the 00222 // new size is smaller. 00223 // <br>A flag is set indicating that the TSMDataColumn 00224 // access functions do not need to size the cache. 00225 uInt calcCacheSize (uInt rownr, const IPosition& sliceShape, 00226 const IPosition& windowStart, 00227 const IPosition& windowLength, 00228 const IPosition& axisPath) const; 00229 00230 // Set the cache size using the <src>calcCacheSize</src> 00231 // function mentioned above. 00232 void setCacheSize (uInt rownr, const IPosition& sliceShape, 00233 const IPosition& windowStart, 00234 const IPosition& windowLength, 00235 const IPosition& axisPath, 00236 Bool forceSmaller); 00237 00238 // Set the cache size for accessing the hypercube containing the given row. 00239 // When the give cache size exceeds the maximum cache size with more 00240 // than 10%, the maximum cache size is used instead. 00241 // <br>When forceSmaller is False, the cache is not resized when the 00242 // new size is smaller. 00243 // <br>A flag is set indicating that the TSMDataColumn 00244 // access functions do not need to size the cache. 00245 void setCacheSize (uInt rownr, uInt nbuckets, Bool forceSmaller); 00246 00247 // Sets the cache size using the hypercube instead of the row number. 00248 // Useful for iterating over all hypercubes. 00249 void setHypercubeCacheSize (uInt hypercube, uInt nbuckets, Bool forceSmaller); 00250 00251 // Determine if the user set the cache size (using setCacheSize). 00252 Bool userSetCache (uInt rownr) const; 00253 00254 // Empty the caches used by the hypercubes in this storage manager. 00255 // It will flush the caches as needed and remove all buckets from them 00256 // resulting in a possibly large drop in memory used. 00257 // It also clears the userSetCache flag. 00258 void emptyCaches(); 00259 00260 // Show the statistics of all caches used. 00261 void showCacheStatistics (ostream& os) const; 00262 00263 // Get the length of the data for the given number of pixels. 00264 // This can be used to calculate the length of a tile. 00265 uInt getLengthOffset (uInt nrPixels, Block<uInt>& dataOffset, 00266 Block<uInt>& localOffset, 00267 uInt& localTileLength) const; 00268 00269 // Get the number of coordinate vectors. 00270 uInt nrCoordVector() const; 00271 00272 // Get the nr of rows in this storage manager. 00273 uInt nrow() const; 00274 00275 // Does the storage manager allow to add rows? (yes) 00276 Bool canAddRow() const; 00277 00278 // Get the default tile shape. 00279 // By default it returns a zero-length IPosition. 00280 virtual IPosition defaultTileShape() const; 00281 00282 // Return the number of hypercubes. 00283 uInt nhypercubes() const; 00284 00285 // Test if only one hypercube is used by this storage manager. 00286 // If not, throw an exception. Otherwise return the hypercube. 00287 virtual TSMCube* singleHypercube(); 00288 00289 // Get the given hypercube. 00290 // <group> 00291 const TSMCube* getTSMCube (uInt hypercube) const; 00292 TSMCube* getTSMCube (uInt hypercube); 00293 // </group> 00294 00295 // Get the hypercube in which the given row is stored. 00296 // <group> 00297 const TSMCube* getHypercube (uInt rownr) const; 00298 virtual TSMCube* getHypercube (uInt rownr) = 0; 00299 // </group> 00300 00301 // Get the hypercube in which the given row is stored. 00302 // It also returns the position of the row in that hypercube. 00303 virtual TSMCube* getHypercube (uInt rownr, IPosition& position) = 0; 00304 00305 // Make the correct TSMCube type (depending on tsmOption()). 00306 TSMCube* makeTSMCube (TSMFile* file, const IPosition& cubeShape, 00307 const IPosition& tileShape, 00308 const Record& values, Int64 fileOffset=-1); 00309 00310 // Read a tile and convert the data to local format. 00311 void readTile (char* local, const Block<uInt>& localOffset, 00312 const char* external, const Block<uInt>& externalOffset, 00313 uInt nrpixels); 00314 00315 // Write a tile after converting the data to external format. 00316 void writeTile (char* external, const Block<uInt>& externalOffset, 00317 const char* local, const Block<uInt>& localOffset, 00318 uInt nrpixels); 00319 00320 // Get the TSMFile object with the given sequence number. 00321 TSMFile* getFile (uInt sequenceNumber); 00322 00323 // Open the storage manager for an existing table. 00324 virtual void open (uInt nrrow, AipsIO&); 00325 00326 // Resync the storage manager with the new file contents. 00327 virtual void resync (uInt nrrow); 00328 00329 // Reopen all files used in this storage manager for read/write access. 00330 virtual void reopenRW(); 00331 00332 // The data manager will be deleted (because all its columns are 00333 // requested to be deleted). 00334 // So clean up the things needed (e.g. delete files). 00335 virtual void deleteManager(); 00336 00337 // Create a column in the storage manager on behalf of a table column. 00338 // <group> 00339 // Create a scalar column. 00340 DataManagerColumn* makeScalarColumn (const String& name, int dataType, 00341 const String& dataTypeID); 00342 // Create a direct array column. 00343 DataManagerColumn* makeDirArrColumn (const String& name, int dataType, 00344 const String& dataTypeID); 00345 // Create an indirect array column. 00346 DataManagerColumn* makeIndArrColumn (const String& name, int dataType, 00347 const String& dataTypeID); 00348 // </group> 00349 00350 // The TiledStMan wants to do reallocateColumn. 00351 Bool canReallocateColumns() const; 00352 00353 // Reallocate the column object if it is part of this data manager. 00354 // It returns a pointer to the new column object. 00355 // It is used to remove the indirection of the TSMColumn objects 00356 // resulting in only one iso. two virtual column calls to get the data. 00357 DataManagerColumn* reallocateColumn (DataManagerColumn* column); 00358 00359 // Set the shape and tile shape of a hypercube. 00360 // By default it throws an "impossible" exception. 00361 virtual void setShape (uInt rownr, TSMCube* hypercube, 00362 const IPosition& shape, 00363 const IPosition& tileShape); 00364 00365 // Check the shape to be set for a hypercube. 00366 // It checks if it matches predefined (fixed shape) columns 00367 // and the shape of already defined coordinate columns. 00368 void checkCubeShape (const TSMCube* hypercube, 00369 const IPosition& cubeShape) const; 00370 00371 // Get the data type of the coordinate column with the given name. 00372 // An exception is thrown when the column is unknown. 00373 int coordinateDataType (const String& columnName) const; 00374 00375 // Initialize the new coordinates for the given cube. 00376 void initCoordinates (TSMCube* hypercube); 00377 00378 // Get pointer to data column object. 00379 const TSMDataColumn* getDataColumn (uInt colnr) const 00380 { return dataCols_p[colnr]; } 00381 00382 protected: 00383 // Set the persistent maximum cache size. 00384 void setPersMaxCacheSize (uInt nbytes); 00385 00386 // Get the bindings of the columns with the given names. 00387 // If bound, the pointer to the TSMColumn object is stored in the block. 00388 // If mustExist is True, an exception is thrown if the column 00389 // is not bound. 00390 // It returns the number of bound columns. 00391 uInt getBindings (const Vector<String>& columnNames, 00392 PtrBlock<TSMColumn*>& colSet, 00393 Bool mustExist) const; 00394 00395 // Function setup calls this function to allow the derived class 00396 // to check specific information. In case of errors, an exception 00397 // should be thrown. 00398 // By default it does nothing. 00399 virtual void setupCheck (const TableDesc& tableDesc, 00400 const Vector<String>& dataNames) const; 00401 00402 // Get the table description needed for the hypercolumn description. 00403 virtual const TableDesc& getDesc() const; 00404 00405 // Check if values are given in the record for all columns in 00406 // the block. Also check if the data types are correct. 00407 // An exception is thrown if something is incorrect. 00408 void checkValues (const PtrBlock<TSMColumn*>& colSet, 00409 const Record& values) const; 00410 00411 // Check if the coordinate values are correct. 00412 // This calls checkValues and checks if their shapes match the 00413 // hypercube shape. 00414 // An exception is thrown if invalid. 00415 void checkCoordinates (const PtrBlock<TSMColumn*>& coordColSet, 00416 const IPosition& cubeShape, 00417 const Record& values) const; 00418 00419 // Check if the shapes of FixedShape data and coordinate columns match. 00420 // An exception is thrown if not. 00421 void checkShapeColumn (const IPosition& shape) const; 00422 00423 // Check if the cube shape matches that of defined coordinates. 00424 void checkCoordinatesShapes (const TSMCube* hypercube, 00425 const IPosition& cubeShape) const; 00426 00427 // Check if the hypercube to be added is correctly defined. 00428 void checkAddHypercube (const IPosition& cubeShape, 00429 const Record& values) const; 00430 00431 // Make a new TSMCube object. 00432 TSMCube* makeHypercube (const IPosition& cubeShape, 00433 const IPosition& tileShape, 00434 const Record& values); 00435 00436 // Get the index of the hypercube with the given id-values. 00437 // If not found, -1 is returned. 00438 Int getCubeIndex (const Record& idValues) const; 00439 00440 // Determine how many rows need to be added for an extension 00441 // (in the last dimension) of a hypercube with the given shape. 00442 uInt addedNrrow (const IPosition& shape, uInt incrInLastDim) const; 00443 00444 // Flush the caches of all hypercubes. 00445 // If data have put and fsync is set, fsync all files. 00446 Bool flushCaches (Bool fsync); 00447 00448 // Let a derived class read the header info. 00449 // This is used by the open and resync function. 00450 virtual void readHeader (uInt nrrow, Bool firstTime) = 0; 00451 00452 // Create the TSM header file. 00453 // It creates an AipsIO object for it. 00454 AipsIO* headerFileCreate(); 00455 00456 // Open the TSM header file. 00457 // It creates an AipsIO object for it. 00458 AipsIO* headerFileOpen(); 00459 00460 // Write the data into the header file. 00461 // The given number of TSMCube objects have to be written. 00462 void headerFilePut (AipsIO& headerFile, uInt nrCube); 00463 00464 // Read the data from the header file. 00465 // When done for the first time, setup() is called to initialize 00466 // the various variables (using the extraNdim variable). 00467 void headerFileGet (AipsIO& headerFile, uInt tabNrrow, Bool firstTime, 00468 Int extraNdim); 00469 00470 // Close the header file. 00471 // It deletes the AipsIO object. 00472 void headerFileClose (AipsIO* headerFile); 00473 00474 // Set up the TiledStMan variables from the table description. 00475 // The argument specifies the number of extra dimensions for the 00476 // hypercube compared to the data array (usually 0 or 1). 00477 // It is only used if no hypercolumn definition exists. 00478 // -1 means that the hypercolumn definition has to be present. 00479 void setup (Int extraNdim=-1); 00480 00481 // Create a TSMFile object and store its pointer at the given index 00482 // in the block. 00483 void createFile (uInt index); 00484 00485 // Convert the scalar data type to an array data type. 00486 // This function is temporary and can disappear when the ColumnDesc 00487 // classes use type TpArray*. 00488 int arrayDataType (int dataType) const; 00489 00490 00491 //# Declare all data members. 00492 // The name of the hypercolumn. 00493 String hypercolumnName_p; 00494 // The number of rows in the columns. 00495 uInt nrrow_p; 00496 // The assembly of all columns. 00497 PtrBlock<TSMColumn*> colSet_p; 00498 // The assembly of all data columns. 00499 PtrBlock<TSMDataColumn*> dataCols_p; 00500 PtrBlock<TSMColumn*> dataColSet_p; 00501 // The assembly of all id columns. 00502 PtrBlock<TSMColumn*> idColSet_p; 00503 // The assembly of all coordinate columns. 00504 PtrBlock<TSMColumn*> coordColSet_p; 00505 // The assembly of all TSMFile objects. 00506 // The first file is for all non-extensible cubes, while the others 00507 // are for one file per extensible cube. 00508 PtrBlock<TSMFile*> fileSet_p; 00509 // The assembly of all TSMCube objects. 00510 PtrBlock<TSMCube*> cubeSet_p; 00511 // The persistent maximum cache size for a hypercube. 00512 uInt persMaxCacheSize_p; 00513 // The actual maximum cache size for a hypercube. 00514 uInt maxCacheSize_p; 00515 // The dimensionality of the hypercolumn. 00516 uInt nrdim_p; 00517 // The number of vector coordinates. 00518 uInt nrCoordVector_p; 00519 // The fixed cell shape. 00520 IPosition fixedCellShape_p; 00521 // Has any data changed since the last flush? 00522 Bool dataChanged_p; 00523 00524 private: 00525 // Forbid copy constructor. 00526 TiledStMan (const TiledStMan&); 00527 00528 // Forbid assignment. 00529 TiledStMan& operator= (const TiledStMan&); 00530 }; 00531 00532 00533 inline uInt TiledStMan::maximumCacheSize() const 00534 { return maxCacheSize_p; } 00535 00536 inline uInt TiledStMan::nrCoordVector() const 00537 { return nrCoordVector_p; } 00538 00539 inline uInt TiledStMan::nrow() const 00540 { return nrrow_p; } 00541 00542 inline uInt TiledStMan::nhypercubes() const 00543 { return cubeSet_p.nelements(); } 00544 00545 inline void TiledStMan::setDataChanged() 00546 { dataChanged_p = True; } 00547 00548 inline const TSMCube* TiledStMan::getTSMCube (uInt hypercube) const 00549 { return const_cast<TiledStMan*>(this)->getTSMCube (hypercube); } 00550 00551 inline const TSMCube* TiledStMan::getHypercube (uInt rownr) const 00552 { return const_cast<TiledStMan*>(this)->getHypercube (rownr); } 00553 00554 inline void TiledStMan::setPersMaxCacheSize (uInt nbytes) 00555 { 00556 persMaxCacheSize_p = nbytes; 00557 maxCacheSize_p = nbytes; 00558 } 00559 00560 00561 00562 00563 } //# NAMESPACE CASACORE - END 00564 00565 #endif