TiledStMan.h

Go to the documentation of this file.
00001 //# TiledStMan.h: Base class for Tiled Storage Managers
00002 //# Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
00003 //# Associated Universities, Inc. Washington DC, USA.
00004 //#
00005 //# This library is free software; you can redistribute it and/or modify it
00006 //# under the terms of the GNU Library General Public License as published by
00007 //# the Free Software Foundation; either version 2 of the License, or (at your
00008 //# option) any later version.
00009 //#
00010 //# This library is distributed in the hope that it will be useful, but WITHOUT
00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
00012 //# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
00013 //# License for more details.
00014 //#
00015 //# You should have received a copy of the GNU Library General Public License
00016 //# along with this library; if not, write to the Free Software Foundation,
00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
00018 //#
00019 //# Correspondence concerning AIPS++ should be addressed as follows:
00020 //#        Internet email: aips2-request@nrao.edu.
00021 //#        Postal address: AIPS++ Project Office
00022 //#                        National Radio Astronomy Observatory
00023 //#                        520 Edgemont Road
00024 //#                        Charlottesville, VA 22903-2475 USA
00025 //#
00026 //# $Id$
00027 
00028 #ifndef TABLES_TILEDSTMAN_H
00029 #define TABLES_TILEDSTMAN_H
00030 
00031 
00032 //# Includes
00033 #include <casacore/casa/aips.h>
00034 #include <casacore/tables/DataMan/DataManager.h>
00035 #include <casacore/casa/Containers/Block.h>
00036 #include <casacore/casa/Arrays/IPosition.h>
00037 #include <casacore/casa/OS/Conversion.h>
00038 #include <casacore/casa/BasicSL/String.h>
00039 
00040 namespace casacore { //# NAMESPACE CASACORE - BEGIN
00041 
00042 //# Forward Declarations
00043 class TSMColumn;
00044 class TSMDataColumn;
00045 class TSMCube;
00046 class TSMFile;
00047 class TableDesc;
00048 class Record;
00049 template<class T> class Vector;
00050 
00051 
00052 // <summary>
00053 // Base class for Tiled Storage Manager classes
00054 // </summary>
00055 
00056 // <use visibility=export>
00057 
00058 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
00059 // </reviewed>
00060 
00061 // <prerequisite>
00062 //# Classes you should understand before using this one.
00063 //   <li> Description of Tiled Storage Manager in module file
00064 //        <linkto module=Tables:TiledStMan>Tables.h</linkto>
00065 //   <li> <linkto class=DataManager>DataManager</linkto>
00066 //   <li> <linkto class=TSMColumn>TSMColumn</linkto>
00067 //   <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
00068 //        for a discussion of the maximum cache size
00069 // </prerequisite>
00070 
00071 // <synopsis> 
00072 // TiledStMan is the base class for Tiled Storage Managers.
00073 // A tiled storage manager is capable of storing a hypercolumn
00074 // (as defined by <linkto file="TableDesc.h#defineHypercolumn">
00075 // TableDesc::defineHypercolumn</linkto>)
00076 // in one or more hypercubes.
00077 // <br>It is not necessary to define a hypercolumn. If not defined,
00078 // it is assumed that all columns bound to this storage manager are
00079 // data columns. At least one of the columns must have a fixed
00080 // dimensionality and is used to determine the hypercube dimnensionality.
00081 // <br>The general concept of these storage managers is explained in the
00082 // <linkto module="Tables:TiledStMan">Tables module description</linkto>.
00083 // <p>
00084 // TiledStMan contains all common functions for the different tiled
00085 // storage managers. In particular, it contains functions
00086 // to check if the definitions of the shapes of hypercubes, coordinates, and
00087 // data cells are consistent.
00088 // It also contains various data members and functions to make them
00089 // persistent by writing them into an AipsIO stream.
00090 // </synopsis> 
00091 
00092 // <motivation>
00093 // This base class contains the common functionality of all
00094 // tiled storage managers. The base class is still abstract.
00095 // Only concrete tiled storage managers derived from it can
00096 // be instantiated.
00097 // <p>
00098 // Tiled storage managers make access to array data possible with
00099 // more or less the same efficiency for access along different axes.
00100 // </motivation>
00101 
00102 //# <todo asof="$DATE:$">
00103 //# A List of bugs, limitations, extensions or planned refinements.
00104 //# </todo>
00105 
00106 
00107 class TiledStMan : public DataManager
00108 {
00109 public:
00110     // Create a TiledStMan.
00111     TiledStMan();
00112 
00113     // Create a TiledStMan storage manager.
00114     // The given maximum cache size is persistent,
00115     // thus will be reused when the table is read back. Note that the class
00116     // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
00117     // allows one to overwrite the maximum cache size temporarily.
00118     // Its description contains a discussion about the effects of
00119     // setting a maximum cache.
00120     TiledStMan (const String& hypercolumnName, uInt maximumCacheSize);
00121 
00122     virtual ~TiledStMan();
00123 
00124     // Get the name given to the storage manager.
00125     // This is the name of the hypercolumn.
00126     virtual String dataManagerName() const;
00127 
00128     void setDataManagerName (const String& newHypercolumnName);
00129 
00130     // Return a record containing data manager specifications.
00131     virtual Record dataManagerSpec() const;
00132 
00133     // Get data manager properties that can be modified.
00134     // It is only ActualCacheSize (the actual cache size in buckets).
00135     // It is a subset of the data manager specification.
00136     virtual Record getProperties() const;
00137 
00138     // Modify data manager properties.
00139     // Only ActualCacheSize can be used. It is similar to function setCacheSize
00140     // with <src>canExceedNrBuckets=False</src>.
00141     virtual void setProperties (const Record& spec);
00142 
00143     // Set the flag to "data has changed since last flush".
00144     void setDataChanged();
00145 
00146     // Derive the tile shape from the hypercube shape for the given
00147     // number of pixels per tile. It is tried to get the same number
00148     // of tiles for each dimension.
00149     // When a weight vector is given, the number of tiles for a dimension
00150     // is proportional to the weight.
00151     // <br>After the initial guess it tries to optimize it by trying
00152     // to waste as little space as possible, while trying to keep as close
00153     // to the initial guess. The given tolerance (possibly per axis)
00154     // gives the minimum and maximum possible length of a tile axis
00155     // (minimum = initial_guess*tolerance; maximum = initial_guess/tolerance).
00156     // The heuristic is such that a tile axis length dividing the cube length
00157     // exactly is always favoured.
00158     // The test program <src>tTiledStMan</src> can be used to see how
00159     // the algorithm works out for a given tile size and cube shape.
00160     // <group>
00161     static IPosition makeTileShape (const IPosition& hypercubeShape,
00162                                     Double tolerance = 0.5,
00163                                     uInt maxNrPixelsPerTile = 32768);
00164     static IPosition makeTileShape (const IPosition& hypercubeShape,
00165                                     const Vector<double>& weight,
00166                                     const Vector<double>& tolerance,
00167                                     uInt maxNrPixelsPerTile = 32768);
00168     // </group>
00169 
00170     // Set the maximum cache size (in bytes) in a non-persistent way.
00171     virtual void setMaximumCacheSize (uInt nbytes);
00172 
00173     // Get the current maximum cache size (in bytes).
00174     uInt maximumCacheSize() const;
00175 
00176     // Get the current cache size (in buckets) for the hypercube in
00177     // the given row.
00178     uInt cacheSize (uInt rownr) const;
00179 
00180     // Get the hypercube shape of the data in the given row.
00181     const IPosition& hypercubeShape (uInt rownr) const;
00182 
00183     // Get the tile shape of the data in the given row.
00184     const IPosition& tileShape (uInt rownr) const;
00185 
00186     // Get the bucket size (in bytes) of the hypercube in the given row.
00187     uInt bucketSize (uInt rownr) const;
00188 
00189     // Can the tiled storage manager handle changing array shapes?
00190     // The default is no (but TiledCellStMan can).
00191     virtual Bool canChangeShape() const;
00192 
00193     // Can the tiled storage manager access an entire column.
00194     // TiledColumnStMan can always do that.
00195     // The others might be able to do it (for this time).
00196     // The default implementation returns True if there is only 1 hypercube.
00197     // reask is set to True (because next time things might be different).
00198     virtual Bool canAccessColumn (Bool& reask) const;
00199 
00200     // The data manager supports use of MultiFile.
00201     virtual Bool hasMultiFileSupport() const;
00202 
00203     // Calculate the cache size (in buckets) for accessing the hypercube
00204     // containing the given row. It takes the maximum cache size into
00205     // account (allowing an overdraft of 10%).
00206     // It uses the given axisPath (i.e. traversal order) to determine
00207     // the optimum size. A window can be specified to indicate that only
00208     // the given subset of the hypercube will be accessed.
00209     // <br>
00210     // The length of the slice and window arguments and <src>axisPath</src>
00211     // must be less or equal to the dimensionality of the hypercube.
00212     // The non-specified <src>windowStart</src> parts default to 0.
00213     // The non-specified <src>windowLength</src> parts default to
00214     // the hypercube shape.
00215     // The non-specified <src>sliceShape</src> parts default to 1.
00216     // <br>
00217     // Axispath = [2,0,1] indicates that the z-axis changes most rapidly,
00218     // thereafter x and y. An axis can occur only once in the axisPath.
00219     // The non-specified <src>axisPath</src> parts get the natural order.
00220     // E.g. in the previous example axisPath=[2] defines the same path.
00221     // <br>When forceSmaller is False, the cache is not resized when the
00222     // new size is smaller.
00223     // <br>A flag is set indicating that the TSMDataColumn
00224     // access functions do not need to size the cache.
00225     uInt calcCacheSize (uInt rownr, const IPosition& sliceShape,
00226                         const IPosition& windowStart,
00227                         const IPosition& windowLength,
00228                         const IPosition& axisPath) const;
00229 
00230     // Set the cache size using the <src>calcCacheSize</src>
00231     // function mentioned above.
00232     void setCacheSize (uInt rownr, const IPosition& sliceShape,
00233                        const IPosition& windowStart,
00234                        const IPosition& windowLength,
00235                        const IPosition& axisPath,
00236                        Bool forceSmaller);
00237 
00238     // Set the cache size for accessing the hypercube containing the given row.
00239     // When the give cache size exceeds the maximum cache size with more
00240     // than 10%, the maximum cache size is used instead.
00241     // <br>When forceSmaller is False, the cache is not resized when the
00242     // new size is smaller.
00243     // <br>A flag is set indicating that the TSMDataColumn
00244     // access functions do not need to size the cache.
00245     void setCacheSize (uInt rownr, uInt nbuckets, Bool forceSmaller);
00246 
00247     // Sets the cache size using the hypercube instead of the row number.
00248     // Useful for iterating over all hypercubes.
00249     void setHypercubeCacheSize (uInt hypercube, uInt nbuckets, Bool forceSmaller);
00250 
00251     // Determine if the user set the cache size (using setCacheSize).
00252     Bool userSetCache (uInt rownr) const;
00253 
00254     // Empty the caches used by the hypercubes in this storage manager.
00255     // It will flush the caches as needed and remove all buckets from them
00256     // resulting in a possibly large drop in memory used.
00257     // It also clears the userSetCache flag.
00258     void emptyCaches();
00259 
00260     // Show the statistics of all caches used.
00261     void showCacheStatistics (ostream& os) const;
00262 
00263     // Get the length of the data for the given number of pixels.
00264     // This can be used to calculate the length of a tile.
00265     uInt getLengthOffset (uInt nrPixels, Block<uInt>& dataOffset,
00266                           Block<uInt>& localOffset,
00267                           uInt& localTileLength) const;
00268 
00269     // Get the number of coordinate vectors.
00270     uInt nrCoordVector() const;
00271 
00272     // Get the nr of rows in this storage manager.
00273     uInt nrow() const;
00274 
00275     // Does the storage manager allow to add rows? (yes)
00276     Bool canAddRow() const;
00277 
00278     // Get the default tile shape.
00279     // By default it returns a zero-length IPosition.
00280     virtual IPosition defaultTileShape() const;
00281 
00282     // Return the number of hypercubes.
00283     uInt nhypercubes() const;
00284 
00285     // Test if only one hypercube is used by this storage manager.
00286     // If not, throw an exception. Otherwise return the hypercube.
00287     virtual TSMCube* singleHypercube();
00288 
00289     // Get the given hypercube.
00290     // <group>
00291     const TSMCube* getTSMCube (uInt hypercube) const;
00292     TSMCube* getTSMCube (uInt hypercube);
00293     // </group>
00294     
00295     // Get the hypercube in which the given row is stored.
00296     // <group>
00297     const TSMCube* getHypercube (uInt rownr) const;
00298     virtual TSMCube* getHypercube (uInt rownr) = 0;
00299     // </group>
00300 
00301     // Get the hypercube in which the given row is stored.
00302     // It also returns the position of the row in that hypercube.
00303     virtual TSMCube* getHypercube (uInt rownr, IPosition& position) = 0;
00304 
00305     // Make the correct TSMCube type (depending on tsmOption()).
00306     TSMCube* makeTSMCube (TSMFile* file, const IPosition& cubeShape,
00307                           const IPosition& tileShape,
00308                           const Record& values, Int64 fileOffset=-1);
00309 
00310     // Read a tile and convert the data to local format.
00311     void readTile (char* local, const Block<uInt>& localOffset,
00312                    const char* external, const Block<uInt>& externalOffset,
00313                    uInt nrpixels);
00314 
00315     // Write a tile after converting the data to external format.
00316     void writeTile (char* external, const Block<uInt>& externalOffset,
00317                     const char* local, const Block<uInt>& localOffset,
00318                     uInt nrpixels);
00319 
00320     // Get the TSMFile object with the given sequence number.
00321     TSMFile* getFile (uInt sequenceNumber);
00322 
00323     // Open the storage manager for an existing table.
00324     virtual void open (uInt nrrow, AipsIO&);
00325 
00326     // Resync the storage manager with the new file contents.
00327     virtual void resync (uInt nrrow);
00328 
00329     // Reopen all files used in this storage manager for read/write access.
00330     virtual void reopenRW();
00331 
00332     // The data manager will be deleted (because all its columns are
00333     // requested to be deleted).
00334     // So clean up the things needed (e.g. delete files).
00335     virtual void deleteManager();
00336 
00337     // Create a column in the storage manager on behalf of a table column.
00338     // <group>
00339     // Create a scalar column.
00340     DataManagerColumn* makeScalarColumn (const String& name, int dataType,
00341                                          const String& dataTypeID);
00342     // Create a direct array column.
00343     DataManagerColumn* makeDirArrColumn (const String& name, int dataType,
00344                                          const String& dataTypeID);
00345     // Create an indirect array column.
00346     DataManagerColumn* makeIndArrColumn (const String& name, int dataType,
00347                                          const String& dataTypeID);
00348     // </group>
00349 
00350     // The TiledStMan wants to do reallocateColumn.
00351     Bool canReallocateColumns() const;
00352 
00353     // Reallocate the column object if it is part of this data manager.
00354     // It returns a pointer to the new column object.
00355     // It is used to remove the indirection of the TSMColumn objects
00356     // resulting in only one iso. two virtual column calls to get the data.
00357     DataManagerColumn* reallocateColumn (DataManagerColumn* column);
00358 
00359     // Set the shape and tile shape of a hypercube.
00360     // By default it throws an "impossible" exception.
00361     virtual void setShape (uInt rownr, TSMCube* hypercube,
00362                            const IPosition& shape,
00363                            const IPosition& tileShape);
00364 
00365     // Check the shape to be set for a hypercube.
00366     // It checks if it matches predefined (fixed shape) columns
00367     // and the shape of already defined coordinate columns.
00368     void checkCubeShape (const TSMCube* hypercube,
00369                          const IPosition& cubeShape) const;
00370 
00371     // Get the data type of the coordinate column with the given name.
00372     // An exception is thrown when the column is unknown.
00373     int coordinateDataType (const String& columnName) const;
00374 
00375     // Initialize the new coordinates for the given cube.
00376     void initCoordinates (TSMCube* hypercube);
00377 
00378     // Get pointer to data column object.
00379     const TSMDataColumn* getDataColumn (uInt colnr) const
00380       { return dataCols_p[colnr]; }
00381 
00382 protected:
00383     // Set the persistent maximum cache size.
00384     void setPersMaxCacheSize (uInt nbytes);
00385 
00386     // Get the bindings of the columns with the given names.
00387     // If bound, the pointer to the TSMColumn object is stored in the block.
00388     // If mustExist is True, an exception is thrown if the column
00389     // is not bound.
00390     // It returns the number of bound columns.
00391     uInt getBindings (const Vector<String>& columnNames,
00392                       PtrBlock<TSMColumn*>& colSet,
00393                       Bool mustExist) const;
00394 
00395     // Function setup calls this function to allow the derived class
00396     // to check specific information. In case of errors, an exception
00397     // should be thrown.
00398     // By default it does nothing.
00399     virtual void setupCheck (const TableDesc& tableDesc,
00400                              const Vector<String>& dataNames) const;
00401 
00402     // Get the table description needed for the hypercolumn description.
00403     virtual const TableDesc& getDesc() const;
00404 
00405     // Check if values are given in the record for all columns in
00406     // the block. Also check if the data types are correct.
00407     // An exception is thrown if something is incorrect.
00408     void checkValues (const PtrBlock<TSMColumn*>& colSet,
00409                       const Record& values) const;
00410 
00411     // Check if the coordinate values are correct.
00412     // This calls checkValues and checks if their shapes match the
00413     // hypercube shape.
00414     // An exception is thrown if invalid.
00415     void checkCoordinates (const PtrBlock<TSMColumn*>& coordColSet,
00416                            const IPosition& cubeShape,
00417                            const Record& values) const;
00418 
00419     // Check if the shapes of FixedShape data and coordinate columns match.
00420     // An exception is thrown if not.
00421     void checkShapeColumn (const IPosition& shape) const;
00422 
00423     // Check if the cube shape matches that of defined coordinates.
00424     void checkCoordinatesShapes (const TSMCube* hypercube,
00425                                  const IPosition& cubeShape) const;
00426 
00427     // Check if the hypercube to be added is correctly defined.
00428     void checkAddHypercube (const IPosition& cubeShape,
00429                             const Record& values) const;
00430 
00431     // Make a new TSMCube object.
00432     TSMCube* makeHypercube (const IPosition& cubeShape,
00433                             const IPosition& tileShape,
00434                             const Record& values);
00435 
00436     // Get the index of the hypercube with the given id-values.
00437     // If not found, -1 is returned.
00438     Int getCubeIndex (const Record& idValues) const;
00439     
00440     // Determine how many rows need to be added for an extension
00441     // (in the last dimension) of a hypercube with the given shape.
00442     uInt addedNrrow (const IPosition& shape, uInt incrInLastDim) const;
00443 
00444     // Flush the caches of all hypercubes.
00445     // If data have put and fsync is set, fsync all files.
00446     Bool flushCaches (Bool fsync);
00447 
00448     // Let a derived class read the header info.
00449     // This is used by the open and resync function.
00450     virtual void readHeader (uInt nrrow, Bool firstTime) = 0;
00451 
00452     // Create the TSM header file.
00453     // It creates an AipsIO object for it.
00454     AipsIO* headerFileCreate();
00455 
00456     // Open the TSM header file.
00457     // It creates an AipsIO object for it.
00458     AipsIO* headerFileOpen();
00459 
00460     // Write the data into the header file.
00461     // The given number of TSMCube objects have to be written.
00462     void headerFilePut (AipsIO& headerFile, uInt nrCube);
00463 
00464     // Read the data from the header file.
00465     // When done for the first time, setup() is called to initialize
00466     // the various variables (using the extraNdim variable).
00467     void headerFileGet (AipsIO& headerFile, uInt tabNrrow, Bool firstTime,
00468                         Int extraNdim);
00469 
00470     // Close the header file.
00471     // It deletes the AipsIO object.
00472     void headerFileClose (AipsIO* headerFile);
00473 
00474     // Set up the TiledStMan variables from the table description.
00475     // The argument specifies the number of extra dimensions for the
00476     // hypercube compared to the data array (usually 0 or 1).
00477     // It is only used if no hypercolumn definition exists.
00478     // -1 means that the hypercolumn definition has to be present.
00479     void setup (Int extraNdim=-1);
00480 
00481     // Create a TSMFile object and store its pointer at the given index
00482     // in the block.
00483     void createFile (uInt index);
00484 
00485     // Convert the scalar data type to an array data type.
00486     // This function is temporary and can disappear when the ColumnDesc
00487     // classes use type TpArray*.
00488     int arrayDataType (int dataType) const;
00489 
00490 
00491     //# Declare all data members.
00492     // The name of the hypercolumn.
00493     String hypercolumnName_p;
00494     // The number of rows in the columns.
00495     uInt  nrrow_p;
00496     // The assembly of all columns.
00497     PtrBlock<TSMColumn*>  colSet_p;
00498     // The assembly of all data columns.
00499     PtrBlock<TSMDataColumn*> dataCols_p;
00500     PtrBlock<TSMColumn*>  dataColSet_p;
00501     // The assembly of all id columns.
00502     PtrBlock<TSMColumn*>  idColSet_p;
00503     // The assembly of all coordinate columns.
00504     PtrBlock<TSMColumn*>  coordColSet_p;
00505     // The assembly of all TSMFile objects.
00506     // The first file is for all non-extensible cubes, while the others
00507     // are for one file per extensible cube.
00508     PtrBlock<TSMFile*> fileSet_p;
00509     // The assembly of all TSMCube objects.
00510     PtrBlock<TSMCube*> cubeSet_p;
00511     // The persistent maximum cache size for a hypercube.
00512     uInt      persMaxCacheSize_p;
00513     // The actual maximum cache size for a hypercube.
00514     uInt      maxCacheSize_p;
00515     // The dimensionality of the hypercolumn.
00516     uInt      nrdim_p;
00517     // The number of vector coordinates.
00518     uInt      nrCoordVector_p;
00519     // The fixed cell shape.
00520     IPosition fixedCellShape_p;
00521     // Has any data changed since the last flush?
00522     Bool      dataChanged_p;
00523 
00524 private:
00525     // Forbid copy constructor.
00526     TiledStMan (const TiledStMan&);
00527 
00528     // Forbid assignment.
00529     TiledStMan& operator= (const TiledStMan&);
00530 };
00531 
00532 
00533 inline uInt TiledStMan::maximumCacheSize() const
00534     { return maxCacheSize_p; }
00535 
00536 inline uInt TiledStMan::nrCoordVector() const
00537     { return nrCoordVector_p; }
00538 
00539 inline uInt TiledStMan::nrow() const
00540     { return nrrow_p; }
00541 
00542 inline uInt TiledStMan::nhypercubes() const
00543     { return cubeSet_p.nelements(); }
00544 
00545 inline void TiledStMan::setDataChanged()
00546     { dataChanged_p = True; }
00547 
00548 inline const TSMCube* TiledStMan::getTSMCube (uInt hypercube) const
00549     { return const_cast<TiledStMan*>(this)->getTSMCube (hypercube); }
00550 
00551 inline const TSMCube* TiledStMan::getHypercube (uInt rownr) const
00552     { return const_cast<TiledStMan*>(this)->getHypercube (rownr); }
00553 
00554 inline void TiledStMan::setPersMaxCacheSize (uInt nbytes)
00555 {
00556     persMaxCacheSize_p = nbytes;
00557     maxCacheSize_p = nbytes;
00558 }
00559 
00560 
00561 
00562 
00563 } //# NAMESPACE CASACORE - END
00564 
00565 #endif
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines

Generated on 31 Aug 2016 for casa by  doxygen 1.6.1