MultiHDF5.h

Go to the documentation of this file.
00001 //# MultiHDF5.h: Class to combine multiple files in a single HDF5 file
00002 //# Copyright (C) 2015
00003 //# Associated Universities, Inc. Washington DC, USA.
00004 //#
00005 //# This library is free software; you can redistribute it and/or modify it
00006 //# under the terms of the GNU Library General Public License as published by
00007 //# the Free Software Foundation; either version 2 of the License, or (at your
00008 //# option) any later version.
00009 //#
00010 //# This library is distributed in the hope that it will be useful, but WITHOUT
00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
00012 //# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
00013 //# License for more details.
00014 //#
00015 //# You should have received a copy of the GNU Library General Public License
00016 //# along with this library; if not, write to the Free Software Foundation,
00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
00018 //#
00019 //# Correspondence concerning AIPS++ should be addressed as follows:
00020 //#        Internet email: aips2-request@nrao.edu.
00021 //#        Postal address: AIPS++ Project Office
00022 //#                        National Radio Astronomy Observatory
00023 //#                        520 Edgemont Road
00024 //#                        Charlottesville, VA 22903-2475 USA
00025 //#
00026 //# $Id: RegularFileIO.h 20551 2009-03-25 00:11:33Z Malte.Marquarding $
00027 
00028 #ifndef CASA_MULTIHDF5_H
00029 #define CASA_MULTIHDF5_H
00030 
00031 //# Includes
00032 #include <casacore/casa/aips.h>
00033 #include <casacore/casa/IO/MultiFile.h>
00034 #include <casacore/casa/HDF5/HDF5File.h>
00035 
00036 
00037 namespace casacore { //# NAMESPACE CASACORE - BEGIN
00038 
00039   // <summary> 
00040   // Class to combine multiple files in a single HDF5 file.
00041   // </summary>
00042 
00043   // <use visibility=export>
00044 
00045   // <reviewed reviewer="" date="" tests="tMultiHDF5" demos="">
00046   // </reviewed>
00047 
00048   // <synopsis> 
00049   // This class is a container file holding multiple virtual files. It is
00050   // primarily meant as a container file for the storage manager files of a
00051   // table to reduce the number of files used (especially for Lustre) and to
00052   // reduce the number of open files (especially when concatenating tables).
00053   // <br>A secondary goal is offering the ability to use an IO buffer size
00054   // that matches the file system well (large buffer size for e.g. ZFS).
00055   //
00056   // The SetupNewTable constructor has a StorageOption argument to define
00057   // if a MultiFile has to be used and if so, the buffer size to use.
00058   // It is also possible to specify that through aipsrc variables.
00059   //
00060   // A virtual file is spread over multiple (fixed size) data blocks in the
00061   // MultiFile. A data block is never shared by multiple files.
00062   // For each virtual file MultiFile keeps a MultiFileInfo object telling
00063   // the file size and the blocks numbers used for the file. When flushing
00064   // the MultiFile, this meta info is written into a header block and,
00065   // if needed, continuation blocks. On open and resync, it is read back.
00066   // <br>
00067   //
00068   // A virtual file is represented by an MFFileIO object, which is derived
00069   // from ByteIO and as such part of the casacore IO framework. It makes it
00070   // possible for applications to access a virtual file in the same way as
00071   // a regular file.
00072   //
00073   // It is possible to delete a virtual file. Its blocks will be added to
00074   // the free block list (which is also stored in the meta info).
00075   // </synopsis>
00076 
00077   // <example>
00078   // In principle it is possible to use the MultiFile functions directly.
00079   // However, in general it is much easier to use an MFFileIO object
00080   // per virtual file as shown below.
00081   // <srcblock>
00082   //    // Create a new MultiFile using a block size of 1 MB.
00083   //    MultiFile mfile("file.mf', ByteIO::New, 1048576);
00084   //    // Create a virtual file in it.
00085   //    MFFileIO mf1(mfile, "mf1", ByteIO::New);
00086   //    // Use it (for example) as the sink of AipsIO.
00087   //    AipsIO stream (&mf1);
00088   //    // Write values.
00089   //    stream << (Int)10;
00090   //    stream << True;
00091   //    // Seek to beginning of file and read data in.
00092   //    stream.setpos (0);
00093   //    Int vali;
00094   //    Bool valb;
00095   //    stream >> vali >> valb;
00096   // </srcblock>
00097   // </example>
00098 
00099   // <todo>
00100   //  <li> write headers at alternating file positions (for robustness)
00101   //  <li> possibly write headers entirely at the end if larger than blocksize
00102   // </todo>
00103 
00104 
00105   class MultiHDF5 : public MultiFileBase
00106   {
00107   public:
00108     // Open or create a MultiHDF5 with the given name.
00109     // Upon creation the block size can be given. If 0, it uses the block size
00110     // of the file system the file is on.
00111     MultiHDF5 (const String& name, ByteIO::OpenOption, Int blockSize=0);
00112 
00113     // The destructor flushes and closes the file.
00114     virtual ~MultiHDF5();
00115 
00116     // Reopen the underlying file for read/write access.
00117     // Nothing will be done if the file is writable already.
00118     // Otherwise it will be reopened and an exception will be thrown
00119     // if it is not possible to reopen it for read/write access.
00120     virtual void reopenRW();
00121 
00122     // Fsync the file (i.e., force the data to be physically written).
00123     virtual void fsync();
00124 
00125   private:
00126     // Do the class-specific actions on adding a file.
00127     virtual void doAddFile (MultiFileInfo&);
00128     // Do the class-specific actions on deleting a file.
00129     virtual void doDeleteFile (MultiFileInfo&);
00130     // Flush the file itself.
00131     virtual void flushFile();
00132     // Flush and close the file.
00133     virtual void close();
00134     // Write the header info.
00135     virtual void writeHeader();
00136     // Read the header info. If always==False, the info is only read if the
00137     // header counter has changed.
00138     virtual void readHeader (Bool always=True);
00139     // Extend the virtual file to fit lastblk.
00140     virtual void extend (MultiFileInfo& info, Int64 lastblk);
00141     // Read a data block.
00142     virtual void readBlock (MultiFileInfo& info, Int64 blknr,
00143                             void* buffer);
00144     // Write a data block.
00145     virtual void writeBlock (MultiFileInfo& info, Int64 blknr,
00146                              const void* buffer);
00147 
00148     //# Data members
00149     HDF5File itsFile;
00150   };
00151 
00152 
00153 } //# NAMESPACE CASACORE - END
00154 
00155 #endif
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines

Generated on 31 Aug 2016 for casa by  doxygen 1.6.1