00001 //# MultiHDF5.h: Class to combine multiple files in a single HDF5 file 00002 //# Copyright (C) 2015 00003 //# Associated Universities, Inc. Washington DC, USA. 00004 //# 00005 //# This library is free software; you can redistribute it and/or modify it 00006 //# under the terms of the GNU Library General Public License as published by 00007 //# the Free Software Foundation; either version 2 of the License, or (at your 00008 //# option) any later version. 00009 //# 00010 //# This library is distributed in the hope that it will be useful, but WITHOUT 00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 00012 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public 00013 //# License for more details. 00014 //# 00015 //# You should have received a copy of the GNU Library General Public License 00016 //# along with this library; if not, write to the Free Software Foundation, 00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. 00018 //# 00019 //# Correspondence concerning AIPS++ should be addressed as follows: 00020 //# Internet email: aips2-request@nrao.edu. 00021 //# Postal address: AIPS++ Project Office 00022 //# National Radio Astronomy Observatory 00023 //# 520 Edgemont Road 00024 //# Charlottesville, VA 22903-2475 USA 00025 //# 00026 //# $Id: RegularFileIO.h 20551 2009-03-25 00:11:33Z Malte.Marquarding $ 00027 00028 #ifndef CASA_MULTIHDF5_H 00029 #define CASA_MULTIHDF5_H 00030 00031 //# Includes 00032 #include <casacore/casa/aips.h> 00033 #include <casacore/casa/IO/MultiFile.h> 00034 #include <casacore/casa/HDF5/HDF5File.h> 00035 00036 00037 namespace casacore { //# NAMESPACE CASACORE - BEGIN 00038 00039 // <summary> 00040 // Class to combine multiple files in a single HDF5 file. 00041 // </summary> 00042 00043 // <use visibility=export> 00044 00045 // <reviewed reviewer="" date="" tests="tMultiHDF5" demos=""> 00046 // </reviewed> 00047 00048 // <synopsis> 00049 // This class is a container file holding multiple virtual files. It is 00050 // primarily meant as a container file for the storage manager files of a 00051 // table to reduce the number of files used (especially for Lustre) and to 00052 // reduce the number of open files (especially when concatenating tables). 00053 // <br>A secondary goal is offering the ability to use an IO buffer size 00054 // that matches the file system well (large buffer size for e.g. ZFS). 00055 // 00056 // The SetupNewTable constructor has a StorageOption argument to define 00057 // if a MultiFile has to be used and if so, the buffer size to use. 00058 // It is also possible to specify that through aipsrc variables. 00059 // 00060 // A virtual file is spread over multiple (fixed size) data blocks in the 00061 // MultiFile. A data block is never shared by multiple files. 00062 // For each virtual file MultiFile keeps a MultiFileInfo object telling 00063 // the file size and the blocks numbers used for the file. When flushing 00064 // the MultiFile, this meta info is written into a header block and, 00065 // if needed, continuation blocks. On open and resync, it is read back. 00066 // <br> 00067 // 00068 // A virtual file is represented by an MFFileIO object, which is derived 00069 // from ByteIO and as such part of the casacore IO framework. It makes it 00070 // possible for applications to access a virtual file in the same way as 00071 // a regular file. 00072 // 00073 // It is possible to delete a virtual file. Its blocks will be added to 00074 // the free block list (which is also stored in the meta info). 00075 // </synopsis> 00076 00077 // <example> 00078 // In principle it is possible to use the MultiFile functions directly. 00079 // However, in general it is much easier to use an MFFileIO object 00080 // per virtual file as shown below. 00081 // <srcblock> 00082 // // Create a new MultiFile using a block size of 1 MB. 00083 // MultiFile mfile("file.mf', ByteIO::New, 1048576); 00084 // // Create a virtual file in it. 00085 // MFFileIO mf1(mfile, "mf1", ByteIO::New); 00086 // // Use it (for example) as the sink of AipsIO. 00087 // AipsIO stream (&mf1); 00088 // // Write values. 00089 // stream << (Int)10; 00090 // stream << True; 00091 // // Seek to beginning of file and read data in. 00092 // stream.setpos (0); 00093 // Int vali; 00094 // Bool valb; 00095 // stream >> vali >> valb; 00096 // </srcblock> 00097 // </example> 00098 00099 // <todo> 00100 // <li> write headers at alternating file positions (for robustness) 00101 // <li> possibly write headers entirely at the end if larger than blocksize 00102 // </todo> 00103 00104 00105 class MultiHDF5 : public MultiFileBase 00106 { 00107 public: 00108 // Open or create a MultiHDF5 with the given name. 00109 // Upon creation the block size can be given. If 0, it uses the block size 00110 // of the file system the file is on. 00111 MultiHDF5 (const String& name, ByteIO::OpenOption, Int blockSize=0); 00112 00113 // The destructor flushes and closes the file. 00114 virtual ~MultiHDF5(); 00115 00116 // Reopen the underlying file for read/write access. 00117 // Nothing will be done if the file is writable already. 00118 // Otherwise it will be reopened and an exception will be thrown 00119 // if it is not possible to reopen it for read/write access. 00120 virtual void reopenRW(); 00121 00122 // Fsync the file (i.e., force the data to be physically written). 00123 virtual void fsync(); 00124 00125 private: 00126 // Do the class-specific actions on adding a file. 00127 virtual void doAddFile (MultiFileInfo&); 00128 // Do the class-specific actions on deleting a file. 00129 virtual void doDeleteFile (MultiFileInfo&); 00130 // Flush the file itself. 00131 virtual void flushFile(); 00132 // Flush and close the file. 00133 virtual void close(); 00134 // Write the header info. 00135 virtual void writeHeader(); 00136 // Read the header info. If always==False, the info is only read if the 00137 // header counter has changed. 00138 virtual void readHeader (Bool always=True); 00139 // Extend the virtual file to fit lastblk. 00140 virtual void extend (MultiFileInfo& info, Int64 lastblk); 00141 // Read a data block. 00142 virtual void readBlock (MultiFileInfo& info, Int64 blknr, 00143 void* buffer); 00144 // Write a data block. 00145 virtual void writeBlock (MultiFileInfo& info, Int64 blknr, 00146 const void* buffer); 00147 00148 //# Data members 00149 HDF5File itsFile; 00150 }; 00151 00152 00153 } //# NAMESPACE CASACORE - END 00154 00155 #endif