00001 //# CompressComplex.h: Virtual column engine to scale a table Complex array 00002 //# Copyright (C) 2001,2002,2003 00003 //# Associated Universities, Inc. Washington DC, USA. 00004 //# 00005 //# This library is free software; you can redistribute it and/or modify it 00006 //# under the terms of the GNU Library General Public License as published by 00007 //# the Free Software Foundation; either version 2 of the License, or (at your 00008 //# option) any later version. 00009 //# 00010 //# This library is distributed in the hope that it will be useful, but WITHOUT 00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 00012 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public 00013 //# License for more details. 00014 //# 00015 //# You should have received a copy of the GNU Library General Public License 00016 //# along with this library; if not, write to the Free Software Foundation, 00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. 00018 //# 00019 //# Correspondence concerning AIPS++ should be addressed as follows: 00020 //# Internet email: aips2-request@nrao.edu. 00021 //# Postal address: AIPS++ Project Office 00022 //# National Radio Astronomy Observatory 00023 //# 520 Edgemont Road 00024 //# Charlottesville, VA 22903-2475 USA 00025 //# 00026 //# $Id$ 00027 00028 #ifndef TABLES_COMPRESSCOMPLEX_H 00029 #define TABLES_COMPRESSCOMPLEX_H 00030 00031 //# Includes 00032 #include <casacore/casa/aips.h> 00033 #include <casacore/tables/DataMan/BaseMappedArrayEngine.h> 00034 #include <casacore/tables/Tables/ScalarColumn.h> 00035 #include <casacore/casa/Arrays/Array.h> 00036 #include <casacore/casa/BasicSL/Complex.h> 00037 00038 00039 namespace casacore { //# NAMESPACE CASACORE - BEGIN 00040 00041 // <summary> 00042 // Virtual column engine to scale a table Complex array 00043 // </summary> 00044 00045 // <use visibility=export> 00046 00047 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tCompressComplex.cc"> 00048 // </reviewed> 00049 00050 // <prerequisite> 00051 //# Classes you should understand before using this one. 00052 // <li> VirtualColumnEngine 00053 // <li> VirtualArrayColumn 00054 // </prerequisite> 00055 00056 // <synopsis> 00057 // CompressComplex is a virtual column engine which scales an array 00058 // of one type to another type to save disk storage. 00059 // This resembles the classic AIPS compress method which scales the 00060 // data from Complex to int. 00061 // The scale factor and offset values can be given in two ways: 00062 // <ul> 00063 // <li> As a fixed values which is used for all arrays in the column. 00064 // These values have to be given when constructing of the engine. 00065 // <li> As the name of a column. In this way each array in the 00066 // column has its own scale and offset value. 00067 // By default it uses auto-scaling (see below). 00068 // Otherwise the scale and offset value in a row must be put 00069 // before the array is put and should not be changed anymore. 00070 // </ul> 00071 // Auto-scaling means that the engine will determine the scale 00072 // and offset value itself when an array (or a slice) is put. 00073 // It does it by mapping the values in the array to the range [-32767,32767]. 00074 // At each put the scale/offset values are changed as needed. 00075 // Note that with auto-scaling <src>putSlice</src> can be somewhat 00076 // slower, because the entire array might need to be rescaled. 00077 // 00078 // As in FITS the scale and offset values are used as: 00079 // <br><src> True_value = Stored_value * scale + offset; </src> 00080 // 00081 // An engine object should be used for one column only, because the stored 00082 // column name is part of the engine. If it would be used for more than 00083 // one column, they would all share the same stored column. 00084 // When the engine is bound to a column, it is checked if the name 00085 // of that column matches the given virtual column name. 00086 // 00087 // The engine can be used for a column containing any kind of array 00088 // (thus direct or indirect, fixed or variable shaped)) as long as the 00089 // virtual array can be stored in the stored array. Thus a fixed shaped 00090 // virtual can use a variable shaped stored, but not vice versa. 00091 // A fixed shape indirect virtual can use a stored with direct arrays. 00092 // 00093 // This class can also serve as an example of how to implement 00094 // a virtual column engine. 00095 // </synopsis> 00096 00097 // <motivation> 00098 // This class allows to store data in a smaller representation. 00099 // It is needed to resemble the classic AIPS compress option. 00100 // 00101 // Because the engine can serve only one column, it was possible to 00102 // combine the engine and the column functionality in one class. 00103 // </motivation> 00104 00105 // <example> 00106 // <srcblock> 00107 // // Create the table description and 2 columns with indirect arrays in it. 00108 // // The Int column will be stored, while the double will be 00109 // // used as virtual. 00110 // TableDesc tableDesc ("", TableDesc::Scratch); 00111 // tableDesc.addColumn (ArrayColumnDesc<Int> ("storedArray")); 00112 // tableDesc.addColumn (ArrayColumnDesc<Complex> ("virtualArray")); 00113 // tableDesc.addColumn (ScalarColumnDesc<Complex> ("scale")); 00114 // tableDesc.addColumn (ScalarColumnDesc<Float> ("offset")); 00115 // 00116 // // Create a new table using the table description. 00117 // SetupNewTable newtab (tableDesc, "tab.data", Table::New); 00118 // 00119 // // Create the array scaling engine (with auto-scale) 00120 // // and bind it to the Complex column. 00121 // CompressComplex scalingEngine("virtualArray", "storedArray", 00122 // "scale", "offset"); 00123 // newtab.bindColumn ("virtualArray", scalingEngine); 00124 // // Create the table. 00125 // Table table (newtab); 00126 // 00127 // // Store a 3-D array (with dim. 2,3,4) into each row of the column. 00128 // // The shape of each array in the column is implicitly set by the put 00129 // // function. This will also set the shape of the underlying Int array. 00130 // ArrayColumn data (table, "virtualArray"); 00131 // Array<double> someArray(IPosition(4,2,3,4)); 00132 // someArray = 0; 00133 // for (uInt i=0, i<10; i++) { // table will have 10 rows 00134 // table.addRow(); 00135 // data.put (i, someArray) 00136 // } 00137 // </srcblock> 00138 // </example> 00139 00140 class CompressComplex : public BaseMappedArrayEngine<Complex, Int> 00141 { 00142 public: 00143 00144 // Construct an engine to scale all arrays in a column with 00145 // the given offset and scale factor. 00146 // StoredColumnName is the name of the column where the scaled 00147 // data will be put and must have data type Int. 00148 // The virtual column using this engine must have data type Complex. 00149 CompressComplex (const String& virtualColumnName, 00150 const String& storedColumnName, 00151 Float scale, 00152 Float offset = 0); 00153 00154 // Construct an engine to scale the arrays in a column. 00155 // The scale and offset values are taken from a column with 00156 // the given names. In that way each array has its own scale factor 00157 // and offset value. 00158 // An exception is thrown if these columns do not exist. 00159 // VirtualColumnName is the name of the virtual column and is used to 00160 // check if the engine gets bound to the correct column. 00161 // StoredColumnName is the name of the column where the scaled 00162 // data will be put and must have data type Int. 00163 // The virtual column using this engine must have data type Complex. 00164 CompressComplex (const String& virtualColumnName, 00165 const String& storedColumnName, 00166 const String& scaleColumnName, 00167 const String& offsetColumnName, 00168 Bool autoScale = True); 00169 00170 // Construct from a record specification as created by getmanagerSpec(). 00171 CompressComplex (const Record& spec); 00172 00173 // Destructor is mandatory. 00174 ~CompressComplex(); 00175 00176 // Return the type name of the engine (i.e. its class name). 00177 virtual String dataManagerType() const; 00178 00179 // Get the name given to the engine (is the virtual column name). 00180 virtual String dataManagerName() const; 00181 00182 // Record a record containing data manager specifications. 00183 virtual Record dataManagerSpec() const; 00184 00185 // Return the name of the class. 00186 // This includes the names of the template arguments. 00187 static String className(); 00188 00189 // Register the class name and the static makeObject "constructor". 00190 // This will make the engine known to the table system. 00191 static void registerClass(); 00192 00193 protected: 00194 // Copy constructor is only used by clone() and derived class. 00195 // (so it is made private). 00196 CompressComplex (const CompressComplex&); 00197 00198 private: 00199 // Assignment is not needed and therefore forbidden 00200 // (so it is made private and not implemented). 00201 CompressComplex& operator= (const CompressComplex&); 00202 00203 // Clone the engine object. 00204 virtual DataManager* clone() const; 00205 00206 protected: 00207 // Initialize the object for a new table. 00208 // It defines the keywords containing the engine parameters. 00209 virtual void create (uInt initialNrrow); 00210 00211 private: 00212 // Preparing consists of setting the writable switch and 00213 // adding the initial number of rows in case of create. 00214 // Furthermore it reads the keywords containing the engine parameters. 00215 virtual void prepare(); 00216 00217 // Reopen the engine for read/write access. 00218 // It makes the column writable if the underlying column is writable. 00219 virtual void reopenRW(); 00220 00221 // Add rows to the table. 00222 // If auto-scaling, it initializes the scale column with 0 00223 // to indicate that no data has been processed yet. 00224 virtual void addRowInit (uInt startRow, uInt nrrow); 00225 00226 // Get an array in the given row. 00227 // This will scale and offset from the underlying array. 00228 virtual void getArray (uInt rownr, Array<Complex>& array); 00229 00230 // Put an array in the given row. 00231 // This will scale and offset to the underlying array. 00232 virtual void putArray (uInt rownr, const Array<Complex>& array); 00233 00234 // Get a section of the array in the given row. 00235 // This will scale and offset from the underlying array. 00236 virtual void getSlice (uInt rownr, const Slicer& slicer, 00237 Array<Complex>& array); 00238 00239 // Put into a section of the array in the given row. 00240 // This will scale and offset to the underlying array. 00241 virtual void putSlice (uInt rownr, const Slicer& slicer, 00242 const Array<Complex>& array); 00243 00244 // Get an entire column. 00245 // This will scale and offset from the underlying array. 00246 virtual void getArrayColumn (Array<Complex>& array); 00247 00248 // Put an entire column. 00249 // This will scale and offset to the underlying array. 00250 virtual void putArrayColumn (const Array<Complex>& array); 00251 00252 // Get some array values in the column. 00253 // This will scale and offset from the underlying array. 00254 virtual void getArrayColumnCells (const RefRows& rownrs, 00255 Array<Complex>& data); 00256 00257 // Put some array values in the column. 00258 // This will scale and offset to the underlying array. 00259 virtual void putArrayColumnCells (const RefRows& rownrs, 00260 const Array<Complex>& data); 00261 00262 // Get a section of all arrays in the column. 00263 // This will scale and offset from the underlying array. 00264 virtual void getColumnSlice (const Slicer& slicer, Array<Complex>& array); 00265 00266 // Put a section of all arrays in the column. 00267 // This will scale and offset to the underlying array. 00268 virtual void putColumnSlice (const Slicer& slicer, 00269 const Array<Complex>& array); 00270 00271 // Get a section of some arrays in the column. 00272 // This will scale and offset from the underlying array. 00273 virtual void getColumnSliceCells (const RefRows& rownrs, 00274 const Slicer& slicer, 00275 Array<Complex>& data); 00276 00277 // Put into a section of some arrays in the column. 00278 // This will scale and offset to the underlying array. 00279 virtual void putColumnSliceCells (const RefRows& rownrs, 00280 const Slicer& slicer, 00281 const Array<Complex>& data); 00282 00283 // Scale and/or offset target to array. 00284 // This is meant when reading an array from the stored column. 00285 // It optimizes for scale=1 and/or offset=0. 00286 virtual void scaleOnGet (Float scale, Float offset, 00287 Array<Complex>& array, 00288 const Array<Int>& target); 00289 00290 // Scale and/or offset array to target. 00291 // This is meant when writing an array into the stored column. 00292 // It optimizes for scale=1 and/or offset=0. 00293 virtual void scaleOnPut (Float scale, Float offset, 00294 const Array<Complex>& array, 00295 Array<Int>& target); 00296 00297 // Scale and/or offset target to array for the entire column. 00298 // When the scale and offset are fixed, it will do the entire array. 00299 // Otherwise it iterates through the array and applies the scale 00300 // and offset per row. 00301 void scaleColumnOnGet (Array<Complex>& array, 00302 const Array<Int>& target); 00303 00304 // Scale and/or offset array to target for the entire column. 00305 // When the scale and offset are fixed, it will do the entire array. 00306 // Otherwise it iterates through the array and applies the scale 00307 // and offset per row. 00308 void scaleColumnOnPut (const Array<Complex>& array, 00309 Array<Int>& target); 00310 00311 protected: 00312 //# Now define the data members. 00313 String scaleName_p; //# name of scale column 00314 String offsetName_p; //# name of offset column 00315 Float scale_p; //# fixed scale factor 00316 Float offset_p; //# fixed offset value 00317 Bool fixed_p; //# scale/offset is fixed 00318 Bool autoScale_p; //# determine scale/offset automatically 00319 ScalarColumn<Float>* scaleColumn_p; //# column with scale value 00320 ScalarColumn<Float>* offsetColumn_p; //# column with offset value 00321 Array<Int> buffer_p; //# buffer to avoid Array constructions 00322 //# (makes multi-threading harder) 00323 00324 // Get the scale value for this row. 00325 Float getScale (uInt rownr); 00326 00327 // Get the offset value for this row. 00328 Float getOffset (uInt rownr); 00329 00330 // Find minimum and maximum from the array data. 00331 // NaN and infinite values are ignored. If no values are finite, 00332 // minimum and maximum are set to NaN. 00333 virtual void findMinMax (Float& minVal, Float& maxVal, 00334 const Array<Complex>& array) const; 00335 00336 // Make scale and offset from the minimum and maximum of the array data. 00337 // If minVal is NaN, scale is set to 0. 00338 void makeScaleOffset (Float& scale, Float& offset, 00339 Float minVal, Float maxVal) const; 00340 00341 // Put a part of an array in a row using given scale/offset values. 00342 void putPart (uInt rownr, const Slicer& slicer, 00343 const Array<Complex>& array, 00344 Float scale, Float offset); 00345 00346 // Fill the array part into the full array and put it using the 00347 // given min/max values. 00348 void putFullPart (uInt rownr, const Slicer& slicer, 00349 Array<Complex>& fullArray, 00350 const Array<Complex>& partArray, 00351 Float minVal, Float maxVal); 00352 00353 public: 00354 // Define the "constructor" to construct this engine when a 00355 // table is read back. 00356 // This "constructor" has to be registered by the user of the engine. 00357 // If the engine is commonly used, its registration can be added 00358 // to the registerAllCtor function in DataManager.cc. 00359 // That function gets automatically invoked by the table system. 00360 static DataManager* makeObject (const String& dataManagerType, 00361 const Record& spec); 00362 }; 00363 00364 00365 00366 00367 // <summary> 00368 // Virtual column engine to scale a table Complex array for Single Dish data 00369 // </summary> 00370 00371 // <use visibility=export> 00372 00373 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tCompressComplex.cc"> 00374 // </reviewed> 00375 00376 // <prerequisite> 00377 //# Classes you should understand before using this one. 00378 // <li> CompressComplex 00379 // </prerequisite> 00380 00381 // <synopsis> 00382 // CompressComplexSD is similar to CompressComplex, but compresses 00383 // in a slighty different way optimized for single dish data. 00384 // Usually the imaginary part of single dish data is 0, so the scaling 00385 // is optimized for it. 00386 // <br>If the imaginary part is 0, the real part is scaled with 15 bits 00387 // extra to get a higher precision. The least significant bit is set to 0 00388 // indicating the imag==0. 00389 // <br>If the imaginary part is not 0, the real part is scaled normally. 00390 // The imaginary part is scaled with 1 bit less. The least significant bit 00391 // is set to 1 indicating that imag!=0. 00392 // </synopsis> 00393 00394 // <motivation> 00395 // This class is created on top of CompressComplex to cope with SD data 00396 // in a better way. Using CompressComplex often makes the imag part non-zero 00397 // if it is scaled as 0. 00398 // </motivation> 00399 00400 // <example> 00401 // <srcblock> 00402 // // Create the table description and 2 columns with indirect arrays in it. 00403 // // The Int column will be stored, while the double will be 00404 // // used as virtual. 00405 // TableDesc tableDesc ("", TableDesc::Scratch); 00406 // tableDesc.addColumn (ArrayColumnDesc<Int> ("storedArray")); 00407 // tableDesc.addColumn (ArrayColumnDesc<Complex> ("virtualArray")); 00408 // tableDesc.addColumn (ScalarColumnDesc<Complex> ("scale")); 00409 // tableDesc.addColumn (ScalarColumnDesc<Float> ("offset")); 00410 // 00411 // // Create a new table using the table description. 00412 // SetupNewTable newtab (tableDesc, "tab.data", Table::New); 00413 // 00414 // // Create the array scaling engine (with auto-scale) 00415 // // and bind it to the Complex column. 00416 // CompressComplexSD scalingEngine("virtualArray", "storedArray", 00417 // "scale", "offset"); 00418 // newtab.bindColumn ("virtualArray", scalingEngine); 00419 // // Create the table. 00420 // Table table (newtab); 00421 // 00422 // // Store a 3-D array (with dim. 2,3,4) into each row of the column. 00423 // // The shape of each array in the column is implicitly set by the put 00424 // // function. This will also set the shape of the underlying Int array. 00425 // ArrayColumn data (table, "virtualArray"); 00426 // Array<double> someArray(IPosition(4,2,3,4)); 00427 // someArray = 0; 00428 // for (uInt i=0, i<10; i++) { // table will have 10 rows 00429 // table.addRow(); 00430 // data.put (i, someArray) 00431 // } 00432 // </srcblock> 00433 // </example> 00434 00435 class CompressComplexSD : public CompressComplex 00436 { 00437 public: 00438 00439 // Construct an engine to scale all arrays in a column with 00440 // the given offset and scale factor. 00441 // StoredColumnName is the name of the column where the scaled 00442 // data will be put and must have data type Int. 00443 // The virtual column using this engine must have data type Complex. 00444 CompressComplexSD (const String& virtualColumnName, 00445 const String& storedColumnName, 00446 Float scale, 00447 Float offset = 0); 00448 00449 // Construct an engine to scale the arrays in a column. 00450 // The scale and offset values are taken from a column with 00451 // the given names. In that way each array has its own scale factor 00452 // and offset value. 00453 // An exception is thrown if these columns do not exist. 00454 // VirtualColumnName is the name of the virtual column and is used to 00455 // check if the engine gets bound to the correct column. 00456 // StoredColumnName is the name of the column where the scaled 00457 // data will be put and must have data type Int. 00458 // The virtual column using this engine must have data type Complex. 00459 CompressComplexSD (const String& virtualColumnName, 00460 const String& storedColumnName, 00461 const String& scaleColumnName, 00462 const String& offsetColumnName, 00463 Bool autoScale = True); 00464 00465 // Construct from a record specification as created by getmanagerSpec(). 00466 CompressComplexSD (const Record& spec); 00467 00468 // Destructor is mandatory. 00469 ~CompressComplexSD(); 00470 00471 // Return the type name of the engine (i.e. its class name). 00472 virtual String dataManagerType() const; 00473 00474 // Return the name of the class. 00475 // This includes the names of the template arguments. 00476 static String className(); 00477 00478 // Register the class name and the static makeObject "constructor". 00479 // This will make the engine known to the table system. 00480 static void registerClass(); 00481 00482 private: 00483 // Copy constructor is only used by clone(). 00484 // (so it is made private). 00485 CompressComplexSD (const CompressComplexSD&); 00486 00487 // Assignment is not needed and therefore forbidden 00488 // (so it is made private and not implemented). 00489 CompressComplexSD& operator= (const CompressComplexSD&); 00490 00491 // Clone the engine object. 00492 virtual DataManager* clone() const; 00493 00494 // Initialize the object for a new table. 00495 // It defines the keywords containing the engine parameters. 00496 virtual void create (uInt initialNrrow); 00497 00498 // Scale and/or offset target to array. 00499 // This is meant when reading an array from the stored column. 00500 // It optimizes for scale=1 and/or offset=0. 00501 virtual void scaleOnGet (Float scale, Float offset, 00502 Array<Complex>& array, 00503 const Array<Int>& target); 00504 00505 // Scale and/or offset array to target. 00506 // This is meant when writing an array into the stored column. 00507 // It optimizes for scale=1 and/or offset=0. 00508 virtual void scaleOnPut (Float scale, Float offset, 00509 const Array<Complex>& array, 00510 Array<Int>& target); 00511 00512 // Find minimum and maximum from the array data. 00513 // NaN and infinite values and zero imaginary parts are ignored. 00514 // If no values are finite, minimum and maximum are set to NaN. 00515 virtual void findMinMax (Float& minVal, Float& maxVal, 00516 const Array<Complex>& array) const; 00517 00518 public: 00519 // Define the "constructor" to construct this engine when a 00520 // table is read back. 00521 // This "constructor" has to be registered by the user of the engine. 00522 // If the engine is commonly used, its registration can be added 00523 // to the registerAllCtor function in DataManager.cc. 00524 // That function gets automatically invoked by the table system. 00525 static DataManager* makeObject (const String& dataManagerType, 00526 const Record& spec); 00527 }; 00528 00529 00530 00531 00532 inline Float CompressComplex::getScale (uInt rownr) 00533 { 00534 return (fixed_p ? scale_p : (*scaleColumn_p)(rownr)); 00535 } 00536 inline Float CompressComplex::getOffset (uInt rownr) 00537 { 00538 return (fixed_p ? offset_p : (*offsetColumn_p)(rownr)); 00539 } 00540 00541 00542 00543 } //# NAMESPACE CASACORE - END 00544 00545 #endif