ParallelImagerMixin.h

Go to the documentation of this file.
00001 /* -*- mode: c++ -*- */
00002 //# ParallelImagerMixin.h: Main class for parallel imaging
00003 //# Copyright (C) 2016
00004 //# Associated Universities, Inc. Washington DC, USA.
00005 //#
00006 //# This library is free software; you can redistribute it and/or modify it
00007 //# under the terms of the GNU Library General Public License as published by
00008 //# the Free Software Foundation; either version 2 of the License, or (at your
00009 //# option) any later version.
00010 //#
00011 //# This library is distributed in the hope that it will be useful, but WITHOUT
00012 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
00013 //# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
00014 //# License for more details.
00015 //#
00016 //# You should have received a copy of the GNU Library General Public License
00017 //# along with this library; if not, write to the Free Software Foundation,
00018 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
00019 //#
00020 //# Correspondence concerning AIPS++ should be addressed as follows:
00021 //#        Internet email: aips2-request@nrao.edu.
00022 //#        Postal address: AIPS++ Project Office
00023 //#                        National Radio Astronomy Observatory
00024 //#                        520 Edgemont Road
00025 //#                        Charlottesville, VA 22903-2475 USA
00026 //#
00027 #ifndef PARALLEL_IMAGER_MIXIN_H_
00028 #define PARALLEL_IMAGER_MIXIN_H_
00029 
00030 #include <string>
00031 #include <vector>
00032 
00033 #include <synthesis/ImagerObjects/MPIGlue.h>
00034 #include <casa/Containers/Record.h>
00035 #include <synthesis/ImagerObjects/ParallelImagerParams.h>
00036 #include <synthesis/ImagerObjects/ParamFieldIterator.h>
00037 #include <synthesis/ImagerObjects/SynthesisUtilMethods.h>
00038 #include <synthesis/ImagerObjects/SynthesisDeconvolverMixin.h>
00039 #include <synthesis/ImagerObjects/SynthesisNormalizerMixin.h>
00040 #include <synthesis/ImagerObjects/SynthesisImagerMixin.h>
00041 #include <synthesis/ImagerObjects/IterationControl.h>
00042 #include <synthesis/ImagerObjects/ContinuumPartitionMixin.h>
00043 #include <synthesis/ImagerObjects/CubePartitionMixin.h>
00044 #include <synthesis/ImagerObjects/SerialPartitionMixin.h>
00045 
00046 namespace casa {
00047 
00073 template <class T>
00074 class ParallelImagerMixin
00075         : public T {
00076 
00077 public:
00078 
00079         // ParallelImagerMixin constructor
00080         //
00081         // The various MPI communicators passed to the constructor are intended to
00082         // represent the following process groups:
00083         //
00084     // * worker_comm: all processes used for significant computation (likely
00085     //   just excluding a "front-end" or user-facing process).
00086         //
00087         // * imaging/normalization/deconvolution_comm: the groups of processes for
00088         //   each of these components. Note that the scope of each of these
00089         //   communicators defines the set of communicating processes that comprise
00090         //   the component; for example, in cube imaging each imaging_comm comprises
00091         //   a single process, whereas in continuum imaging a single imaging_comm
00092         //   comprises all (worker) processes. Such differences are determined by
00093         //   design decisions, and do not reflect inherent features of the
00094         //   framework.
00095         //
00096         // * iteration_comm: processes taking part in iteration control, should be
00097         //   those processes in worker_comm with the possible addition of a process
00098         //   for the tclean "front-end".
00099         //
00100         // Note that there may be overlap in the process groups for each of these
00101         // communicators. The mixin classes in this framework support such usage,
00102         // although the viability of that support depends on the usage of these
00103         // communicators by the wrapped components in addition to the mixin
00104         // classes. The current component classes make no use of these
00105         // communicators, so overlapping process groups are supported. When
00106         // component classes are using their respective communicators, it is
00107         // sufficient for safely calling MPI routines from component code to ensure
00108         // that concurrent access to a communicator by this framework and any
00109         // component threads is avoided. [One way this can be achieved is by
00110         // limiting calls to MPI routines on the provided communicator only to
00111         // methods directly called by this framework.]
00112         //
00113         ParallelImagerMixin(MPI_Comm worker_comm,
00114                             MPI_Comm imaging_comm,
00115                             MPI_Comm normalization_comm,
00116                             MPI_Comm deconvolution_comm,
00117                             MPI_Comm iteration_comm,
00118                             int niter,
00119                             bool calculate_psf,
00120                             bool calculate_residual,
00121                             string save_model,
00122                             ParallelImagerParams &params)
00123         : niter(niter)
00124         , calculate_psf(calculate_psf)
00125         , calculate_residual(calculate_residual)
00126         , save_model(save_model)
00127         , worker_comm(worker_comm)
00128         , imaging_comm(imaging_comm)
00129         , deconvolution_comm(deconvolution_comm)
00130         , normalization_comm(normalization_comm)
00131         , iteration_comm(iteration_comm) {
00132                 // Get parameters for this process
00133                 ParallelImagerParams my_params = T::get_params(worker_comm, params);
00134 
00135                 // Convert parameters to other formats used by synthesis imaging
00136                 // components, putting them into vectors by field index (not field key
00137                 // as used by the Records).
00138                 auto to_synthesis_params_select = [] (const Record &r) {
00139                         SynthesisParamsSelect pars;
00140                         pars.fromRecord(r);
00141                         return pars;
00142                 };
00143                 vector<SynthesisParamsSelect> selection_params =
00144                         transformed_by_field<SynthesisParamsSelect>(
00145                                 my_params.selection, to_synthesis_params_select, "ms");
00146 
00147                 auto to_synthesis_params_image = [] (const Record &r) {
00148                         SynthesisParamsImage pars;
00149                         pars.fromRecord(r);
00150                         return pars;
00151                 };
00152                 vector<SynthesisParamsImage> image_params =
00153                         transformed_by_field<SynthesisParamsImage>(
00154                                 my_params.image, to_synthesis_params_image);
00155 
00156                 auto to_synthesis_params_grid = [] (const Record &r) {
00157                         SynthesisParamsGrid pars;
00158                         pars.fromRecord(r);
00159                         return pars;
00160                 };
00161                 vector<SynthesisParamsGrid> grid_params =
00162                         transformed_by_field<SynthesisParamsGrid>(
00163                                 my_params.grid, to_synthesis_params_grid);
00164 
00165                 auto to_synthesis_params_deconv = [] (const Record &r) {
00166                         SynthesisParamsDeconv pars;
00167                         pars.fromRecord(r);
00168                         return pars;
00169                 };
00170                 vector<SynthesisParamsDeconv> deconvolution_params =
00171                         transformed_by_field<SynthesisParamsDeconv>(
00172                                 my_params.deconvolution, to_synthesis_params_deconv);
00173 
00174                 auto to_vector_params = [] (const Record &r) {
00175                         Record result = r;
00176                         return result;
00177                 };
00178                 vector<Record> normalization_params =
00179                         transformed_by_field<Record>(my_params.normalization, to_vector_params);
00180 
00181                 // Configure components
00182                 T::setup_imager(imaging_comm, selection_params, image_params,
00183                                 grid_params, my_params.weight);
00184                 T::setup_normalizer(normalization_comm, normalization_params);
00185                 T::setup_deconvolver(deconvolution_comm, deconvolution_params);
00186                 // don't initialize iteration control on any rank until all workers have
00187                 // completed initialization (need second barrier for case in which there
00188                 // are processes in iteration_comm that are not in worker_comm)
00189                 MPI_Barrier(worker_comm);
00190                 MPI_Barrier(iteration_comm);
00191                 T::setup_iteration_controller(iteration_comm, my_params.iteration);
00192         }
00193 
00194         ~ParallelImagerMixin() {
00195                 T::teardown_imager();
00196                 T::teardown_normalizer();
00197                 T::teardown_deconvolver();
00198                 T::teardown_iteration_controller();
00199                 auto free_comm = [](MPI_Comm *comm) {
00200                         if (*comm != MPI_COMM_NULL
00201                             && *comm != MPI_COMM_SELF
00202                             && *comm != MPI_COMM_WORLD)
00203                                 MPI_Comm_free(comm);
00204                 };
00205                 free_comm(&worker_comm);
00206                 free_comm(&imaging_comm);
00207                 free_comm(&normalization_comm);
00208                 free_comm(&deconvolution_comm);
00209                 free_comm(&iteration_comm);
00210         }
00211 
00212         // Top level imaging method. Note that differences in parallel continuum,
00213         // parallel cube, and serial imaging are not apparent at this level.
00214         Record clean() {
00215                 if (calculate_psf) {
00216                         T::make_psf();
00217                         T::normalize_psf();
00218                 }
00219                 if (niter >= 0) {
00220                         if (calculate_residual) {
00221                                 run_major_cycle();
00222                         }
00223                         else if (niter == 0 && save_model != "none") {
00224                                 T::normalize_model();
00225                                 T::predict_model();
00226                         }
00227                         if (niter > 0)
00228                                 while (run_minor_cycle())
00229                                         run_major_cycle();
00230                 }
00231                 T::restore_images();
00232                 T::concat_images("virtualnomove");
00233                 Record result = T::get_summary(); // includes plot_report
00234                 MPI_Barrier(worker_comm);
00235                 return result;
00236         }
00237 
00238 protected:
00239 
00240         void run_major_cycle() {
00241                 T::normalize_model();
00242                 T::execute_major_cycle();
00243                 T::normalize_residual();
00244                 T::denormalize_model();
00245         }
00246 
00247         bool run_minor_cycle() {
00248                 T::initialize_minor_cycle();
00249                 bool result = !T::is_clean_complete();
00250                 if (result) T::execute_minor_cycle();
00251                 return result;
00252         }
00253 
00254         int niter;
00255 
00256         bool calculate_psf;
00257 
00258         bool calculate_residual;
00259 
00260         string save_model;
00261 
00262         MPI_Comm worker_comm;
00263 
00264         MPI_Comm imaging_comm;
00265 
00266         MPI_Comm deconvolution_comm;
00267 
00268         MPI_Comm normalization_comm;
00269 
00270         MPI_Comm iteration_comm;
00271 
00272         // Convenience function for transforming input parameter Record fields.
00273         template<class T1>
00274         static vector<T1> transformed_by_field(Record &rec,
00275                                                T1 (*fn)(const Record &),
00276                                                const string &prefix = "") {
00277                 vector<T1> result;
00278                 auto add_to_result = [&](const Record &rec) {
00279                         result.push_back(fn(rec));
00280                 };
00281                 std::for_each(ParamFieldIterator::begin(&rec, prefix),
00282                               ParamFieldIterator::end(&rec, prefix),
00283                               add_to_result);
00284                 return result;
00285         };
00286 };
00287 
00288 // Parallel continuum imager class
00289 typedef ParallelImagerMixin<
00290         ContinuumPartitionMixin<
00291                 SynthesisImagerMixin<
00292                         SynthesisNormalizerMixin<
00293                                 SynthesisDeconvolverMixin<
00294                                         IterationControl> > > > >
00295 ContinuumParallelImagerImpl;
00296 
00297 // Parallel cube imager class
00298 typedef ParallelImagerMixin<
00299         CubePartitionMixin<
00300                 SynthesisImagerMixin<
00301                         SynthesisNormalizerMixin<
00302                                 SynthesisDeconvolverMixin<
00303                                         IterationControl> > > > >
00304 CubeParallelImagerImpl;
00305 
00306 // Serial (non-MPI) imager class -- allows ParallelImagerMixin type to be used
00307 // regardless of serial vs parallel CASA. However, the naming is
00308 // unfortunate...suggestions are welcome!
00309 typedef ParallelImagerMixin<
00310         SerialPartitionMixin<
00311                 SynthesisImagerMixin<
00312                         SynthesisNormalizerMixin<
00313                                 SynthesisDeconvolverMixin<
00314                                         IterationControl> > > > >
00315 SerialParallelImagerImpl;
00316 
00317 } // namespace casa
00318 
00319 #endif // PARALLEL_IMAGER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines

Generated on 31 Aug 2016 for casa by  doxygen 1.6.1