TableParse.h

Go to the documentation of this file.
00001 //# TableParse.h: Classes to hold results from table grammar parser
00002 //# Copyright (C) 1994,1995,1997,1998,1999,2000,2001,2003
00003 //# Associated Universities, Inc. Washington DC, USA.
00004 //#
00005 //# This library is free software; you can redistribute it and/or modify it
00006 //# under the terms of the GNU Library General Public License as published by
00007 //# the Free Software Foundation; either version 2 of the License, or (at your
00008 //# option) any later version.
00009 //#
00010 //# This library is distributed in the hope that it will be useful, but WITHOUT
00011 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
00012 //# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
00013 //# License for more details.
00014 //#
00015 //# You should have received a copy of the GNU Library General Public License
00016 //# along with this library; if not, write to the Free Software Foundation,
00017 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
00018 //#
00019 //# Correspondence concerning AIPS++ should be addressed as follows:
00020 //#        Internet email: aips2-request@nrao.edu.
00021 //#        Postal address: AIPS++ Project Office
00022 //#                        National Radio Astronomy Observatory
00023 //#                        520 Edgemont Road
00024 //#                        Charlottesville, VA 22903-2475 USA
00025 //#
00026 //# $Id$
00027 
00028 #ifndef TABLES_TABLEPARSE_H
00029 #define TABLES_TABLEPARSE_H
00030 
00031 //# Includes
00032 #include <casacore/casa/aips.h>
00033 #include <casacore/tables/Tables/Table.h>
00034 #include <casacore/tables/Tables/TableDesc.h>
00035 #include <casacore/tables/TaQL/ExprNode.h>
00036 #include <casacore/tables/TaQL/TaQLResult.h>
00037 #include <casacore/tables/TaQL/ExprGroup.h>
00038 #include <casacore/casa/BasicSL/String.h>
00039 #include <casacore/casa/Utilities/Sort.h>
00040 #include <casacore/casa/Containers/Record.h>
00041 #include <casacore/casa/Containers/Block.h>
00042 #include <map>
00043 #include <vector>
00044 #include <limits>
00045 
00046 namespace casacore { //# NAMESPACE CASACORE - BEGIN
00047 
00048 //# Forward Declarations
00049 class TableExprNodeSet;
00050 class TableExprNodeSetElem;
00051 class TableExprNodeIndex;
00052 class TableColumn;
00053 class AipsIO;
00054 template<class T> class Vector;
00055 
00056 
00057 // <summary>
00058 // Class to hold values from table grammar parser
00059 // </summary>
00060 
00061 // <use visibility=local>
00062 
00063 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tTableGram">
00064 // </reviewed>
00065 
00066 // <prerequisite>
00067 //# Classes you should understand before using this one.
00068 // </prerequisite>
00069 
00070 // <etymology>
00071 // TableParse is the class used to parse a table command.
00072 // </etymology>
00073 
00074 // <synopsis> 
00075 // TableParse is used by the parser of table select statements.
00076 // The parser is written in Bison and Flex in files TableGram.y and .l.
00077 // The statements in there use the routines in this file to act
00078 // upon a reduced rule.
00079 // Since multiple tables can be given (with a shorthand), the table
00080 // names are stored in a container. The variable names can be qualified
00081 // by the table name and will be looked up in the appropriate table.
00082 //
00083 // A select command is similar to SQL and can look like:
00084 //    SELECT columns FROM tab1 sh1, tab2 sh2, tab3 WHERE
00085 //           sh1.field == 3*sh1.field2 ... ORDERBY columns GIVING table
00086 // This is described in more detail in TableGram.l.
00087 //
00088 // The class TableParse only contains information about a table
00089 // used in the table command.
00090 //
00091 // Global functions are used to operate on the information.
00092 // The main function is the global function tableCommand.
00093 // It executes the given TaQL command and returns the resulting table.
00094 // This is, in fact, the only function to be used by a user.
00095 // </synopsis> 
00096 
00097 // <motivation>
00098 // It is necessary to be able to give a table select command in ASCII.
00099 // This can be used in a CLI or in the table browser to get a subset
00100 // of a table or to sort a table.
00101 // </motivation>
00102 
00103 //# <todo asof="$DATE:$">
00104 //# A List of bugs, limitations, extensions or planned refinements.
00105 //# </todo>
00106 
00107 
00108 class TableParse
00109 {
00110 
00111 public:
00112   // Default constructor for container class.
00113   TableParse();
00114 
00115   // Copy constructor (copy semantics).
00116   TableParse (const TableParse&);
00117 
00118   // Assignment (copy semantics).
00119   TableParse& operator= (const TableParse&);
00120 
00121   // Associate the table and the shorthand.
00122   TableParse (const Table& table, const String& shorthand);
00123 
00124   // Test if shorthand matches.
00125   Bool test (const String& shortHand) const;
00126 
00127   // Get the shorthand.
00128   const String& shorthand() const;
00129 
00130   // Get table object.
00131   const Table& table() const;
00132 
00133 private:
00134   String  shorthand_p;
00135   Table   table_p;
00136 };
00137 
00138 
00139 
00140 // <synopsis>
00141 // Parse and execute the given command.
00142 // It will open (and close) all tables needed.
00143 // It returns the resulting table. 
00144 // The command type (select or update) and the selected or updated
00145 // column names can be returned.
00146 // Zero or more temporary tables can be used in the command
00147 // using the $nnn syntax.
00148 // </synopsis>
00149 // <group name=tableCommand>
00150 TaQLResult tableCommand (const String& command);
00151 
00152 TaQLResult tableCommand (const String& command,
00153                          const Table& tempTable);
00154 TaQLResult tableCommand (const String& command,
00155                          const std::vector<const Table*>& tempTables);
00156 TaQLResult tableCommand (const String& command,
00157                          Vector<String>& columnNames);
00158 TaQLResult tableCommand (const String& command,
00159                          Vector<String>& columnNames,
00160                          String& commandType);
00161 TaQLResult tableCommand (const String& command,
00162                          const std::vector<const Table*>& tempTables,
00163                          Vector<String>& columnNames);
00164 TaQLResult tableCommand (const String& command,
00165                          const std::vector<const Table*>& tempTables,
00166                          Vector<String>& columnNames,
00167                          String& commandType);
00168 // </group>
00169 
00170 
00171 
00172 
00173 // <summary>
00174 // Helper class for sort keys in TableParse
00175 // </summary>
00176 
00177 // <use visibility=local>
00178 
00179 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
00180 // </reviewed>
00181 
00182 // <prerequisite>
00183 //# Classes you should understand before using this one.
00184 //   <li> TableParse
00185 // </prerequisite>
00186 
00187 // <etymology>
00188 // TableParseSort holds a sort expression and order.
00189 // </etymology>
00190 
00191 // <synopsis> 
00192 // A table command is parsed.
00193 // An object of this class is used to hold the sort expression
00194 // and sort order.
00195 // </synopsis> 
00196 
00197 
00198 class TableParseSort
00199 {
00200 public:
00201     // Construct from a given expression.
00202     // The order is not given.
00203     TableParseSort();
00204 
00205     // Construct from a given expression.
00206     // The order is not given.
00207     explicit TableParseSort (const TableExprNode&);
00208 
00209     // Construct from a given expression and for the given order.
00210     TableParseSort (const TableExprNode&, Sort::Order);
00211 
00212     ~TableParseSort();
00213 
00214     // Get the expression node.
00215     const TableExprNode& node() const;
00216 
00217     // Get the sort order.
00218     Sort::Order order() const;
00219 
00220     // Is the order given?
00221     Bool orderGiven() const;
00222 
00223 private:
00224     // Check if the node results in a scalar and does not contain
00225     // aggregate functions.
00226     void checkNode() const;
00227 
00228     TableExprNode node_p;
00229     Sort::Order   order_p;
00230     Bool          given_p;
00231 };
00232 
00233 
00234 
00235 
00236 // <summary>
00237 // Helper class for updates in TableParse
00238 // </summary>
00239 
00240 // <use visibility=local>
00241 
00242 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
00243 // </reviewed>
00244 
00245 // <prerequisite>
00246 //# Classes you should understand before using this one.
00247 //   <li> TableParse
00248 // </prerequisite>
00249 
00250 // <etymology>
00251 // TableParseUpdate holds a column name, optional indices, optional mask,
00252 // and an update expression.
00253 // </etymology>
00254 
00255 // <synopsis> 
00256 // A table command is parsed.
00257 // An object of this class is used to hold the column name, optional indices,
00258 // and value expression for the UPDATE command.
00259 // </synopsis> 
00260 
00261 
00262 class TableParseUpdate
00263 {
00264 public:
00265   TableParseUpdate()
00266     : indexPtr_p(0) {}
00267 
00268   // Construct from a column name and expression.
00269   // By default it checks if no aggregate functions are used.
00270   TableParseUpdate (const String& columnName,
00271                     const String& columnNameMask,
00272                     const TableExprNode&,
00273                     Bool checkAggr=True);
00274 
00275   // Construct from a column name, subscripts or mask, and expression.
00276   // It checks if no aggregate functions are used.
00277   TableParseUpdate (const String& columnName,
00278                     const String& columnNameMask,
00279                     const TableExprNodeSet& indices,
00280                     const TableExprNode&,
00281                     const TaQLStyle&);
00282 
00283   // Construct from a column name, subscripts and mask, and expression.
00284   // It checks if no aggregate functions are used.
00285   // It checks if one of the indices represents subscripts, the other a mask.
00286   TableParseUpdate (const String& columnName,
00287                     const String& columnNameMask,
00288                     const TableExprNodeSet& indices1,
00289                     const TableExprNodeSet& indices2,
00290                     const TableExprNode&,
00291                     const TaQLStyle&);
00292   // Handle the subscripts or mask.
00293   // It checks if subscripts or mask was not already used.
00294   void handleIndices (const TableExprNodeSet& indices,
00295                       const TaQLStyle& style);
00296   ~TableParseUpdate();
00297 
00298   // Set the column name.
00299   void setColumnName (const String& name);
00300 
00301   // Set the column name forthe mask.
00302   void setColumnNameMask (const String& name);
00303 
00304   // Get the column name.
00305   const String& columnName() const;
00306 
00307   // Get the possible column name for the mask.
00308   const String& columnNameMask() const;
00309 
00310   // Tell if the mask is given first (i.e., before slice).
00311   Bool maskFirst() const
00312     { return maskFirst_p; }
00313 
00314   // Get the pointer to the indices.
00315   TableExprNodeIndex* indexPtr() const;
00316 
00317   // Get the index expression node.
00318   const TableExprNode& indexNode() const;
00319 
00320   // Get the expression node.
00321   // <group>
00322   const TableExprNode& node() const;
00323   TableExprNode& node();
00324   // </group>
00325 
00326   // Get the mask.
00327   const TableExprNode& mask() const
00328     { return mask_p; }
00329 
00330   // Adapt the possible unit of the expression to the possible unit
00331   // of the column.
00332   void adaptUnit (const Unit& columnUnit);
00333 
00334 private:
00335   String              columnName_p;
00336   String              columnNameMask_p;
00337   Bool                maskFirst_p;  //# True = mask is given before slice
00338   TableExprNodeIndex* indexPtr_p;   //# copy of pointer in indexNode_p
00339   TableExprNode       indexNode_p;
00340   TableExprNode       mask_p;
00341   TableExprNode       node_p;
00342 };
00343 
00344 
00345 
00346 
00347 // <summary>
00348 // Select-class for flex/bison scanner/parser for TableParse
00349 // </summary>
00350 
00351 // <use visibility=local>
00352 
00353 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
00354 // </reviewed>
00355 
00356 // <prerequisite>
00357 //# Classes you should understand before using this one.
00358 //  <li> TableParse
00359 //  <li> TableGram.l and .y  (flex and bison grammar)
00360 // </prerequisite>
00361 
00362 // <synopsis> 
00363 // This class is needed for the the actions in the flex scanner
00364 // and bison parser.
00365 // This stores the information by constructing TableParse objects
00366 // as needed and storing them in a vector.
00367 // </synopsis> 
00368 
00369 // <motivation>
00370 // It is necessary to be able to give a table select command in ASCII.
00371 // This can be used in a CLI or in the table browser to get a subset
00372 // of a table or to sort a table.
00373 // </motivation>
00374 
00375 //# <todo asof="$DATE:$">
00376 //# A List of bugs, limitations, extensions or planned refinements.
00377 //# </todo>
00378 
00379 
00380 class TableParseSelect
00381 {
00382 public:
00383   enum CommandType {
00384     PSELECT,
00385     PUPDATE,
00386     PINSERT,
00387     PDELETE,
00388     PCOUNT,
00389     PCALC,
00390     PCRETAB,
00391     PALTTAB
00392   };
00393 
00394   enum GroupAggrType {
00395     GROUPBY=1,
00396     AGGR_FUNCS=2,
00397     ONLY_COUNTALL=4
00398   };
00399 
00400   // Construct.
00401   TableParseSelect (CommandType type);
00402 
00403   // Destructor.
00404   ~TableParseSelect();
00405 
00406   // Return the command type.
00407   CommandType commandType() const
00408     { return commandType_p; }
00409 
00410   // Return the expression node.
00411   TableExprNode getNode() const
00412     { return node_p; }
00413 
00414   // Create a temporary table in no tables are given in FROM.
00415   void makeTableNoFrom (const vector<TableParseSelect*>& stack);
00416 
00417   // Execute the select command (select/sort/projection/groupby/having/giving).
00418   // The setInGiving flag tells if a set in the GIVING part is allowed.
00419   // The mustSelect flag tells if a SELECT command must do something.
00420   // Usually that is required, but not for a SELECT in an INSERT command.
00421   // Optionally the maximum nr of rows to be selected can be given.
00422   // It will be used as the default value for the LIMIT clause.
00423   // 0 = no maximum.
00424   void execute (Bool showTimings, Bool setInGiving,
00425                 Bool mustSelect, uInt maxRow, Bool doTracing=False);
00426 
00427   // Execute a query in a from clause resulting in a Table.
00428   Table doFromQuery (Bool showTimings);
00429 
00430   // Execute a subquery and create an appropriate node for the result.
00431   TableExprNode doSubQuery (Bool showTimings);
00432 
00433   // Test if a subquery has sufficient elements.
00434   // It uses default LIMIT=1, but that can be overidden in the subquery.
00435   // The flag tells if NOT EXISTS or EXISTS was given.
00436   TableExprNode doExists (Bool noexists, Bool showTimings);
00437 
00438   // Show the expression tree.
00439   void show (ostream& os) const;
00440 
00441   // Keep the selection expression.
00442   void handleWhere (const TableExprNode&);
00443 
00444   // Keep the groupby expressions.
00445   // It checks if they are all scalar expressions.
00446   void handleGroupby (const vector<TableExprNode>&, Bool rollup);
00447 
00448   // Keep the having expression.
00449   void handleHaving (const TableExprNode&);
00450 
00451   // Keep the expression of a calculate command.
00452   void handleCalcComm (const TableExprNode&);
00453 
00454   // Keep the create table command.
00455   void handleCreTab (const Record& dmInfo);
00456 
00457   // Keep the column specification in a create table command.
00458   void handleColSpec (const String& columnName, const String& dataType,
00459                       const Record& spec, Bool isCOrder=False);
00460 
00461   // Reopen the table (for update) used in the ALTER TABLE command.
00462   void handleAltTab();
00463 
00464   // Add columns to the table of ALTER TABLE.
00465   // The column descriptions have already been added to tableDesc_p.
00466   void handleAddCol (const Record& dmInfo);
00467 
00468   // Add a keyword or replace a keyword with the value of another keyword.
00469   // The keywords can be table or column keywords (col::key).
00470   ValueHolder getRecFld (const String& name);
00471 
00472   // Define a field with the given data type in the Record.
00473   static void setRecFld (RecordInterface& rec,
00474                          const String& name,
00475                          const String& dtype,
00476                          const ValueHolder& vh);
00477 
00478   // Get the type string. If empty, it is made from the given
00479   // data type.
00480   static String getTypeString (const String& typeStr, DataType type);
00481 
00482   // Add a keyword or replace a keyword with a value.
00483   // The keyword can be a table or column keyword (col::key).
00484   // The data type string can be empty leaving the data type unchanged.
00485   void handleSetKey (const String& name, const String& dtype,
00486                      const ValueHolder& value);
00487 
00488   // Rename a table or column keyword.
00489   void handleRenameKey (const String& oldName, const String& newName);
00490 
00491   // Remove a table or column keyword.
00492   void handleRemoveKey (const String& name);
00493 
00494   // Split the given name into optional shorthand, column and fields.
00495   // Find the keywordset for it and fill in the final keyword name.
00496   // It is a helper function for handleSetKey, etc.
00497   TableRecord& findKeyword (const String& name, String& keyName);
00498 
00499   // Add an update object.
00500   void addUpdate (TableParseUpdate* upd);
00501 
00502   // Set the insert expressions for all rows.
00503   void setInsertExprs (const std::vector<TableExprNode> exprs)
00504     { insertExprs_p = exprs; }
00505 
00506   // Keep the update expressions.
00507   void handleUpdate();
00508 
00509   // Make ready for the insert expression.
00510   // The first one uses values (added via addUpdate),
00511   // the second one a subquery.
00512   // <group>
00513   void handleInsert();
00514   void handleInsert (TableParseSelect* sel);
00515   // </group>
00516 
00517   // Make ready for a COUNT command.
00518   // It checks if all column expressions are scalar.
00519   void handleCount();
00520 
00521   // Keep the sort expressions.
00522   void handleSort (const std::vector<TableParseSort>& sortList,
00523                    Bool noDuplicates, Sort::Order defaultSortOrder);
00524 
00525   // Evaluate and keep limit/offset/stride given as start:end:incr
00526   void handleLimit (const TableExprNodeSetElem& expr);
00527 
00528   // Evaluate and keep the limit value.
00529   void handleLimit (const TableExprNode& expr);
00530 
00531   // Evaluate and keep the offset value.
00532   void handleOffset (const TableExprNode& expr);
00533 
00534   // Evaluate and add the rows.
00535   void handleAddRow (const TableExprNode& expr);
00536 
00537   // Add a table nr, name, or object to the container.
00538   void addTable (Int tabnr, const String& name,
00539                  const Table& table,
00540                  const String& shorthand,
00541                  const vector<const Table*> tempTables,
00542                  const vector<TableParseSelect*>& stack);
00543 
00544   // Make a Table object for given name, seqnr or so.
00545   // If <src>alwaysOpen=False</src> the table will only be looked up,
00546   // but not opened if not found. This is meant for concatenated tables
00547   // in TaQLNodeHandler.
00548   Table makeTable (Int tabnr, const String& name,
00549                    const Table& ftab,
00550                    const String& shorthand,
00551                    const vector<const Table*> tempTables,
00552                    const vector<TableParseSelect*>& stack,
00553                    Bool alwaysOpen=True);
00554 
00555   // Replace the first table (used by CALC command).
00556   void replaceTable (const Table& table);
00557 
00558   // Find the keyword or column name and create a TableExprNode from it.
00559   // If <src>tryProj=True</src> it is first tried if the column is a coluymn
00560   // in the projected table (i.e., result from the SELECT part).
00561   TableExprNode handleKeyCol (const String& name, Bool tryProj);
00562 
00563   // Handle a slice operator.
00564   static TableExprNode handleSlice (const TableExprNode& array,
00565                                     const TableExprNodeSet& indices,
00566                                     const TaQLStyle&);
00567 
00568   // Handle a function.
00569   TableExprNode handleFunc (const String& name,
00570                             const TableExprNodeSet& arguments,
00571                             const TaQLStyle&);
00572 
00573   // Make a function object node for the given function name and arguments.
00574   // The ignoreFuncs vector contains invalid function codes.
00575   static TableExprNode makeFuncNode (TableParseSelect*,
00576                                      const String& name,
00577                                      const TableExprNodeSet& arguments,
00578                                      const Vector<int>& ignoreFuncs,
00579                                      const Table& table,
00580                                      const TaQLStyle&);
00581 
00582   // Add a column to the list of column names.
00583   void handleColumn (Int type, const String& name, const TableExprNode& expr,
00584                      const String& newName, const String& nameMask,
00585                      const String& newDtype);
00586 
00587   // Finish the addition of columns to the list of column names.
00588   void handleColumnFinish (Bool distinct);
00589 
00590   // Set the DataManager info for a new table.
00591   void setDMInfo (const Record& dminfo)
00592     { dminfo_p = dminfo;}
00593 
00594   // Handle the name and type given in a GIVING clause.
00595   void handleGiving (const String& name, const Record& type);
00596 
00597   // Handle the set given in a GIVING clause.
00598   void handleGiving (const TableExprNodeSet&);
00599 
00600   // Get the projected column names.
00601   const Block<String>& getColumnNames() const;
00602 
00603   // Get the resulting table.
00604   const Table& getTable() const;
00605 
00606   // An exception is thrown if the node uses an aggregate function.
00607   static void checkAggrFuncs (const TableExprNode& node);
00608 
00609   // Split a name into its parts (shorthand, column and field names).
00610   // True is returned if the name contained a keyword part.
00611   // In that case fieldNames contains the keyword name and the possible
00612   // subfields. The possible shorthand and the column name are
00613   // filled in if it is a column keyword.
00614   // If the name represents a column, fieldNames contains the subfields
00615   // of the column (for the case where the column contains records).
00616   // If the name is invalid, an exception is thrown if checkError=True.
00617   // Otherwise the name is treated as a normal name without keyword.
00618   // If allowEmtpy is True, :: is allowed, otherwise an error is thrown.
00619   static Bool splitName (String& shorthand, String& columnName,
00620                          Vector<String>& fieldNames, const String& name,
00621                          Bool checkError, Bool isKeyword, Bool allowNoKey);
00622 
00623 private:
00624   // Test if groupby or aggregate functions are given.
00625   // <br> bit 0:  on = groupby is given
00626   // <br> bit 1:  on = aggregate functions are given
00627   // <br> bit 2:  on = only select count(*) aggregate function is given
00628   Int testGroupAggr (vector<TableExprNodeRep*>& aggr) const;
00629 
00630   // Get the aggregate functions used in SELECT and HAVING.
00631   vector<TableExprNodeRep*> getAggrNodes() const;
00632 
00633   // Try to make a UDF function node for the given function name and arguments.
00634   static TableExprNode makeUDFNode (TableParseSelect*,
00635                                     const String& name,
00636                                     const TableExprNodeSet& arguments,
00637                                     const Table& table,
00638                                     const TaQLStyle&);
00639 
00640   // Find the function code belonging to a function name.
00641   // Functions to be ignored can be given (as function type values).
00642   // If the function name is unknown, NRFUNC is returned.
00643   static TableExprFuncNode::FunctionType findFunc
00644                                    (const String& name,
00645                                     uInt narguments,
00646                                     const Vector<Int>& ignoreFuncs);
00647 
00648   // Do the update step.
00649   // Rows 0,1,2,.. in UpdTable are updated from the expression result
00650   // for the rows in the given rownrs vector.
00651   void doUpdate (Bool showTimings, const Table& origTable,
00652                  Table& updTable, const Vector<uInt>& rownrs,
00653                  const CountedPtr<TableExprGroupResult>& groups =
00654                  CountedPtr<TableExprGroupResult>());
00655 
00656   // Do the insert step and return a selection containing the new rows.
00657   Table doInsert (Bool showTimings, Table& table);
00658 
00659   // Do the delete step.
00660   void doDelete (Bool showTimings, Table& table);
00661 
00662   // Do the count step returning a memory table containing the unique
00663   // column values and the counts of the column values.
00664   Table doCount (Bool showTimings, const Table&);
00665 
00666   // Do the projection step returning a table containing the projection.
00667   Table doProject (Bool showTimings, const Table&,
00668                    const CountedPtr<TableExprGroupResult>& groups =
00669                    CountedPtr<TableExprGroupResult>());
00670 
00671   // Do the projection containing column expressions.
00672   // Use the selected or unselected columns depending on <src>useSel</src>.
00673   Table doProjectExpr (Bool useSel,
00674                        const CountedPtr<TableExprGroupResult>& groups);
00675 
00676   // Create a table using the given parameters.
00677   // The variables set by handleGiven are used for name and type.
00678   Table createTable (const TableDesc& td,
00679                      Int64 nrow, const Record& dmInfo);
00680 
00681   // Make the (empty) table for the epxression in the SELECT clause.
00682   void makeProjectExprTable();
00683 
00684   // Fill projectExprSelColumn_p telling the columns to be projected
00685   // at the first stage.
00686   void makeProjectExprSel();
00687 
00688   // Add a column node to applySelNodes_p.
00689   void addApplySelNode (const TableExprNode& node)
00690     { applySelNodes_p.push_back (node); }
00691 
00692   // Set the selected rows for the column objects in applySelNodes_p.
00693   // These nodes refer the original table. They requires different row
00694   // numbers than the selected groups and projected columns.
00695   // rownrs_p is changed to use row 0..n.
00696   // It returns the Table containing the subset of rows in the input Table.
00697   Table adjustApplySelNodes (const Table&);
00698 
00699   // Do the groupby/aggregate step and return its result.
00700   CountedPtr<TableExprGroupResult> doGroupby
00701   (bool showTimings, vector<TableExprNodeRep*> aggrNodes,
00702    Int groupAggrUsed);
00703 
00704   // Do the HAVING step.
00705   void doHaving (Bool showTimings,
00706                  const CountedPtr<TableExprGroupResult>& groups);
00707 
00708   // Do a groupby/aggregate step that only does a 'select count(*)'.
00709   CountedPtr<TableExprGroupResult> doOnlyCountAll (TableExprNodeRep* aggrNode);
00710 
00711   // Do a full groupby/aggregate step.
00712   CountedPtr<TableExprGroupResult> doGroupByAggr
00713   (const vector<TableExprNodeRep*>& aggrNodes);
00714 
00715   // Do the sort step.
00716   void doSort (Bool showTimings);
00717 
00718   // Do the limit/offset step.
00719   void  doLimOff (Bool showTimings);
00720   Table doLimOff (Bool showTimings, const Table& table);
00721 
00722   // Do the 'select distinct' step.
00723   Table doDistinct (Bool showTimings, const Table& table);
00724 
00725   // Finish the table (rename, copy, and/or flush).
00726   Table doFinish (Bool showTimings, Table& table);
00727 
00728   // Update the values in the columns (helpers of doUpdate).
00729   // <group>
00730   template<typename TCOL, typename TNODE>
00731   void updateValue (uInt row, const TableExprId& rowid,
00732                     Bool isScalarCol, const TableExprNode& node,
00733                     const Array<Bool>& mask, Bool maskFirst,
00734                     TableColumn& col, const Slicer* slicerPtr,
00735                     ArrayColumn<Bool>& maskCol);
00736   template<typename TCOL, typename TNODE>
00737   void updateScalar (uInt row, const TableExprId& rowid,
00738                      const TableExprNode& node,
00739                      TableColumn& col);
00740   template<typename TCOL, typename TNODE>
00741   void updateArray (uInt row, const TableExprId& rowid,
00742                     const TableExprNode& node,
00743                     const Array<TNODE>& res,
00744                     ArrayColumn<TCOL>& col);
00745   template<typename TCOL, typename TNODE>
00746   void updateSlice (uInt row, const TableExprId& rowid,
00747                     const TableExprNode& node,
00748                     const Array<TNODE>& res,
00749                     const Slicer& slice,
00750                     ArrayColumn<TCOL>& col);
00751   template<typename TCOL, typename TNODE>
00752   void copyMaskedValue (uInt row, ArrayColumn<TCOL>& acol,
00753                         const Slicer* slicerPtr,
00754                         const TNODE* val,
00755                         uInt incr, const Array<Bool>& mask);
00756   Array<Bool> makeMaskSlice (const Array<Bool>& mask,
00757                              Bool maskFirst,
00758                              const IPosition& shapeCol,
00759                              const Slicer* slicerPtr);
00760   void checkMaskColumn (Bool hasMask,
00761                         const ArrayColumn<Bool>& maskCol,
00762                         const TableColumn& col);
00763   // </group>
00764 
00765   // Make a data type from the string.
00766   // It checks if it is compatible with the given (expression) data type.
00767   DataType makeDataType (DataType dtype, const String& dtstr,
00768                          const String& colName);
00769 
00770   // Get the order for this key. Use the default order_p if not
00771   // explicitly given with the key.
00772   Sort::Order getOrder (const TableParseSort& key) const;
00773 
00774   // Make an array from the contents of a column in a subquery.
00775   TableExprNode getColSet();
00776 
00777   // Make a set from the results of the subquery.
00778   TableExprNode makeSubSet() const;
00779 
00780   // Evaluate an int scalar expression.
00781   Int64 evalIntScaExpr (const TableExprNode& expr) const;
00782 
00783   // Find a table for the given shorthand.
00784   // If no shorthand is given, the first table is returned (if there).
00785   // If not found, a null Table object is returned.
00786   Table findTable (const String& shorthand) const;
00787 
00788   // Handle the selection of a wildcarded column name.
00789   void handleWildColumn (Int stringType, const String& name);
00790 
00791   // Add the description of a column to the table description.
00792   // ndim < 0 means a scalar column.
00793   void addColumnDesc (TableDesc& td, DataType dtype,
00794                       const String& colName, Int options,
00795                       Int ndim, const IPosition& shape,
00796                       const String& dmType, const String& dmGroup,
00797                       const String& comment,
00798                       const TableRecord& keywordSet,
00799                       const String& unitName);
00800 
00801   // Find the names of all stored columns in a table.
00802   Block<String> getStoredColumns (const Table& tab) const;
00803 
00804   // Try to find the keyword representing a table in one of the tables
00805   // in any select block (from inner to outer).
00806   // If not found, an exception is thrown.
00807   static Table tableKey (const String& fullName,
00808                          const String& shorthand, const String& columnName,
00809                          const Vector<String>& fieldNames,
00810                          const vector<TableParseSelect*>& stack);
00811 
00812   // Try to find the keyword representing a table in the given table.
00813   // If the columnName is empty, the keyword is a table keyword.
00814   // If not found, a null Table object is returned.
00815   static Table findTableKey (const Table& table, const String& columnName,
00816                              const Vector<String>& keyNames);
00817 
00818   // Check if the tables used in selection columns have the same
00819   // size as the first table given in FROM.
00820   void checkTableProjSizes() const;
00821 
00822   // Create the set of aggregate functions and groupby keys in case
00823   // a single groupby key is given.
00824   // This offers much faster map access then doGroupByAggrMultiple.
00825   template<typename T>
00826   vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrSingleKey
00827   (const vector<TableExprNodeRep*>& aggrNodes)
00828   {
00829     // We have to group the data according to the (possibly empty) groupby.
00830     // We step through the table in the normal order which may not be the
00831     // groupby order.
00832     // A map<key,int> is used to keep track of the results where the int
00833     // is the index in a vector of a set of aggregate function objects.
00834     vector<CountedPtr<TableExprGroupFuncSet> > funcSets;
00835     std::map<T, int> keyFuncMap;
00836     T lastKey = std::numeric_limits<T>::max();
00837     int groupnr = -1;
00838     // Loop through all rows.
00839     // For each row generate the key to get the right entry.
00840     TableExprId rowid(0);
00841     T key;
00842     for (uInt i=0; i<rownrs_p.size(); ++i) {
00843       rowid.setRownr (rownrs_p[i]);
00844       groupbyNodes_p[0].get (rowid, key);
00845       if (key != lastKey) {
00846         typename std::map<T, int>::iterator iter = keyFuncMap.find (key);
00847         if (iter == keyFuncMap.end()) {
00848           groupnr = funcSets.size();
00849           keyFuncMap[key] = groupnr;
00850           funcSets.push_back (new TableExprGroupFuncSet (aggrNodes));
00851         } else {
00852           groupnr = iter->second;
00853         }
00854       }
00855       rowid.setRownr (rownrs_p[i]);
00856       funcSets[groupnr]->apply (rowid);
00857     }
00858     return funcSets;
00859   }
00860 
00861   // Create the set of aggregate functions and groupby keys in case
00862   // multiple keys are given.
00863   vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrMultipleKeys
00864   (const vector<TableExprNodeRep*>& aggrNodes);
00865 
00866   //# Command type.
00867   CommandType commandType_p;
00868   //# Table description for a series of column descriptions.
00869   TableDesc tableDesc_p;
00870   //# Vector of TableParse objects.
00871   //# This is needed for the functions above, otherwise they have no
00872   //# way to communicate.
00873   vector<TableParse> fromTables_p;
00874   //# Block of selected column names (new name in case of select).
00875   Block<String> columnNames_p;
00876   //# Block of selected mask column names (for masked arrays).
00877   Block<String> columnNameMasks_p;
00878   //# Block of selected column expressions.
00879   Block<TableExprNode> columnExpr_p;
00880   //# The old name for a selected column.
00881   Block<String> columnOldNames_p;
00882   //# The new data type for a column.
00883   Block<String> columnDtypes_p;
00884   //# The keywords used in a column.
00885   Block<TableRecord> columnKeywords_p;
00886   //# Number of real expressions used in selected columns.
00887   uInt nrSelExprUsed_p;
00888   //# Distinct values in output?
00889   Bool distinct_p;
00890   //# Name and type of the resulting table (from GIVING part).
00891   String resultName_p;
00892   uInt   resultType_p;    //# 0-unknown 1=memory 2=scratch 3=plain
00893   Bool   resultCreated_p; //# Has the result table been created?
00894   StorageOption storageOption_p;
00895   Table::EndianFormat endianFormat_p;
00896   Bool overwrite_p;
00897   Record dminfo_p;
00898   //# Resulting set (from GIVING part).
00899   TableExprNodeSet* resultSet_p;
00900   //# The WHERE expression tree.
00901   TableExprNode node_p;
00902   //# The GROUPBY expressions.
00903   vector<TableExprNode> groupbyNodes_p;
00904   Bool groupbyRollup_p;   //# use ROLLUP in GROUPBY? 
00905   //# The HAVING expression.
00906   TableExprNode havingNode_p;
00907   //# The possible limit (= max nr of selected rows) (0 means no limit).
00908   Int64 limit_p;
00909   //# The possible last row (0 means no end; can be <0).
00910   //# limit_p and endrow_p cannot be both !=0.
00911   Int64 endrow_p;
00912   //# The possible offset (= nr of selected rows to skip).
00913   Int64 offset_p;
00914   //# The possible stride in offset:endrow:stride.
00915   Int64 stride_p;
00916   //# The update and insert list.
00917   std::vector<TableParseUpdate*> update_p;
00918   //# The insert expressions (possibly for multiple rows).
00919   std::vector<TableExprNode> insertExprs_p;
00920   //# The table selection to be inserted.
00921   TableParseSelect* insSel_p;
00922   //# The sort list.
00923   std::vector<TableParseSort> sort_p;
00924   //# The noDuplicates sort switch.
00925   Bool  noDupl_p;
00926   //# The default sort order.
00927   Sort::Order order_p;
00928   //# All nodes that need to be adjusted for a selection of rownrs.
00929   //# It can consist of column nodes and the rowid function node.
00930   //# Some nodes (in aggregate functions) can later be disabled for adjustment.
00931   vector<TableExprNode> applySelNodes_p;
00932   //# The resulting table.
00933   Table table_p;
00934   //# The first table used when creating a column object.
00935   //# All other tables used for them should have the same size.
00936   Table  firstColTable_p;
00937   String firstColName_p;
00938   //# The table resulting from a projection with expressions.
00939   Table projectExprTable_p;
00940   //# The projected columns used in the HAVING and ORDERBY clauses.
00941   Block<uInt>  projectExprSubset_p;
00942   Block<Bool>  projectExprSelColumn_p;
00943   //# The resulting row numbers.
00944   Vector<uInt> rownrs_p;
00945 };
00946 
00947 
00948 
00949 //# Implement the inline functions.
00950 inline Bool TableParse::test (const String& str) const
00951   { return (shorthand_p == str  ?  True : False); }
00952 
00953 inline const String& TableParse::shorthand() const
00954   { return shorthand_p; }
00955 
00956 inline const Table& TableParse::table() const
00957   { return table_p; }
00958 
00959 
00960 inline void TableParseUpdate::setColumnName (const String& name)
00961   { columnName_p = name; }
00962 inline void TableParseUpdate::setColumnNameMask (const String& name)
00963   { columnNameMask_p = name; }
00964 inline const String& TableParseUpdate::columnName() const
00965   { return columnName_p; }
00966 inline const String& TableParseUpdate::columnNameMask() const
00967   { return columnNameMask_p; }
00968 inline TableExprNodeIndex* TableParseUpdate::indexPtr() const
00969   { return indexPtr_p; }
00970 inline const TableExprNode& TableParseUpdate::indexNode() const
00971   { return indexNode_p; }
00972 inline const TableExprNode& TableParseUpdate::node() const
00973   { return node_p; }
00974 inline TableExprNode& TableParseUpdate::node()
00975   { return node_p; }
00976 inline void TableParseUpdate::adaptUnit (const Unit& columnUnit)
00977   { node_p.adaptUnit (columnUnit); }
00978 
00979 inline const TableExprNode& TableParseSort::node() const
00980   { return node_p; }
00981 inline Bool TableParseSort::orderGiven() const
00982   { return given_p; }
00983 inline Sort::Order TableParseSort::order() const
00984   { return order_p; }
00985 
00986 
00987 inline const Block<String>& TableParseSelect::getColumnNames() const
00988   { return columnNames_p; }
00989 
00990 inline const Table& TableParseSelect::getTable() const
00991   { return table_p; }
00992 
00993 inline void TableParseSelect::addUpdate (TableParseUpdate* upd)
00994   { update_p.push_back (upd); }
00995 
00996 inline Sort::Order TableParseSelect::getOrder (const TableParseSort& key) const
00997   { return (key.orderGiven()  ?  key.order() : order_p); }
00998 
00999 
01000 } //# NAMESPACE CASACORE - END
01001 
01002 #endif
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines

Generated on 31 Aug 2016 for casa by  doxygen 1.6.1