00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028 #ifndef TABLES_TABLEPARSE_H
00029 #define TABLES_TABLEPARSE_H
00030
00031
00032 #include <casacore/casa/aips.h>
00033 #include <casacore/tables/Tables/Table.h>
00034 #include <casacore/tables/Tables/TableDesc.h>
00035 #include <casacore/tables/TaQL/ExprNode.h>
00036 #include <casacore/tables/TaQL/TaQLResult.h>
00037 #include <casacore/tables/TaQL/ExprGroup.h>
00038 #include <casacore/casa/BasicSL/String.h>
00039 #include <casacore/casa/Utilities/Sort.h>
00040 #include <casacore/casa/Containers/Record.h>
00041 #include <casacore/casa/Containers/Block.h>
00042 #include <map>
00043 #include <vector>
00044 #include <limits>
00045
00046 namespace casacore {
00047
00048
00049 class TableExprNodeSet;
00050 class TableExprNodeSetElem;
00051 class TableExprNodeIndex;
00052 class TableColumn;
00053 class AipsIO;
00054 template<class T> class Vector;
00055
00056
00057
00058
00059
00060
00061
00062
00063
00064
00065
00066
00067
00068
00069
00070
00071
00072
00073
00074
00075
00076
00077
00078
00079
00080
00081
00082
00083
00084
00085
00086
00087
00088
00089
00090
00091
00092
00093
00094
00095
00096
00097
00098
00099
00100
00101
00102
00103
00104
00105
00106
00107
00108 class TableParse
00109 {
00110
00111 public:
00112
00113 TableParse();
00114
00115
00116 TableParse (const TableParse&);
00117
00118
00119 TableParse& operator= (const TableParse&);
00120
00121
00122 TableParse (const Table& table, const String& shorthand);
00123
00124
00125 Bool test (const String& shortHand) const;
00126
00127
00128 const String& shorthand() const;
00129
00130
00131 const Table& table() const;
00132
00133 private:
00134 String shorthand_p;
00135 Table table_p;
00136 };
00137
00138
00139
00140
00141
00142
00143
00144
00145
00146
00147
00148
00149
00150 TaQLResult tableCommand (const String& command);
00151
00152 TaQLResult tableCommand (const String& command,
00153 const Table& tempTable);
00154 TaQLResult tableCommand (const String& command,
00155 const std::vector<const Table*>& tempTables);
00156 TaQLResult tableCommand (const String& command,
00157 Vector<String>& columnNames);
00158 TaQLResult tableCommand (const String& command,
00159 Vector<String>& columnNames,
00160 String& commandType);
00161 TaQLResult tableCommand (const String& command,
00162 const std::vector<const Table*>& tempTables,
00163 Vector<String>& columnNames);
00164 TaQLResult tableCommand (const String& command,
00165 const std::vector<const Table*>& tempTables,
00166 Vector<String>& columnNames,
00167 String& commandType);
00168
00169
00170
00171
00172
00173
00174
00175
00176
00177
00178
00179
00180
00181
00182
00183
00184
00185
00186
00187
00188
00189
00190
00191
00192
00193
00194
00195
00196
00197
00198 class TableParseSort
00199 {
00200 public:
00201
00202
00203 TableParseSort();
00204
00205
00206
00207 explicit TableParseSort (const TableExprNode&);
00208
00209
00210 TableParseSort (const TableExprNode&, Sort::Order);
00211
00212 ~TableParseSort();
00213
00214
00215 const TableExprNode& node() const;
00216
00217
00218 Sort::Order order() const;
00219
00220
00221 Bool orderGiven() const;
00222
00223 private:
00224
00225
00226 void checkNode() const;
00227
00228 TableExprNode node_p;
00229 Sort::Order order_p;
00230 Bool given_p;
00231 };
00232
00233
00234
00235
00236
00237
00238
00239
00240
00241
00242
00243
00244
00245
00246
00247
00248
00249
00250
00251
00252
00253
00254
00255
00256
00257
00258
00259
00260
00261
00262 class TableParseUpdate
00263 {
00264 public:
00265 TableParseUpdate()
00266 : indexPtr_p(0) {}
00267
00268
00269
00270 TableParseUpdate (const String& columnName,
00271 const String& columnNameMask,
00272 const TableExprNode&,
00273 Bool checkAggr=True);
00274
00275
00276
00277 TableParseUpdate (const String& columnName,
00278 const String& columnNameMask,
00279 const TableExprNodeSet& indices,
00280 const TableExprNode&,
00281 const TaQLStyle&);
00282
00283
00284
00285
00286 TableParseUpdate (const String& columnName,
00287 const String& columnNameMask,
00288 const TableExprNodeSet& indices1,
00289 const TableExprNodeSet& indices2,
00290 const TableExprNode&,
00291 const TaQLStyle&);
00292
00293
00294 void handleIndices (const TableExprNodeSet& indices,
00295 const TaQLStyle& style);
00296 ~TableParseUpdate();
00297
00298
00299 void setColumnName (const String& name);
00300
00301
00302 void setColumnNameMask (const String& name);
00303
00304
00305 const String& columnName() const;
00306
00307
00308 const String& columnNameMask() const;
00309
00310
00311 Bool maskFirst() const
00312 { return maskFirst_p; }
00313
00314
00315 TableExprNodeIndex* indexPtr() const;
00316
00317
00318 const TableExprNode& indexNode() const;
00319
00320
00321
00322 const TableExprNode& node() const;
00323 TableExprNode& node();
00324
00325
00326
00327 const TableExprNode& mask() const
00328 { return mask_p; }
00329
00330
00331
00332 void adaptUnit (const Unit& columnUnit);
00333
00334 private:
00335 String columnName_p;
00336 String columnNameMask_p;
00337 Bool maskFirst_p;
00338 TableExprNodeIndex* indexPtr_p;
00339 TableExprNode indexNode_p;
00340 TableExprNode mask_p;
00341 TableExprNode node_p;
00342 };
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352
00353
00354
00355
00356
00357
00358
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369
00370
00371
00372
00373
00374
00375
00376
00377
00378
00379
00380 class TableParseSelect
00381 {
00382 public:
00383 enum CommandType {
00384 PSELECT,
00385 PUPDATE,
00386 PINSERT,
00387 PDELETE,
00388 PCOUNT,
00389 PCALC,
00390 PCRETAB,
00391 PALTTAB
00392 };
00393
00394 enum GroupAggrType {
00395 GROUPBY=1,
00396 AGGR_FUNCS=2,
00397 ONLY_COUNTALL=4
00398 };
00399
00400
00401 TableParseSelect (CommandType type);
00402
00403
00404 ~TableParseSelect();
00405
00406
00407 CommandType commandType() const
00408 { return commandType_p; }
00409
00410
00411 TableExprNode getNode() const
00412 { return node_p; }
00413
00414
00415 void makeTableNoFrom (const vector<TableParseSelect*>& stack);
00416
00417
00418
00419
00420
00421
00422
00423
00424 void execute (Bool showTimings, Bool setInGiving,
00425 Bool mustSelect, uInt maxRow, Bool doTracing=False);
00426
00427
00428 Table doFromQuery (Bool showTimings);
00429
00430
00431 TableExprNode doSubQuery (Bool showTimings);
00432
00433
00434
00435
00436 TableExprNode doExists (Bool noexists, Bool showTimings);
00437
00438
00439 void show (ostream& os) const;
00440
00441
00442 void handleWhere (const TableExprNode&);
00443
00444
00445
00446 void handleGroupby (const vector<TableExprNode>&, Bool rollup);
00447
00448
00449 void handleHaving (const TableExprNode&);
00450
00451
00452 void handleCalcComm (const TableExprNode&);
00453
00454
00455 void handleCreTab (const Record& dmInfo);
00456
00457
00458 void handleColSpec (const String& columnName, const String& dataType,
00459 const Record& spec, Bool isCOrder=False);
00460
00461
00462 void handleAltTab();
00463
00464
00465
00466 void handleAddCol (const Record& dmInfo);
00467
00468
00469
00470 ValueHolder getRecFld (const String& name);
00471
00472
00473 static void setRecFld (RecordInterface& rec,
00474 const String& name,
00475 const String& dtype,
00476 const ValueHolder& vh);
00477
00478
00479
00480 static String getTypeString (const String& typeStr, DataType type);
00481
00482
00483
00484
00485 void handleSetKey (const String& name, const String& dtype,
00486 const ValueHolder& value);
00487
00488
00489 void handleRenameKey (const String& oldName, const String& newName);
00490
00491
00492 void handleRemoveKey (const String& name);
00493
00494
00495
00496
00497 TableRecord& findKeyword (const String& name, String& keyName);
00498
00499
00500 void addUpdate (TableParseUpdate* upd);
00501
00502
00503 void setInsertExprs (const std::vector<TableExprNode> exprs)
00504 { insertExprs_p = exprs; }
00505
00506
00507 void handleUpdate();
00508
00509
00510
00511
00512
00513 void handleInsert();
00514 void handleInsert (TableParseSelect* sel);
00515
00516
00517
00518
00519 void handleCount();
00520
00521
00522 void handleSort (const std::vector<TableParseSort>& sortList,
00523 Bool noDuplicates, Sort::Order defaultSortOrder);
00524
00525
00526 void handleLimit (const TableExprNodeSetElem& expr);
00527
00528
00529 void handleLimit (const TableExprNode& expr);
00530
00531
00532 void handleOffset (const TableExprNode& expr);
00533
00534
00535 void handleAddRow (const TableExprNode& expr);
00536
00537
00538 void addTable (Int tabnr, const String& name,
00539 const Table& table,
00540 const String& shorthand,
00541 const vector<const Table*> tempTables,
00542 const vector<TableParseSelect*>& stack);
00543
00544
00545
00546
00547
00548 Table makeTable (Int tabnr, const String& name,
00549 const Table& ftab,
00550 const String& shorthand,
00551 const vector<const Table*> tempTables,
00552 const vector<TableParseSelect*>& stack,
00553 Bool alwaysOpen=True);
00554
00555
00556 void replaceTable (const Table& table);
00557
00558
00559
00560
00561 TableExprNode handleKeyCol (const String& name, Bool tryProj);
00562
00563
00564 static TableExprNode handleSlice (const TableExprNode& array,
00565 const TableExprNodeSet& indices,
00566 const TaQLStyle&);
00567
00568
00569 TableExprNode handleFunc (const String& name,
00570 const TableExprNodeSet& arguments,
00571 const TaQLStyle&);
00572
00573
00574
00575 static TableExprNode makeFuncNode (TableParseSelect*,
00576 const String& name,
00577 const TableExprNodeSet& arguments,
00578 const Vector<int>& ignoreFuncs,
00579 const Table& table,
00580 const TaQLStyle&);
00581
00582
00583 void handleColumn (Int type, const String& name, const TableExprNode& expr,
00584 const String& newName, const String& nameMask,
00585 const String& newDtype);
00586
00587
00588 void handleColumnFinish (Bool distinct);
00589
00590
00591 void setDMInfo (const Record& dminfo)
00592 { dminfo_p = dminfo;}
00593
00594
00595 void handleGiving (const String& name, const Record& type);
00596
00597
00598 void handleGiving (const TableExprNodeSet&);
00599
00600
00601 const Block<String>& getColumnNames() const;
00602
00603
00604 const Table& getTable() const;
00605
00606
00607 static void checkAggrFuncs (const TableExprNode& node);
00608
00609
00610
00611
00612
00613
00614
00615
00616
00617
00618
00619 static Bool splitName (String& shorthand, String& columnName,
00620 Vector<String>& fieldNames, const String& name,
00621 Bool checkError, Bool isKeyword, Bool allowNoKey);
00622
00623 private:
00624
00625
00626
00627
00628 Int testGroupAggr (vector<TableExprNodeRep*>& aggr) const;
00629
00630
00631 vector<TableExprNodeRep*> getAggrNodes() const;
00632
00633
00634 static TableExprNode makeUDFNode (TableParseSelect*,
00635 const String& name,
00636 const TableExprNodeSet& arguments,
00637 const Table& table,
00638 const TaQLStyle&);
00639
00640
00641
00642
00643 static TableExprFuncNode::FunctionType findFunc
00644 (const String& name,
00645 uInt narguments,
00646 const Vector<Int>& ignoreFuncs);
00647
00648
00649
00650
00651 void doUpdate (Bool showTimings, const Table& origTable,
00652 Table& updTable, const Vector<uInt>& rownrs,
00653 const CountedPtr<TableExprGroupResult>& groups =
00654 CountedPtr<TableExprGroupResult>());
00655
00656
00657 Table doInsert (Bool showTimings, Table& table);
00658
00659
00660 void doDelete (Bool showTimings, Table& table);
00661
00662
00663
00664 Table doCount (Bool showTimings, const Table&);
00665
00666
00667 Table doProject (Bool showTimings, const Table&,
00668 const CountedPtr<TableExprGroupResult>& groups =
00669 CountedPtr<TableExprGroupResult>());
00670
00671
00672
00673 Table doProjectExpr (Bool useSel,
00674 const CountedPtr<TableExprGroupResult>& groups);
00675
00676
00677
00678 Table createTable (const TableDesc& td,
00679 Int64 nrow, const Record& dmInfo);
00680
00681
00682 void makeProjectExprTable();
00683
00684
00685
00686 void makeProjectExprSel();
00687
00688
00689 void addApplySelNode (const TableExprNode& node)
00690 { applySelNodes_p.push_back (node); }
00691
00692
00693
00694
00695
00696
00697 Table adjustApplySelNodes (const Table&);
00698
00699
00700 CountedPtr<TableExprGroupResult> doGroupby
00701 (bool showTimings, vector<TableExprNodeRep*> aggrNodes,
00702 Int groupAggrUsed);
00703
00704
00705 void doHaving (Bool showTimings,
00706 const CountedPtr<TableExprGroupResult>& groups);
00707
00708
00709 CountedPtr<TableExprGroupResult> doOnlyCountAll (TableExprNodeRep* aggrNode);
00710
00711
00712 CountedPtr<TableExprGroupResult> doGroupByAggr
00713 (const vector<TableExprNodeRep*>& aggrNodes);
00714
00715
00716 void doSort (Bool showTimings);
00717
00718
00719 void doLimOff (Bool showTimings);
00720 Table doLimOff (Bool showTimings, const Table& table);
00721
00722
00723 Table doDistinct (Bool showTimings, const Table& table);
00724
00725
00726 Table doFinish (Bool showTimings, Table& table);
00727
00728
00729
00730 template<typename TCOL, typename TNODE>
00731 void updateValue (uInt row, const TableExprId& rowid,
00732 Bool isScalarCol, const TableExprNode& node,
00733 const Array<Bool>& mask, Bool maskFirst,
00734 TableColumn& col, const Slicer* slicerPtr,
00735 ArrayColumn<Bool>& maskCol);
00736 template<typename TCOL, typename TNODE>
00737 void updateScalar (uInt row, const TableExprId& rowid,
00738 const TableExprNode& node,
00739 TableColumn& col);
00740 template<typename TCOL, typename TNODE>
00741 void updateArray (uInt row, const TableExprId& rowid,
00742 const TableExprNode& node,
00743 const Array<TNODE>& res,
00744 ArrayColumn<TCOL>& col);
00745 template<typename TCOL, typename TNODE>
00746 void updateSlice (uInt row, const TableExprId& rowid,
00747 const TableExprNode& node,
00748 const Array<TNODE>& res,
00749 const Slicer& slice,
00750 ArrayColumn<TCOL>& col);
00751 template<typename TCOL, typename TNODE>
00752 void copyMaskedValue (uInt row, ArrayColumn<TCOL>& acol,
00753 const Slicer* slicerPtr,
00754 const TNODE* val,
00755 uInt incr, const Array<Bool>& mask);
00756 Array<Bool> makeMaskSlice (const Array<Bool>& mask,
00757 Bool maskFirst,
00758 const IPosition& shapeCol,
00759 const Slicer* slicerPtr);
00760 void checkMaskColumn (Bool hasMask,
00761 const ArrayColumn<Bool>& maskCol,
00762 const TableColumn& col);
00763
00764
00765
00766
00767 DataType makeDataType (DataType dtype, const String& dtstr,
00768 const String& colName);
00769
00770
00771
00772 Sort::Order getOrder (const TableParseSort& key) const;
00773
00774
00775 TableExprNode getColSet();
00776
00777
00778 TableExprNode makeSubSet() const;
00779
00780
00781 Int64 evalIntScaExpr (const TableExprNode& expr) const;
00782
00783
00784
00785
00786 Table findTable (const String& shorthand) const;
00787
00788
00789 void handleWildColumn (Int stringType, const String& name);
00790
00791
00792
00793 void addColumnDesc (TableDesc& td, DataType dtype,
00794 const String& colName, Int options,
00795 Int ndim, const IPosition& shape,
00796 const String& dmType, const String& dmGroup,
00797 const String& comment,
00798 const TableRecord& keywordSet,
00799 const String& unitName);
00800
00801
00802 Block<String> getStoredColumns (const Table& tab) const;
00803
00804
00805
00806
00807 static Table tableKey (const String& fullName,
00808 const String& shorthand, const String& columnName,
00809 const Vector<String>& fieldNames,
00810 const vector<TableParseSelect*>& stack);
00811
00812
00813
00814
00815 static Table findTableKey (const Table& table, const String& columnName,
00816 const Vector<String>& keyNames);
00817
00818
00819
00820 void checkTableProjSizes() const;
00821
00822
00823
00824
00825 template<typename T>
00826 vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrSingleKey
00827 (const vector<TableExprNodeRep*>& aggrNodes)
00828 {
00829
00830
00831
00832
00833
00834 vector<CountedPtr<TableExprGroupFuncSet> > funcSets;
00835 std::map<T, int> keyFuncMap;
00836 T lastKey = std::numeric_limits<T>::max();
00837 int groupnr = -1;
00838
00839
00840 TableExprId rowid(0);
00841 T key;
00842 for (uInt i=0; i<rownrs_p.size(); ++i) {
00843 rowid.setRownr (rownrs_p[i]);
00844 groupbyNodes_p[0].get (rowid, key);
00845 if (key != lastKey) {
00846 typename std::map<T, int>::iterator iter = keyFuncMap.find (key);
00847 if (iter == keyFuncMap.end()) {
00848 groupnr = funcSets.size();
00849 keyFuncMap[key] = groupnr;
00850 funcSets.push_back (new TableExprGroupFuncSet (aggrNodes));
00851 } else {
00852 groupnr = iter->second;
00853 }
00854 }
00855 rowid.setRownr (rownrs_p[i]);
00856 funcSets[groupnr]->apply (rowid);
00857 }
00858 return funcSets;
00859 }
00860
00861
00862
00863 vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrMultipleKeys
00864 (const vector<TableExprNodeRep*>& aggrNodes);
00865
00866
00867 CommandType commandType_p;
00868
00869 TableDesc tableDesc_p;
00870
00871
00872
00873 vector<TableParse> fromTables_p;
00874
00875 Block<String> columnNames_p;
00876
00877 Block<String> columnNameMasks_p;
00878
00879 Block<TableExprNode> columnExpr_p;
00880
00881 Block<String> columnOldNames_p;
00882
00883 Block<String> columnDtypes_p;
00884
00885 Block<TableRecord> columnKeywords_p;
00886
00887 uInt nrSelExprUsed_p;
00888
00889 Bool distinct_p;
00890
00891 String resultName_p;
00892 uInt resultType_p;
00893 Bool resultCreated_p;
00894 StorageOption storageOption_p;
00895 Table::EndianFormat endianFormat_p;
00896 Bool overwrite_p;
00897 Record dminfo_p;
00898
00899 TableExprNodeSet* resultSet_p;
00900
00901 TableExprNode node_p;
00902
00903 vector<TableExprNode> groupbyNodes_p;
00904 Bool groupbyRollup_p;
00905
00906 TableExprNode havingNode_p;
00907
00908 Int64 limit_p;
00909
00910
00911 Int64 endrow_p;
00912
00913 Int64 offset_p;
00914
00915 Int64 stride_p;
00916
00917 std::vector<TableParseUpdate*> update_p;
00918
00919 std::vector<TableExprNode> insertExprs_p;
00920
00921 TableParseSelect* insSel_p;
00922
00923 std::vector<TableParseSort> sort_p;
00924
00925 Bool noDupl_p;
00926
00927 Sort::Order order_p;
00928
00929
00930
00931 vector<TableExprNode> applySelNodes_p;
00932
00933 Table table_p;
00934
00935
00936 Table firstColTable_p;
00937 String firstColName_p;
00938
00939 Table projectExprTable_p;
00940
00941 Block<uInt> projectExprSubset_p;
00942 Block<Bool> projectExprSelColumn_p;
00943
00944 Vector<uInt> rownrs_p;
00945 };
00946
00947
00948
00949
00950 inline Bool TableParse::test (const String& str) const
00951 { return (shorthand_p == str ? True : False); }
00952
00953 inline const String& TableParse::shorthand() const
00954 { return shorthand_p; }
00955
00956 inline const Table& TableParse::table() const
00957 { return table_p; }
00958
00959
00960 inline void TableParseUpdate::setColumnName (const String& name)
00961 { columnName_p = name; }
00962 inline void TableParseUpdate::setColumnNameMask (const String& name)
00963 { columnNameMask_p = name; }
00964 inline const String& TableParseUpdate::columnName() const
00965 { return columnName_p; }
00966 inline const String& TableParseUpdate::columnNameMask() const
00967 { return columnNameMask_p; }
00968 inline TableExprNodeIndex* TableParseUpdate::indexPtr() const
00969 { return indexPtr_p; }
00970 inline const TableExprNode& TableParseUpdate::indexNode() const
00971 { return indexNode_p; }
00972 inline const TableExprNode& TableParseUpdate::node() const
00973 { return node_p; }
00974 inline TableExprNode& TableParseUpdate::node()
00975 { return node_p; }
00976 inline void TableParseUpdate::adaptUnit (const Unit& columnUnit)
00977 { node_p.adaptUnit (columnUnit); }
00978
00979 inline const TableExprNode& TableParseSort::node() const
00980 { return node_p; }
00981 inline Bool TableParseSort::orderGiven() const
00982 { return given_p; }
00983 inline Sort::Order TableParseSort::order() const
00984 { return order_p; }
00985
00986
00987 inline const Block<String>& TableParseSelect::getColumnNames() const
00988 { return columnNames_p; }
00989
00990 inline const Table& TableParseSelect::getTable() const
00991 { return table_p; }
00992
00993 inline void TableParseSelect::addUpdate (TableParseUpdate* upd)
00994 { update_p.push_back (upd); }
00995
00996 inline Sort::Order TableParseSelect::getOrder (const TableParseSort& key) const
00997 { return (key.orderGiven() ? key.order() : order_p); }
00998
00999
01000 }
01001
01002 #endif