001/* 002// This software is subject to the terms of the Eclipse Public License v1.0 003// Agreement, available at the following URL: 004// http://www.eclipse.org/legal/epl-v10.html. 005// You must accept the terms of that agreement to use this software. 006// 007// Copyright (C) 2002-2005 Julian Hyde 008// Copyright (C) 2005-2013 Pentaho and others 009// All Rights Reserved. 010*/ 011package mondrian.rolap.agg; 012 013import mondrian.olap.MondrianException; 014import mondrian.olap.MondrianProperties; 015import mondrian.olap.Util; 016import mondrian.resource.MondrianResource; 017import mondrian.rolap.*; 018import mondrian.rolap.agg.SegmentCacheManager.AbortException; 019import mondrian.rolap.cache.SegmentCacheIndex; 020import mondrian.server.Locus; 021import mondrian.server.monitor.SqlStatementEvent; 022import mondrian.spi.*; 023import mondrian.util.*; 024 025import org.apache.log4j.Logger; 026 027import java.io.Serializable; 028import java.sql.ResultSet; 029import java.sql.SQLException; 030import java.sql.Statement; 031import java.util.*; 032import java.util.concurrent.*; 033 034/** 035 * <p>The <code>SegmentLoader</code> queries database and loads the data into 036 * the given set of segments.</p> 037 * 038 * <p>It reads a segment of <code>measure</code>, where <code>columns</code> 039 * are constrained to <code>values</code>. Each entry in <code>values</code> 040 * can be null, meaning don't constrain, or can have several values. For 041 * example, <code>getSegment({Unit_sales}, {Region, State, Year}, {"West"}, 042 * {"CA", "OR", "WA"}, null})</code> returns sales in states CA, OR and WA 043 * in the Western region, for all years.</p> 044 * 045 * <p>It will also look at the {@link MondrianProperties#SegmentCache} property 046 * and make usage of the SegmentCache provided as an SPI. 047 * 048 * @author Thiyagu, LBoudreau 049 * @since 24 May 2007 050 */ 051public class SegmentLoader { 052 053 private static final Logger LOGGER = Logger.getLogger(SegmentLoader.class); 054 055 private final SegmentCacheManager cacheMgr; 056 057 /** 058 * Creates a SegmentLoader. 059 * 060 * @param cacheMgr Cache manager 061 */ 062 public SegmentLoader(SegmentCacheManager cacheMgr) { 063 this.cacheMgr = cacheMgr; 064 } 065 066 /** 067 * Loads data for all the segments of the GroupingSets. If the grouping sets 068 * list contains more than one Grouping Set then data is loaded using the 069 * GROUP BY GROUPING SETS sql. Else if only one grouping set is passed in 070 * the list data is loaded without using GROUP BY GROUPING SETS sql. If the 071 * database does not support grouping sets 072 * {@link mondrian.spi.Dialect#supportsGroupingSets()} then 073 * grouping sets list should always have only one element in it. 074 * 075 * <p>For example, if list has 2 grouping sets with columns A, B, C and B, C 076 * respectively, then the SQL will be 077 * "GROUP BY GROUPING SETS ((A, B, C), (B, C))". 078 * 079 * <p>Else if the list has only one grouping set then sql would be without 080 * grouping sets. 081 * 082 * <p>The <code>groupingSets</code> list should be topological order, with 083 * more detailed higher-level grouping sets occurring first. In other words, 084 * the first element of the list should always be the detailed grouping 085 * set (default grouping set), followed by grouping sets which can be 086 * rolled-up on this detailed grouping set. 087 * In the example (A, B, C) is the detailed grouping set and (B, C) is 088 * rolled-up using the detailed. 089 * 090 * <p>Grouping sets are removed from the {@code groupingSets} list as they 091 * are loaded.</p> 092 * 093 * @param cellRequestCount Number of missed cells that led to this request 094 * @param groupingSets List of grouping sets whose segments are loaded 095 * @param compoundPredicateList Compound predicates 096 * @param segmentFutures List of futures wherein each statement will place 097 * a list of the segments it has loaded, when it 098 * completes 099 */ 100 public void load( 101 int cellRequestCount, 102 List<GroupingSet> groupingSets, 103 List<StarPredicate> compoundPredicateList, 104 List<Future<Map<Segment, SegmentWithData>>> segmentFutures) 105 { 106 if (!MondrianProperties.instance().DisableCaching.get()) { 107 for (GroupingSet groupingSet : groupingSets) { 108 for (Segment segment : groupingSet.getSegments()) { 109 final SegmentCacheIndex index = 110 cacheMgr.getIndexRegistry().getIndex(segment.star); 111 index.add( 112 segment.getHeader(), 113 true, 114 new SegmentBuilder.StarSegmentConverter( 115 segment.measure, 116 compoundPredicateList)); 117 // Make sure that we are registered as a client of 118 // the segment by invoking getFuture. 119 Util.discard( 120 index.getFuture( 121 Locus.peek().execution, 122 segment.getHeader())); 123 } 124 } 125 } 126 try { 127 segmentFutures.add( 128 cacheMgr.sqlExecutor.submit( 129 new SegmentLoadCommand( 130 Locus.peek(), 131 this, 132 cellRequestCount, 133 groupingSets, 134 compoundPredicateList))); 135 } catch (Exception e) { 136 throw new MondrianException(e); 137 } 138 } 139 140 private static class SegmentLoadCommand 141 implements Callable<Map<Segment, SegmentWithData>> 142 { 143 private final Locus locus; 144 private final SegmentLoader segmentLoader; 145 private final int cellRequestCount; 146 private final List<GroupingSet> groupingSets; 147 private final List<StarPredicate> compoundPredicateList; 148 149 public SegmentLoadCommand( 150 Locus locus, 151 SegmentLoader segmentLoader, 152 int cellRequestCount, 153 List<GroupingSet> groupingSets, 154 List<StarPredicate> compoundPredicateList) 155 { 156 this.locus = locus; 157 this.segmentLoader = segmentLoader; 158 this.cellRequestCount = cellRequestCount; 159 this.groupingSets = groupingSets; 160 this.compoundPredicateList = compoundPredicateList; 161 } 162 163 public Map<Segment, SegmentWithData> call() throws Exception { 164 Locus.push(locus); 165 try { 166 return segmentLoader.loadImpl( 167 cellRequestCount, 168 groupingSets, 169 compoundPredicateList); 170 } finally { 171 Locus.pop(locus); 172 } 173 } 174 } 175 176 private Map<Segment, SegmentWithData> loadImpl( 177 int cellRequestCount, 178 List<GroupingSet> groupingSets, 179 List<StarPredicate> compoundPredicateList) 180 { 181 SqlStatement stmt = null; 182 GroupingSetsList groupingSetsList = 183 new GroupingSetsList(groupingSets); 184 RolapStar.Column[] defaultColumns = 185 groupingSetsList.getDefaultColumns(); 186 187 final Map<Segment, SegmentWithData> segmentMap = 188 new HashMap<Segment, SegmentWithData>(); 189 Throwable throwable = null; 190 try { 191 int arity = defaultColumns.length; 192 SortedSet<Comparable>[] axisValueSets = 193 getDistinctValueWorkspace(arity); 194 195 stmt = createExecuteSql( 196 cellRequestCount, 197 groupingSetsList, 198 compoundPredicateList); 199 200 if (stmt == null) { 201 // Nothing to do. We're done here. 202 return segmentMap; 203 } 204 205 boolean[] axisContainsNull = new boolean[arity]; 206 207 RowList rows = 208 processData( 209 stmt, 210 axisContainsNull, 211 axisValueSets, 212 groupingSetsList); 213 214 boolean sparse = 215 setAxisDataAndDecideSparseUse( 216 axisValueSets, 217 axisContainsNull, 218 groupingSetsList, 219 rows); 220 221 final Map<BitKey, GroupingSetsList.Cohort> groupingDataSetsMap = 222 createDataSetsForGroupingSets( 223 groupingSetsList, 224 sparse, 225 rows.getTypes().subList( 226 arity, rows.getTypes().size())); 227 228 loadDataToDataSets( 229 groupingSetsList, rows, groupingDataSetsMap); 230 231 setDataToSegments( 232 groupingSetsList, 233 groupingDataSetsMap, 234 segmentMap); 235 236 return segmentMap; 237 } catch (Throwable e) { 238 throwable = e; 239 if (stmt == null) { 240 throw new MondrianException(e); 241 } 242 throw stmt.handle(e); 243 } finally { 244 if (stmt != null) { 245 stmt.close(); 246 } 247 setFailOnStillLoadingSegments( 248 segmentMap, groupingSetsList, throwable); 249 } 250 } 251 252 /** 253 * Called when a segment has been loaded from SQL, to put into the segment 254 * index and the external cache. 255 * 256 * @param header Segment header 257 * @param body Segment body 258 */ 259 private void cacheSegment( 260 RolapStar star, 261 SegmentHeader header, 262 SegmentBody body) 263 { 264 // Write the segment into external cache. 265 // 266 // It would be a mistake to do this from the cacheMgr -- because the 267 // calls may take time. The cacheMgr's actions must all be quick. We 268 // are a worker, so we have plenty of time. 269 // 270 // Also note that we push the segments to external cache after we have 271 // called cacheMgr.loadSucceeded. That call will allow the current 272 // query to proceed. 273 if (!MondrianProperties.instance().DisableCaching.get()) { 274 cacheMgr.compositeCache.put(header, body); 275 cacheMgr.loadSucceeded(star, header, body); 276 } 277 } 278 279 private boolean setFailOnStillLoadingSegments( 280 Map<Segment, SegmentWithData> segmentMap, 281 GroupingSetsList groupingSetsList, 282 Throwable throwable) 283 { 284 int n = 0; 285 for (GroupingSet groupingSet : groupingSetsList.getGroupingSets()) { 286 for (Segment segment : groupingSet.getSegments()) { 287 if (!segmentMap.containsKey(segment)) { 288 if (throwable == null) { 289 throwable = 290 new RuntimeException("Segment failed to load"); 291 } 292 final SegmentHeader header = segment.getHeader(); 293 cacheMgr.loadFailed( 294 segment.star, 295 header, 296 throwable); 297 ++n; 298 } 299 } 300 } 301 return n > 0; 302 } 303 304 /** 305 * Loads data to the datasets. If the grouping sets is used, 306 * dataset is fetched from groupingDataSetMap using grouping bit keys of 307 * the row data. If grouping sets is not used, data is loaded on to 308 * nonGroupingDataSets. 309 */ 310 private void loadDataToDataSets( 311 GroupingSetsList groupingSetsList, 312 RowList rows, 313 Map<BitKey, GroupingSetsList.Cohort> groupingDataSetMap) 314 { 315 int arity = groupingSetsList.getDefaultColumns().length; 316 SegmentAxis[] axes = groupingSetsList.getDefaultAxes(); 317 int segmentLength = groupingSetsList.getDefaultSegments().size(); 318 319 final List<SqlStatement.Type> types = rows.getTypes(); 320 final boolean useGroupingSet = groupingSetsList.useGroupingSets(); 321 for (rows.first(); rows.next();) { 322 final BitKey groupingBitKey; 323 final GroupingSetsList.Cohort cohort; 324 if (useGroupingSet) { 325 groupingBitKey = 326 (BitKey) rows.getObject( 327 groupingSetsList.getGroupingBitKeyIndex()); 328 cohort = groupingDataSetMap.get(groupingBitKey); 329 } else { 330 groupingBitKey = null; 331 cohort = groupingDataSetMap.get(BitKey.EMPTY); 332 } 333 final int[] pos = cohort.pos; 334 for (int j = 0, k = 0; j < arity; j++) { 335 final SqlStatement.Type type = types.get(j); 336 switch (type) { 337 // TODO: different treatment for INT, LONG, DOUBLE 338 case OBJECT: 339 case STRING: 340 case INT: 341 case LONG: 342 case DOUBLE: 343 Object o = rows.getObject(j); 344 if (useGroupingSet 345 && (o == null || o == RolapUtil.sqlNullValue) 346 && groupingBitKey.get( 347 groupingSetsList.findGroupingFunctionIndex(j))) 348 { 349 continue; 350 } 351 SegmentAxis axis = axes[j]; 352 if (o == null) { 353 o = RolapUtil.sqlNullValue; 354 } 355 // Note: We believe that all value types are Comparable. 356 // In JDK 1.4, Boolean did not implement Comparable, but 357 // that's too minor/long ago to worry about. 358 int offset = axis.getOffset((Comparable) o); 359 pos[k++] = offset; 360 break; 361 default: 362 throw Util.unexpected(type); 363 } 364 } 365 366 for (int j = 0; j < segmentLength; j++) { 367 cohort.segmentDatasetList.get(j).populateFrom( 368 pos, rows, arity + j); 369 } 370 } 371 } 372 373 private boolean setAxisDataAndDecideSparseUse( 374 SortedSet<Comparable>[] axisValueSets, 375 boolean[] axisContainsNull, 376 GroupingSetsList groupingSetsList, 377 RowList rows) 378 { 379 SegmentAxis[] axes = groupingSetsList.getDefaultAxes(); 380 RolapStar.Column[] allColumns = groupingSetsList.getDefaultColumns(); 381 // Figure out size of dense array, and allocate it, or use a sparse 382 // array if appropriate. 383 boolean sparse = false; 384 int n = 1; 385 for (int i = 0; i < axes.length; i++) { 386 SortedSet<Comparable> valueSet = axisValueSets[i]; 387 axes[i] = 388 new SegmentAxis( 389 groupingSetsList.getDefaultPredicates()[i], 390 valueSet, 391 axisContainsNull[i]); 392 int size = axes[i].getKeys().length; 393 setAxisDataToGroupableList( 394 groupingSetsList, 395 valueSet, 396 axisContainsNull[i], 397 allColumns[i]); 398 int previous = n; 399 n *= size; 400 if ((n < previous) || (n < size)) { 401 // Overflow has occurred. 402 n = Integer.MAX_VALUE; 403 sparse = true; 404 } 405 } 406 return useSparse(sparse, n, rows); 407 } 408 409 boolean useSparse(boolean sparse, int n, RowList rows) { 410 sparse = sparse || useSparse((double) n, (double) rows.size()); 411 return sparse; 412 } 413 414 private void setDataToSegments( 415 GroupingSetsList groupingSetsList, 416 Map<BitKey, GroupingSetsList.Cohort> datasetsMap, 417 Map<Segment, SegmentWithData> segmentSlotMap) 418 { 419 List<GroupingSet> groupingSets = groupingSetsList.getGroupingSets(); 420 for (int i = 0; i < groupingSets.size(); i++) { 421 List<Segment> segments = groupingSets.get(i).getSegments(); 422 GroupingSetsList.Cohort cohort = 423 datasetsMap.get( 424 groupingSetsList.getRollupColumnsBitKeyList().get(i)); 425 for (int j = 0; j < segments.size(); j++) { 426 Segment segment = segments.get(j); 427 final SegmentDataset segmentDataset = 428 cohort.segmentDatasetList.get(j); 429 final SegmentWithData segmentWithData = 430 new SegmentWithData( 431 segment, 432 segmentDataset, 433 cohort.axes); 434 435 segmentSlotMap.put(segment, segmentWithData); 436 437 final SegmentHeader header = segmentWithData.getHeader(); 438 final SegmentBody body = 439 segmentWithData.getData().createSegmentBody( 440 new AbstractList< 441 Pair<SortedSet<Comparable>, Boolean>>() 442 { 443 public Pair<SortedSet<Comparable>, Boolean> get( 444 int index) 445 { 446 return segmentWithData.axes[index] 447 .getValuesAndIndicator(); 448 } 449 450 public int size() { 451 return segmentWithData.axes.length; 452 } 453 }); 454 455 // Send a message to the agg manager. It will place the segment 456 // in the index. 457 cacheSegment(segment.star, header, body); 458 } 459 } 460 } 461 462 private Map<BitKey, GroupingSetsList.Cohort> createDataSetsForGroupingSets( 463 GroupingSetsList groupingSetsList, 464 boolean sparse, 465 List<SqlStatement.Type> types) 466 { 467 if (!groupingSetsList.useGroupingSets()) { 468 final GroupingSetsList.Cohort datasets = createDataSets( 469 sparse, 470 groupingSetsList.getDefaultSegments(), 471 groupingSetsList.getDefaultAxes(), 472 types); 473 return Collections.singletonMap(BitKey.EMPTY, datasets); 474 } 475 Map<BitKey, GroupingSetsList.Cohort> datasetsMap = 476 new HashMap<BitKey, GroupingSetsList.Cohort>(); 477 List<GroupingSet> groupingSets = groupingSetsList.getGroupingSets(); 478 List<BitKey> groupingColumnsBitKeyList = 479 groupingSetsList.getRollupColumnsBitKeyList(); 480 for (int i = 0; i < groupingSets.size(); i++) { 481 GroupingSet groupingSet = groupingSets.get(i); 482 GroupingSetsList.Cohort cohort = 483 createDataSets( 484 sparse, 485 groupingSet.getSegments(), 486 groupingSet.getAxes(), 487 types); 488 datasetsMap.put(groupingColumnsBitKeyList.get(i), cohort); 489 } 490 return datasetsMap; 491 } 492 493 private int calculateMaxDataSize(SegmentAxis[] axes) { 494 int n = 1; 495 for (SegmentAxis axis : axes) { 496 n *= axis.getKeys().length; 497 } 498 return n; 499 } 500 501 private GroupingSetsList.Cohort createDataSets( 502 boolean sparse, 503 List<Segment> segments, 504 SegmentAxis[] axes, 505 List<SqlStatement.Type> types) 506 { 507 final List<SegmentDataset> datasets = 508 new ArrayList<SegmentDataset>(segments.size()); 509 final int n; 510 if (sparse) { 511 n = 0; 512 } else { 513 n = calculateMaxDataSize(axes); 514 } 515 for (int i = 0; i < segments.size(); i++) { 516 final Segment segment = segments.get(i); 517 datasets.add(segment.createDataset(axes, sparse, types.get(i), n)); 518 } 519 return new GroupingSetsList.Cohort(datasets, axes); 520 } 521 522 private void setAxisDataToGroupableList( 523 GroupingSetsList groupingSetsList, 524 SortedSet<Comparable> valueSet, 525 boolean axisContainsNull, 526 RolapStar.Column column) 527 { 528 for (GroupingSet groupingSet 529 : groupingSetsList.getRollupGroupingSets()) 530 { 531 RolapStar.Column[] columns = groupingSet.getColumns(); 532 for (int i = 0; i < columns.length; i++) { 533 if (columns[i].equals(column)) { 534 groupingSet.getAxes()[i] = 535 new SegmentAxis( 536 groupingSet.getPredicates()[i], 537 valueSet, 538 axisContainsNull); 539 } 540 } 541 } 542 } 543 544 /** 545 * Creates and executes a SQL statement to retrieve the set of cells 546 * specified by a GroupingSetsList. 547 * 548 * <p>This method may be overridden in tests. 549 * 550 * @param cellRequestCount Number of missed cells that led to this request 551 * @param groupingSetsList Grouping 552 * @param compoundPredicateList Compound predicate list 553 * @return An executed SQL statement, or null 554 */ 555 SqlStatement createExecuteSql( 556 int cellRequestCount, 557 final GroupingSetsList groupingSetsList, 558 List<StarPredicate> compoundPredicateList) 559 { 560 RolapStar star = groupingSetsList.getStar(); 561 Pair<String, List<SqlStatement.Type>> pair = 562 AggregationManager.generateSql( 563 groupingSetsList, compoundPredicateList); 564 final Locus locus = 565 new SqlStatement.StatementLocus( 566 Locus.peek().execution, 567 "Segment.load", 568 "Error while loading segment", 569 SqlStatementEvent.Purpose.CELL_SEGMENT, 570 cellRequestCount); 571 572 // When caching is enabled, we must register the SQL statement 573 // in the index. We don't want to cancel SQL statements that are shared 574 // across threads unless it is safe. 575 final Util.Functor1<Void, Statement> callbackWithCaching = 576 new Util.Functor1<Void, Statement>() { 577 public Void apply(final Statement stmt) { 578 cacheMgr.execute( 579 new SegmentCacheManager.Command<Void>() { 580 public Void call() throws Exception { 581 boolean atLeastOneActive = false; 582 for (Segment seg 583 : groupingSetsList.getDefaultSegments()) 584 { 585 final SegmentCacheIndex index = 586 cacheMgr.getIndexRegistry() 587 .getIndex(seg.star); 588 // Make sure to check if the segment still 589 // exists in the index. It could have been 590 // removed by a cancellation request since 591 // then. 592 if (index.contains(seg.getHeader())) { 593 index.linkSqlStatement( 594 seg.getHeader(), stmt); 595 atLeastOneActive = true; 596 } 597 if (!atLeastOneActive) { 598 // There are no segments to load. 599 // Throw this so that the segment thread 600 // knows to stop. 601 throw new AbortException(); 602 } 603 } 604 return null; 605 } 606 public Locus getLocus() { 607 return locus; 608 } 609 }); 610 return null; 611 } 612 }; 613 614 // When using no cache, we register the SQL statement directly 615 // with the execution instance for cleanup. 616 final Util.Functor1<Void, Statement> callbackNoCaching = 617 new Util.Functor1<Void, Statement>() { 618 public Void apply(final Statement stmt) { 619 locus.execution.registerStatement(locus, stmt); 620 return null; 621 } 622 }; 623 624 try { 625 return RolapUtil.executeQuery( 626 star.getDataSource(), 627 pair.left, 628 pair.right, 629 0, 630 0, 631 locus, 632 -1, 633 -1, 634 // Only one of the two callbacks are required, depending if we 635 // cache the segments or not. 636 MondrianProperties.instance().DisableCaching.get() 637 ? callbackNoCaching 638 : callbackWithCaching); 639 } catch (Throwable t) { 640 if (Util.getMatchingCause(t, AbortException.class) != null) { 641 return null; 642 } else { 643 throw new MondrianException( 644 "Failed to load segment form SQL", 645 t); 646 } 647 } 648 } 649 650 RowList processData( 651 SqlStatement stmt, 652 final boolean[] axisContainsNull, 653 final SortedSet<Comparable>[] axisValueSets, 654 final GroupingSetsList groupingSetsList) throws SQLException 655 { 656 List<Segment> segments = groupingSetsList.getDefaultSegments(); 657 int measureCount = segments.size(); 658 ResultSet rawRows = loadData(stmt, groupingSetsList); 659 assert stmt != null; 660 final List<SqlStatement.Type> types = stmt.guessTypes(); 661 int arity = axisValueSets.length; 662 final int groupingColumnStartIndex = arity + measureCount; 663 664 // If we're using grouping sets, the SQL query will have a number of 665 // indicator columns, and we roll these into a single BitSet column in 666 // the processed data set. 667 final List<SqlStatement.Type> processedTypes; 668 if (groupingSetsList.useGroupingSets()) { 669 processedTypes = 670 new ArrayList<SqlStatement.Type>( 671 types.subList(0, groupingColumnStartIndex)); 672 processedTypes.add(SqlStatement.Type.OBJECT); 673 } else { 674 processedTypes = types; 675 } 676 final RowList processedRows = new RowList(processedTypes, 100); 677 678 while (rawRows.next()) { 679 checkResultLimit(++stmt.rowCount); 680 processedRows.createRow(); 681 682 // get the columns 683 int columnIndex = 0; 684 for (int axisIndex = 0; axisIndex < arity; 685 axisIndex++, columnIndex++) 686 { 687 final SqlStatement.Type type = types.get(columnIndex); 688 switch (type) { 689 case OBJECT: 690 case STRING: 691 Object o = rawRows.getObject(columnIndex + 1); 692 if (o == null) { 693 o = RolapUtil.sqlNullValue; 694 if (!groupingSetsList.useGroupingSets() 695 || !isAggregateNull( 696 rawRows, groupingColumnStartIndex, 697 groupingSetsList, 698 axisIndex)) 699 { 700 axisContainsNull[axisIndex] = true; 701 } 702 } else { 703 // We assume that all values are Comparable. Boolean 704 // wasn't Comparable until JDK 1.5, but we can live with 705 // that bug because JDK 1.4 is no longer important. 706 axisValueSets[axisIndex].add((Comparable) o); 707 } 708 processedRows.setObject(columnIndex, o); 709 break; 710 case INT: 711 final int intValue = rawRows.getInt(columnIndex + 1); 712 if (intValue == 0 && rawRows.wasNull()) { 713 if (!groupingSetsList.useGroupingSets() 714 || !isAggregateNull( 715 rawRows, groupingColumnStartIndex, 716 groupingSetsList, 717 axisIndex)) 718 { 719 axisContainsNull[axisIndex] = true; 720 } 721 processedRows.setNull(columnIndex, true); 722 } else { 723 axisValueSets[axisIndex].add(intValue); 724 processedRows.setInt(columnIndex, intValue); 725 } 726 break; 727 case LONG: 728 final long longValue = rawRows.getLong(columnIndex + 1); 729 if (longValue == 0 && rawRows.wasNull()) { 730 if (!groupingSetsList.useGroupingSets() 731 || !isAggregateNull( 732 rawRows, 733 groupingColumnStartIndex, 734 groupingSetsList, 735 axisIndex)) 736 { 737 axisContainsNull[axisIndex] = true; 738 } 739 processedRows.setNull(columnIndex, true); 740 } else { 741 axisValueSets[axisIndex].add(longValue); 742 processedRows.setLong(columnIndex, longValue); 743 } 744 break; 745 case DOUBLE: 746 final double doubleValue = 747 rawRows.getDouble(columnIndex + 1); 748 if (doubleValue == 0 && rawRows.wasNull()) { 749 if (!groupingSetsList.useGroupingSets() 750 || !isAggregateNull( 751 rawRows, groupingColumnStartIndex, 752 groupingSetsList, 753 axisIndex)) 754 { 755 axisContainsNull[axisIndex] = true; 756 } 757 } 758 axisValueSets[axisIndex].add(doubleValue); 759 processedRows.setDouble(columnIndex, doubleValue); 760 break; 761 default: 762 throw Util.unexpected(type); 763 } 764 } 765 766 // pre-compute which measures are numeric 767 final boolean[] numeric = new boolean[measureCount]; 768 int k = 0; 769 for (Segment segment : segments) { 770 numeric[k++] = segment.measure.getDatatype().isNumeric(); 771 } 772 773 // get the measure 774 for (int i = 0; i < measureCount; i++, columnIndex++) { 775 final SqlStatement.Type type = 776 types.get(columnIndex); 777 switch (type) { 778 case OBJECT: 779 case STRING: 780 Object o = rawRows.getObject(columnIndex + 1); 781 if (o == null) { 782 o = Util.nullValue; // convert to placeholder 783 } else if (numeric[i]) { 784 if (o instanceof Double) { 785 // nothing to do 786 } else if (o instanceof Number) { 787 o = ((Number) o).doubleValue(); 788 } else if (o instanceof byte[]) { 789 // On MySQL 5.0 in German locale, values can come 790 // out as byte arrays. Don't know why. Bug 1594119. 791 o = Double.parseDouble(new String((byte[]) o)); 792 } else { 793 o = Double.parseDouble(o.toString()); 794 } 795 } 796 processedRows.setObject(columnIndex, o); 797 break; 798 case INT: 799 final int intValue = rawRows.getInt(columnIndex + 1); 800 processedRows.setInt(columnIndex, intValue); 801 if (intValue == 0 && rawRows.wasNull()) { 802 processedRows.setNull(columnIndex, true); 803 } 804 break; 805 case LONG: 806 final long longValue = rawRows.getLong(columnIndex + 1); 807 processedRows.setLong(columnIndex, longValue); 808 if (longValue == 0 && rawRows.wasNull()) { 809 processedRows.setNull(columnIndex, true); 810 } 811 break; 812 case DOUBLE: 813 final double doubleValue = 814 rawRows.getDouble(columnIndex + 1); 815 processedRows.setDouble(columnIndex, doubleValue); 816 if (doubleValue == 0 && rawRows.wasNull()) { 817 processedRows.setNull(columnIndex, true); 818 } 819 break; 820 default: 821 throw Util.unexpected(type); 822 } 823 } 824 825 if (groupingSetsList.useGroupingSets()) { 826 processedRows.setObject( 827 columnIndex, 828 getRollupBitKey( 829 groupingSetsList.getRollupColumns().size(), 830 rawRows, columnIndex)); 831 } 832 } 833 return processedRows; 834 } 835 836 private void checkResultLimit(int currentCount) { 837 final int limit = 838 MondrianProperties.instance().ResultLimit.get(); 839 if (limit > 0 && currentCount > limit) { 840 throw MondrianResource.instance() 841 .SegmentFetchLimitExceeded.ex(limit); 842 } 843 } 844 845 /** 846 * Generates bit key representing roll up columns 847 */ 848 BitKey getRollupBitKey(int arity, ResultSet rowList, int k) 849 throws SQLException 850 { 851 BitKey groupingBitKey = BitKey.Factory.makeBitKey(arity); 852 for (int i = 0; i < arity; i++) { 853 int o = rowList.getInt(k + i + 1); 854 if (o == 1) { 855 groupingBitKey.set(i); 856 } 857 } 858 return groupingBitKey; 859 } 860 861 private boolean isAggregateNull( 862 ResultSet rowList, 863 int groupingColumnStartIndex, 864 GroupingSetsList groupingSetsList, 865 int axisIndex) throws SQLException 866 { 867 int groupingFunctionIndex = 868 groupingSetsList.findGroupingFunctionIndex(axisIndex); 869 if (groupingFunctionIndex == -1) { 870 // Not a rollup column 871 return false; 872 } 873 return rowList.getInt( 874 groupingColumnStartIndex + groupingFunctionIndex + 1) == 1; 875 } 876 877 ResultSet loadData( 878 SqlStatement stmt, 879 GroupingSetsList groupingSetsList) 880 throws SQLException 881 { 882 int arity = groupingSetsList.getDefaultColumns().length; 883 int measureCount = groupingSetsList.getDefaultSegments().size(); 884 int groupingFunctionsCount = groupingSetsList.getRollupColumns().size(); 885 List<SqlStatement.Type> types = stmt.guessTypes(); 886 assert arity + measureCount + groupingFunctionsCount == types.size(); 887 888 return stmt.getResultSet(); 889 } 890 891 SortedSet<Comparable>[] getDistinctValueWorkspace(int arity) { 892 // Workspace to build up lists of distinct values for each axis. 893 SortedSet<Comparable>[] axisValueSets = new SortedSet[arity]; 894 for (int i = 0; i < axisValueSets.length; i++) { 895 axisValueSets[i] = 896 Util.PreJdk15 897 ? new TreeSet<Comparable>(BooleanComparator.INSTANCE) 898 : new TreeSet<Comparable>(); 899 } 900 return axisValueSets; 901 } 902 903 /** 904 * Decides whether to use a sparse representation for this segment, using 905 * the formula described 906 * {@link mondrian.olap.MondrianProperties#SparseSegmentCountThreshold 907 * here}. 908 * 909 * @param possibleCount Number of values in the space. 910 * @param actualCount Actual number of values. 911 * @return Whether to use a sparse representation. 912 */ 913 static boolean useSparse( 914 final double possibleCount, 915 final double actualCount) 916 { 917 final MondrianProperties properties = MondrianProperties.instance(); 918 double densityThreshold = 919 properties.SparseSegmentDensityThreshold.get(); 920 if (densityThreshold < 0) { 921 densityThreshold = 0; 922 } 923 if (densityThreshold > 1) { 924 densityThreshold = 1; 925 } 926 int countThreshold = properties.SparseSegmentCountThreshold.get(); 927 if (countThreshold < 0) { 928 countThreshold = 0; 929 } 930 boolean sparse = 931 (possibleCount - countThreshold) * densityThreshold > 932 actualCount; 933 if (possibleCount < countThreshold) { 934 assert !sparse 935 : "Should never use sparse if count is less " 936 + "than threshold, possibleCount=" + possibleCount 937 + ", actualCount=" + actualCount 938 + ", countThreshold=" + countThreshold 939 + ", densityThreshold=" + densityThreshold; 940 } 941 if (possibleCount == actualCount) { 942 assert !sparse 943 : "Should never use sparse if result is 100% dense: " 944 + "possibleCount=" + possibleCount 945 + ", actualCount=" + actualCount 946 + ", countThreshold=" + countThreshold 947 + ", densityThreshold=" + densityThreshold; 948 } 949 return sparse; 950 } 951 952 /** 953 * This is a private abstraction wrapper to perform 954 * rollups. It allows us to rollup from a mix of segments 955 * coming from either the local cache or the external one. 956 */ 957 abstract class SegmentRollupWrapper { 958 abstract BitKey getConstrainedColumnsBitKey(); 959 abstract SegmentColumn[] getConstrainedColumns(); 960 abstract SegmentDataset getDataset(); 961 abstract Object[] getValuesForColumn(SegmentColumn cc); 962 abstract mondrian.spi.SegmentColumn getHeader(); 963 public int hashCode() { 964 return getHeader().hashCode(); 965 } 966 public boolean equals(Object obj) { 967 return getHeader().equals(obj); 968 } 969 } 970 971 /** 972 * Collection of rows, each with a set of columns of type Object, double, or 973 * int. Native types are not boxed. 974 */ 975 protected static class RowList { 976 private final Column[] columns; 977 private int rowCount = 0; 978 private int capacity = 0; 979 private int currentRow = -1; 980 981 /** 982 * Creates a RowList. 983 * 984 * @param types Column types 985 */ 986 RowList(List<SqlStatement.Type> types) { 987 this(types, 100); 988 } 989 990 /** 991 * Creates a RowList with a specified initial capacity. 992 * 993 * @param types Column types 994 * @param capacity Initial capacity 995 */ 996 RowList(List<SqlStatement.Type> types, int capacity) { 997 this.columns = new Column[types.size()]; 998 this.capacity = capacity; 999 for (int i = 0; i < columns.length; i++) { 1000 columns[i] = Column.forType(i, types.get(i), capacity); 1001 } 1002 } 1003 1004 void createRow() { 1005 currentRow = rowCount++; 1006 if (rowCount > capacity) { 1007 capacity *= 3; 1008 for (Column column : columns) { 1009 column.resize(capacity); 1010 } 1011 } 1012 } 1013 1014 void setObject(int column, Object value) { 1015 columns[column].setObject(currentRow, value); 1016 } 1017 1018 void setDouble(int column, double value) { 1019 columns[column].setDouble(currentRow, value); 1020 } 1021 1022 void setInt(int column, int value) { 1023 columns[column].setInt(currentRow, value); 1024 } 1025 1026 void setLong(int column, long value) { 1027 columns[column].setLong(currentRow, value); 1028 } 1029 1030 public int size() { 1031 return rowCount; 1032 } 1033 1034 public void createRow(ResultSet resultSet) throws SQLException { 1035 createRow(); 1036 for (Column column : columns) { 1037 column.populateFrom(currentRow, resultSet); 1038 } 1039 } 1040 1041 public List<SqlStatement.Type> getTypes() { 1042 return new AbstractList<SqlStatement.Type>() { 1043 public SqlStatement.Type get(int index) { 1044 return columns[index].type; 1045 } 1046 1047 public int size() { 1048 return columns.length; 1049 } 1050 }; 1051 } 1052 1053 /** 1054 * Moves to before the first row. 1055 */ 1056 public void first() { 1057 currentRow = -1; 1058 } 1059 1060 /** 1061 * Moves to after the last row. 1062 */ 1063 public void last() { 1064 currentRow = rowCount; 1065 } 1066 1067 /** 1068 * Moves forward one row, or returns false if at the last row. 1069 * 1070 * @return whether moved forward 1071 */ 1072 public boolean next() { 1073 if (currentRow < rowCount - 1) { 1074 ++currentRow; 1075 return true; 1076 } 1077 return false; 1078 } 1079 1080 /** 1081 * Moves backward one row, or returns false if at the first row. 1082 * 1083 * @return whether moved backward 1084 */ 1085 public boolean previous() { 1086 if (currentRow > 0) { 1087 --currentRow; 1088 return true; 1089 } 1090 return false; 1091 } 1092 1093 /** 1094 * Returns the object in the given column of the current row. 1095 * 1096 * @param columnIndex Column index 1097 * @return Value of the column 1098 */ 1099 public Object getObject(int columnIndex) { 1100 return columns[columnIndex].getObject(currentRow); 1101 } 1102 1103 public int getInt(int columnIndex) { 1104 return columns[columnIndex].getInt(currentRow); 1105 } 1106 1107 public double getDouble(int columnIndex) { 1108 return columns[columnIndex].getDouble(currentRow); 1109 } 1110 1111 public boolean isNull(int columnIndex) { 1112 return columns[columnIndex].isNull(currentRow); 1113 } 1114 1115 public void setNull(int columnIndex, boolean b) { 1116 columns[columnIndex].setNull(currentRow, b); 1117 } 1118 1119 static abstract class Column { 1120 final int ordinal; 1121 final SqlStatement.Type type; 1122 1123 protected Column(int ordinal, SqlStatement.Type type) { 1124 this.ordinal = ordinal; 1125 this.type = type; 1126 } 1127 1128 static Column forType( 1129 int ordinal, 1130 SqlStatement.Type type, 1131 int capacity) 1132 { 1133 switch (type) { 1134 case OBJECT: 1135 case STRING: 1136 return new ObjectColumn(ordinal, type, capacity); 1137 case INT: 1138 return new IntColumn(ordinal, type, capacity); 1139 case LONG: 1140 return new LongColumn(ordinal, type, capacity); 1141 case DOUBLE: 1142 return new DoubleColumn(ordinal, type, capacity); 1143 default: 1144 throw Util.unexpected(type); 1145 } 1146 } 1147 1148 public abstract void resize(int newSize); 1149 1150 public void setObject(int row, Object value) { 1151 throw new UnsupportedOperationException(); 1152 } 1153 1154 public void setDouble(int row, double value) { 1155 throw new UnsupportedOperationException(); 1156 } 1157 1158 public void setInt(int row, int value) { 1159 throw new UnsupportedOperationException(); 1160 } 1161 1162 public void setLong(int row, long value) { 1163 throw new UnsupportedOperationException(); 1164 } 1165 1166 public void setNull(int row, boolean b) { 1167 throw new UnsupportedOperationException(); 1168 } 1169 1170 public abstract void populateFrom(int row, ResultSet resultSet) 1171 throws SQLException; 1172 1173 public Object getObject(int row) { 1174 throw new UnsupportedOperationException(); 1175 } 1176 1177 public int getInt(int row) { 1178 throw new UnsupportedOperationException(); 1179 } 1180 1181 public double getDouble(int row) { 1182 throw new UnsupportedOperationException(); 1183 } 1184 1185 protected abstract int getCapacity(); 1186 1187 public abstract boolean isNull(int row); 1188 } 1189 1190 static class ObjectColumn extends Column { 1191 private Object[] objects; 1192 1193 ObjectColumn(int ordinal, SqlStatement.Type type, int size) { 1194 super(ordinal, type); 1195 objects = new Object[size]; 1196 } 1197 1198 protected int getCapacity() { 1199 return objects.length; 1200 } 1201 1202 public boolean isNull(int row) { 1203 return objects[row] == null; 1204 } 1205 1206 public void resize(int newSize) { 1207 objects = Util.copyOf(objects, newSize); 1208 } 1209 1210 public void populateFrom(int row, ResultSet resultSet) 1211 throws SQLException 1212 { 1213 objects[row] = resultSet.getObject(ordinal + 1); 1214 } 1215 1216 public void setObject(int row, Object value) { 1217 objects[row] = value; 1218 } 1219 1220 public Object getObject(int row) { 1221 return objects[row]; 1222 } 1223 } 1224 1225 static abstract class NativeColumn extends Column { 1226 protected BitSet nullIndicators; 1227 1228 NativeColumn(int ordinal, SqlStatement.Type type) { 1229 super(ordinal, type); 1230 } 1231 1232 public void setNull(int row, boolean b) { 1233 getNullIndicators().set(row, b); 1234 } 1235 1236 protected BitSet getNullIndicators() { 1237 if (nullIndicators == null) { 1238 nullIndicators = new BitSet(getCapacity()); 1239 } 1240 return nullIndicators; 1241 } 1242 } 1243 1244 static class IntColumn extends NativeColumn { 1245 private int[] ints; 1246 1247 IntColumn(int ordinal, SqlStatement.Type type, int size) { 1248 super(ordinal, type); 1249 ints = new int[size]; 1250 } 1251 1252 public void resize(int newSize) { 1253 ints = Util.copyOf(ints, newSize); 1254 } 1255 1256 public void populateFrom(int row, ResultSet resultSet) 1257 throws SQLException 1258 { 1259 int i = ints[row] = resultSet.getInt(ordinal + 1); 1260 if (i == 0) { 1261 getNullIndicators().set(row, resultSet.wasNull()); 1262 } 1263 } 1264 1265 public void setInt(int row, int value) { 1266 ints[row] = value; 1267 } 1268 1269 public int getInt(int row) { 1270 return ints[row]; 1271 } 1272 1273 public boolean isNull(int row) { 1274 return ints[row] == 0 1275 && nullIndicators != null 1276 && nullIndicators.get(row); 1277 } 1278 1279 protected int getCapacity() { 1280 return ints.length; 1281 } 1282 1283 public Integer getObject(int row) { 1284 return isNull(row) ? null : ints[row]; 1285 } 1286 } 1287 1288 static class LongColumn extends NativeColumn { 1289 private long[] longs; 1290 1291 LongColumn(int ordinal, SqlStatement.Type type, int size) { 1292 super(ordinal, type); 1293 longs = new long[size]; 1294 } 1295 1296 public void resize(int newSize) { 1297 longs = Util.copyOf(longs, newSize); 1298 } 1299 1300 public void populateFrom(int row, ResultSet resultSet) 1301 throws SQLException 1302 { 1303 long i = longs[row] = resultSet.getLong(ordinal + 1); 1304 if (i == 0) { 1305 getNullIndicators().set(row, resultSet.wasNull()); 1306 } 1307 } 1308 1309 public void setLong(int row, long value) { 1310 longs[row] = value; 1311 } 1312 1313 public long getLong(int row) { 1314 return longs[row]; 1315 } 1316 1317 public boolean isNull(int row) { 1318 return longs[row] == 0 1319 && nullIndicators != null 1320 && nullIndicators.get(row); 1321 } 1322 1323 protected int getCapacity() { 1324 return longs.length; 1325 } 1326 1327 public Long getObject(int row) { 1328 return isNull(row) ? null : longs[row]; 1329 } 1330 } 1331 1332 static class DoubleColumn extends NativeColumn { 1333 private double[] doubles; 1334 1335 DoubleColumn(int ordinal, SqlStatement.Type type, int size) { 1336 super(ordinal, type); 1337 doubles = new double[size]; 1338 } 1339 1340 public void resize(int newSize) { 1341 doubles = Util.copyOf(doubles, newSize); 1342 } 1343 1344 public void populateFrom(int row, ResultSet resultSet) 1345 throws SQLException 1346 { 1347 double d = doubles[row] = resultSet.getDouble(ordinal + 1); 1348 if (d == 0d) { 1349 getNullIndicators().set(row, resultSet.wasNull()); 1350 } 1351 } 1352 1353 public void setDouble(int row, double value) { 1354 doubles[row] = value; 1355 } 1356 1357 public double getDouble(int row) { 1358 return doubles[row]; 1359 } 1360 1361 protected int getCapacity() { 1362 return doubles.length; 1363 } 1364 1365 public boolean isNull(int row) { 1366 return doubles[row] == 0d 1367 && nullIndicators != null 1368 && nullIndicators.get(row); 1369 } 1370 1371 public Double getObject(int row) { 1372 return isNull(row) ? null : doubles[row]; 1373 } 1374 } 1375 1376 public interface Handler { 1377 } 1378 } 1379 1380 private static class BooleanComparator 1381 implements Comparator<Object>, Serializable 1382 { 1383 public static final BooleanComparator INSTANCE = 1384 new BooleanComparator(); 1385 1386 private BooleanComparator() { 1387 if (Util.PreJdk15) { 1388 // This class exists to work around the fact that Boolean is not 1389 // Comparable until JDK 1.5. 1390 assert !(Comparable.class.isAssignableFrom(Boolean.class)); 1391 } else { 1392 assert Comparable.class.isAssignableFrom(Boolean.class); 1393 } 1394 } 1395 1396 public int compare(Object o1, Object o2) { 1397 if (o1 instanceof Boolean) { 1398 boolean b1 = (Boolean) o1; 1399 if (o2 instanceof Boolean) { 1400 boolean b2 = (Boolean) o2; 1401 return b1 == b2 1402 ? 0 1403 : (b1 ? 1 : -1); 1404 } else { 1405 return -1; 1406 } 1407 } else { 1408 return ((Comparable) o1).compareTo(o2); 1409 } 1410 } 1411 } 1412} 1413 1414// End SegmentLoader.java