001/*
002// This software is subject to the terms of the Eclipse Public License v1.0
003// Agreement, available at the following URL:
004// http://www.eclipse.org/legal/epl-v10.html.
005// You must accept the terms of that agreement to use this software.
006//
007// Copyright (C) 2002-2005 Julian Hyde
008// Copyright (C) 2005-2013 Pentaho and others
009// All Rights Reserved.
010*/
011package mondrian.rolap.agg;
012
013import mondrian.olap.MondrianException;
014import mondrian.olap.MondrianProperties;
015import mondrian.olap.Util;
016import mondrian.resource.MondrianResource;
017import mondrian.rolap.*;
018import mondrian.rolap.agg.SegmentCacheManager.AbortException;
019import mondrian.rolap.cache.SegmentCacheIndex;
020import mondrian.server.Locus;
021import mondrian.server.monitor.SqlStatementEvent;
022import mondrian.spi.*;
023import mondrian.util.*;
024
025import org.apache.log4j.Logger;
026
027import java.io.Serializable;
028import java.sql.ResultSet;
029import java.sql.SQLException;
030import java.sql.Statement;
031import java.util.*;
032import java.util.concurrent.*;
033
034/**
035 * <p>The <code>SegmentLoader</code> queries database and loads the data into
036 * the given set of segments.</p>
037 *
038 * <p>It reads a segment of <code>measure</code>, where <code>columns</code>
039 * are constrained to <code>values</code>.  Each entry in <code>values</code>
040 * can be null, meaning don't constrain, or can have several values. For
041 * example, <code>getSegment({Unit_sales}, {Region, State, Year}, {"West"},
042 * {"CA", "OR", "WA"}, null})</code> returns sales in states CA, OR and WA
043 * in the Western region, for all years.</p>
044 *
045 * <p>It will also look at the {@link MondrianProperties#SegmentCache} property
046 * and make usage of the SegmentCache provided as an SPI.
047 *
048 * @author Thiyagu, LBoudreau
049 * @since 24 May 2007
050 */
051public class SegmentLoader {
052
053    private static final Logger LOGGER = Logger.getLogger(SegmentLoader.class);
054
055    private final SegmentCacheManager cacheMgr;
056
057    /**
058     * Creates a SegmentLoader.
059     *
060     * @param cacheMgr Cache manager
061     */
062    public SegmentLoader(SegmentCacheManager cacheMgr) {
063        this.cacheMgr = cacheMgr;
064    }
065
066    /**
067     * Loads data for all the segments of the GroupingSets. If the grouping sets
068     * list contains more than one Grouping Set then data is loaded using the
069     * GROUP BY GROUPING SETS sql. Else if only one grouping set is passed in
070     * the list data is loaded without using GROUP BY GROUPING SETS sql. If the
071     * database does not support grouping sets
072     * {@link mondrian.spi.Dialect#supportsGroupingSets()} then
073     * grouping sets list should always have only one element in it.
074     *
075     * <p>For example, if list has 2 grouping sets with columns A, B, C and B, C
076     * respectively, then the SQL will be
077     * "GROUP BY GROUPING SETS ((A, B, C), (B, C))".
078     *
079     * <p>Else if the list has only one grouping set then sql would be without
080     * grouping sets.
081     *
082     * <p>The <code>groupingSets</code> list should be topological order, with
083     * more detailed higher-level grouping sets occurring first. In other words,
084     * the first element of the list should always be the detailed grouping
085     * set (default grouping set), followed by grouping sets which can be
086     * rolled-up on this detailed grouping set.
087     * In the example (A, B, C) is the detailed grouping set and (B, C) is
088     * rolled-up using the detailed.
089     *
090     * <p>Grouping sets are removed from the {@code groupingSets} list as they
091     * are loaded.</p>
092     *
093     * @param cellRequestCount Number of missed cells that led to this request
094     * @param groupingSets   List of grouping sets whose segments are loaded
095     * @param compoundPredicateList Compound predicates
096     * @param segmentFutures List of futures wherein each statement will place
097     *                       a list of the segments it has loaded, when it
098     *                       completes
099     */
100    public void load(
101        int cellRequestCount,
102        List<GroupingSet> groupingSets,
103        List<StarPredicate> compoundPredicateList,
104        List<Future<Map<Segment, SegmentWithData>>> segmentFutures)
105    {
106        if (!MondrianProperties.instance().DisableCaching.get()) {
107            for (GroupingSet groupingSet : groupingSets) {
108                for (Segment segment : groupingSet.getSegments()) {
109                    final SegmentCacheIndex index =
110                        cacheMgr.getIndexRegistry().getIndex(segment.star);
111                    index.add(
112                        segment.getHeader(),
113                        true,
114                        new SegmentBuilder.StarSegmentConverter(
115                            segment.measure,
116                            compoundPredicateList));
117                    // Make sure that we are registered as a client of
118                    // the segment by invoking getFuture.
119                    Util.discard(
120                        index.getFuture(
121                            Locus.peek().execution,
122                            segment.getHeader()));
123                }
124            }
125        }
126        try {
127            segmentFutures.add(
128                cacheMgr.sqlExecutor.submit(
129                    new SegmentLoadCommand(
130                        Locus.peek(),
131                        this,
132                        cellRequestCount,
133                        groupingSets,
134                        compoundPredicateList)));
135        } catch (Exception e) {
136            throw new MondrianException(e);
137        }
138    }
139
140    private static class SegmentLoadCommand
141        implements Callable<Map<Segment, SegmentWithData>>
142    {
143        private final Locus locus;
144        private final SegmentLoader segmentLoader;
145        private final int cellRequestCount;
146        private final List<GroupingSet> groupingSets;
147        private final List<StarPredicate> compoundPredicateList;
148
149        public SegmentLoadCommand(
150            Locus locus,
151            SegmentLoader segmentLoader,
152            int cellRequestCount,
153            List<GroupingSet> groupingSets,
154            List<StarPredicate> compoundPredicateList)
155        {
156            this.locus = locus;
157            this.segmentLoader = segmentLoader;
158            this.cellRequestCount = cellRequestCount;
159            this.groupingSets = groupingSets;
160            this.compoundPredicateList = compoundPredicateList;
161        }
162
163        public Map<Segment, SegmentWithData> call() throws Exception {
164            Locus.push(locus);
165            try {
166                return segmentLoader.loadImpl(
167                    cellRequestCount,
168                    groupingSets,
169                    compoundPredicateList);
170            } finally {
171                Locus.pop(locus);
172            }
173        }
174    }
175
176    private Map<Segment, SegmentWithData> loadImpl(
177        int cellRequestCount,
178        List<GroupingSet> groupingSets,
179        List<StarPredicate> compoundPredicateList)
180    {
181        SqlStatement stmt = null;
182        GroupingSetsList groupingSetsList =
183            new GroupingSetsList(groupingSets);
184        RolapStar.Column[] defaultColumns =
185            groupingSetsList.getDefaultColumns();
186
187        final Map<Segment, SegmentWithData> segmentMap =
188            new HashMap<Segment, SegmentWithData>();
189        Throwable throwable = null;
190        try {
191            int arity = defaultColumns.length;
192            SortedSet<Comparable>[] axisValueSets =
193                getDistinctValueWorkspace(arity);
194
195            stmt = createExecuteSql(
196                cellRequestCount,
197                groupingSetsList,
198                compoundPredicateList);
199
200            if (stmt == null) {
201                // Nothing to do. We're done here.
202                return segmentMap;
203            }
204
205            boolean[] axisContainsNull = new boolean[arity];
206
207            RowList rows =
208                processData(
209                    stmt,
210                    axisContainsNull,
211                    axisValueSets,
212                    groupingSetsList);
213
214            boolean sparse =
215                setAxisDataAndDecideSparseUse(
216                    axisValueSets,
217                    axisContainsNull,
218                    groupingSetsList,
219                    rows);
220
221            final Map<BitKey, GroupingSetsList.Cohort> groupingDataSetsMap =
222                createDataSetsForGroupingSets(
223                    groupingSetsList,
224                    sparse,
225                    rows.getTypes().subList(
226                        arity, rows.getTypes().size()));
227
228            loadDataToDataSets(
229                groupingSetsList, rows, groupingDataSetsMap);
230
231            setDataToSegments(
232                groupingSetsList,
233                groupingDataSetsMap,
234                segmentMap);
235
236            return segmentMap;
237        } catch (Throwable e) {
238            throwable = e;
239            if (stmt == null) {
240                throw new MondrianException(e);
241            }
242            throw stmt.handle(e);
243        } finally {
244            if (stmt != null) {
245                stmt.close();
246            }
247            setFailOnStillLoadingSegments(
248                segmentMap, groupingSetsList, throwable);
249        }
250    }
251
252    /**
253     * Called when a segment has been loaded from SQL, to put into the segment
254     * index and the external cache.
255     *
256     * @param header Segment header
257     * @param body Segment body
258     */
259    private void cacheSegment(
260        RolapStar star,
261        SegmentHeader header,
262        SegmentBody body)
263    {
264        // Write the segment into external cache.
265        //
266        // It would be a mistake to do this from the cacheMgr -- because the
267        // calls may take time. The cacheMgr's actions must all be quick. We
268        // are a worker, so we have plenty of time.
269        //
270        // Also note that we push the segments to external cache after we have
271        // called cacheMgr.loadSucceeded. That call will allow the current
272        // query to proceed.
273        if (!MondrianProperties.instance().DisableCaching.get()) {
274            cacheMgr.compositeCache.put(header, body);
275            cacheMgr.loadSucceeded(star, header, body);
276        }
277    }
278
279    private boolean setFailOnStillLoadingSegments(
280        Map<Segment, SegmentWithData> segmentMap,
281        GroupingSetsList groupingSetsList,
282        Throwable throwable)
283    {
284        int n = 0;
285        for (GroupingSet groupingSet : groupingSetsList.getGroupingSets()) {
286            for (Segment segment : groupingSet.getSegments()) {
287                if (!segmentMap.containsKey(segment)) {
288                    if (throwable == null) {
289                        throwable =
290                            new RuntimeException("Segment failed to load");
291                    }
292                    final SegmentHeader header = segment.getHeader();
293                    cacheMgr.loadFailed(
294                        segment.star,
295                        header,
296                        throwable);
297                    ++n;
298                }
299            }
300        }
301        return n > 0;
302    }
303
304    /**
305     * Loads data to the datasets. If the grouping sets is used,
306     * dataset is fetched from groupingDataSetMap using grouping bit keys of
307     * the row data. If grouping sets is not used, data is loaded on to
308     * nonGroupingDataSets.
309     */
310    private void loadDataToDataSets(
311        GroupingSetsList groupingSetsList,
312        RowList rows,
313        Map<BitKey, GroupingSetsList.Cohort> groupingDataSetMap)
314    {
315        int arity = groupingSetsList.getDefaultColumns().length;
316        SegmentAxis[] axes = groupingSetsList.getDefaultAxes();
317        int segmentLength = groupingSetsList.getDefaultSegments().size();
318
319        final List<SqlStatement.Type> types = rows.getTypes();
320        final boolean useGroupingSet = groupingSetsList.useGroupingSets();
321        for (rows.first(); rows.next();) {
322            final BitKey groupingBitKey;
323            final GroupingSetsList.Cohort cohort;
324            if (useGroupingSet) {
325                groupingBitKey =
326                    (BitKey) rows.getObject(
327                        groupingSetsList.getGroupingBitKeyIndex());
328                cohort = groupingDataSetMap.get(groupingBitKey);
329            } else {
330                groupingBitKey = null;
331                cohort = groupingDataSetMap.get(BitKey.EMPTY);
332            }
333            final int[] pos = cohort.pos;
334            for (int j = 0, k = 0; j < arity; j++) {
335                final SqlStatement.Type type = types.get(j);
336                switch (type) {
337                // TODO: different treatment for INT, LONG, DOUBLE
338                case OBJECT:
339                case STRING:
340                case INT:
341                case LONG:
342                case DOUBLE:
343                    Object o = rows.getObject(j);
344                    if (useGroupingSet
345                        && (o == null || o == RolapUtil.sqlNullValue)
346                        && groupingBitKey.get(
347                            groupingSetsList.findGroupingFunctionIndex(j)))
348                    {
349                        continue;
350                    }
351                    SegmentAxis axis = axes[j];
352                    if (o == null) {
353                        o = RolapUtil.sqlNullValue;
354                    }
355                    // Note: We believe that all value types are Comparable.
356                    // In JDK 1.4, Boolean did not implement Comparable, but
357                    // that's too minor/long ago to worry about.
358                    int offset = axis.getOffset((Comparable) o);
359                    pos[k++] = offset;
360                    break;
361                default:
362                    throw Util.unexpected(type);
363                }
364            }
365
366            for (int j = 0; j < segmentLength; j++) {
367                cohort.segmentDatasetList.get(j).populateFrom(
368                    pos, rows, arity + j);
369            }
370        }
371    }
372
373    private boolean setAxisDataAndDecideSparseUse(
374        SortedSet<Comparable>[] axisValueSets,
375        boolean[] axisContainsNull,
376        GroupingSetsList groupingSetsList,
377        RowList rows)
378    {
379        SegmentAxis[] axes = groupingSetsList.getDefaultAxes();
380        RolapStar.Column[] allColumns = groupingSetsList.getDefaultColumns();
381        // Figure out size of dense array, and allocate it, or use a sparse
382        // array if appropriate.
383        boolean sparse = false;
384        int n = 1;
385        for (int i = 0; i < axes.length; i++) {
386            SortedSet<Comparable> valueSet = axisValueSets[i];
387            axes[i] =
388                new SegmentAxis(
389                    groupingSetsList.getDefaultPredicates()[i],
390                    valueSet,
391                    axisContainsNull[i]);
392            int size = axes[i].getKeys().length;
393            setAxisDataToGroupableList(
394                groupingSetsList,
395                valueSet,
396                axisContainsNull[i],
397                allColumns[i]);
398            int previous = n;
399            n *= size;
400            if ((n < previous) || (n < size)) {
401                // Overflow has occurred.
402                n = Integer.MAX_VALUE;
403                sparse = true;
404            }
405        }
406        return useSparse(sparse, n, rows);
407    }
408
409    boolean useSparse(boolean sparse, int n, RowList rows) {
410        sparse = sparse || useSparse((double) n, (double) rows.size());
411        return sparse;
412    }
413
414    private void setDataToSegments(
415        GroupingSetsList groupingSetsList,
416        Map<BitKey, GroupingSetsList.Cohort> datasetsMap,
417        Map<Segment, SegmentWithData> segmentSlotMap)
418    {
419        List<GroupingSet> groupingSets = groupingSetsList.getGroupingSets();
420        for (int i = 0; i < groupingSets.size(); i++) {
421            List<Segment> segments = groupingSets.get(i).getSegments();
422            GroupingSetsList.Cohort cohort =
423                datasetsMap.get(
424                    groupingSetsList.getRollupColumnsBitKeyList().get(i));
425            for (int j = 0; j < segments.size(); j++) {
426                Segment segment = segments.get(j);
427                final SegmentDataset segmentDataset =
428                    cohort.segmentDatasetList.get(j);
429                final SegmentWithData segmentWithData =
430                    new SegmentWithData(
431                        segment,
432                        segmentDataset,
433                        cohort.axes);
434
435                segmentSlotMap.put(segment, segmentWithData);
436
437                final SegmentHeader header = segmentWithData.getHeader();
438                final SegmentBody body =
439                    segmentWithData.getData().createSegmentBody(
440                        new AbstractList<
441                                Pair<SortedSet<Comparable>, Boolean>>()
442                        {
443                            public Pair<SortedSet<Comparable>, Boolean> get(
444                                int index)
445                            {
446                                return segmentWithData.axes[index]
447                                    .getValuesAndIndicator();
448                            }
449
450                            public int size() {
451                                return segmentWithData.axes.length;
452                            }
453                        });
454
455                // Send a message to the agg manager. It will place the segment
456                // in the index.
457                cacheSegment(segment.star, header, body);
458            }
459        }
460    }
461
462    private Map<BitKey, GroupingSetsList.Cohort> createDataSetsForGroupingSets(
463        GroupingSetsList groupingSetsList,
464        boolean sparse,
465        List<SqlStatement.Type> types)
466    {
467        if (!groupingSetsList.useGroupingSets()) {
468            final GroupingSetsList.Cohort datasets = createDataSets(
469                sparse,
470                groupingSetsList.getDefaultSegments(),
471                groupingSetsList.getDefaultAxes(),
472                types);
473            return Collections.singletonMap(BitKey.EMPTY, datasets);
474        }
475        Map<BitKey, GroupingSetsList.Cohort> datasetsMap =
476            new HashMap<BitKey, GroupingSetsList.Cohort>();
477        List<GroupingSet> groupingSets = groupingSetsList.getGroupingSets();
478        List<BitKey> groupingColumnsBitKeyList =
479            groupingSetsList.getRollupColumnsBitKeyList();
480        for (int i = 0; i < groupingSets.size(); i++) {
481            GroupingSet groupingSet = groupingSets.get(i);
482            GroupingSetsList.Cohort cohort =
483                createDataSets(
484                    sparse,
485                    groupingSet.getSegments(),
486                    groupingSet.getAxes(),
487                    types);
488            datasetsMap.put(groupingColumnsBitKeyList.get(i), cohort);
489        }
490        return datasetsMap;
491    }
492
493    private int calculateMaxDataSize(SegmentAxis[] axes) {
494        int n = 1;
495        for (SegmentAxis axis : axes) {
496            n *= axis.getKeys().length;
497        }
498        return n;
499    }
500
501    private GroupingSetsList.Cohort createDataSets(
502        boolean sparse,
503        List<Segment> segments,
504        SegmentAxis[] axes,
505        List<SqlStatement.Type> types)
506    {
507        final List<SegmentDataset> datasets =
508            new ArrayList<SegmentDataset>(segments.size());
509        final int n;
510        if (sparse) {
511            n = 0;
512        } else {
513            n = calculateMaxDataSize(axes);
514        }
515        for (int i = 0; i < segments.size(); i++) {
516            final Segment segment = segments.get(i);
517            datasets.add(segment.createDataset(axes, sparse, types.get(i), n));
518        }
519        return new GroupingSetsList.Cohort(datasets, axes);
520    }
521
522    private void setAxisDataToGroupableList(
523        GroupingSetsList groupingSetsList,
524        SortedSet<Comparable> valueSet,
525        boolean axisContainsNull,
526        RolapStar.Column column)
527    {
528        for (GroupingSet groupingSet
529            : groupingSetsList.getRollupGroupingSets())
530        {
531            RolapStar.Column[] columns = groupingSet.getColumns();
532            for (int i = 0; i < columns.length; i++) {
533                if (columns[i].equals(column)) {
534                    groupingSet.getAxes()[i] =
535                        new SegmentAxis(
536                            groupingSet.getPredicates()[i],
537                            valueSet,
538                            axisContainsNull);
539                }
540            }
541        }
542    }
543
544    /**
545     * Creates and executes a SQL statement to retrieve the set of cells
546     * specified by a GroupingSetsList.
547     *
548     * <p>This method may be overridden in tests.
549     *
550     * @param cellRequestCount Number of missed cells that led to this request
551     * @param groupingSetsList Grouping
552     * @param compoundPredicateList Compound predicate list
553     * @return An executed SQL statement, or null
554     */
555    SqlStatement createExecuteSql(
556        int cellRequestCount,
557        final GroupingSetsList groupingSetsList,
558        List<StarPredicate> compoundPredicateList)
559    {
560        RolapStar star = groupingSetsList.getStar();
561        Pair<String, List<SqlStatement.Type>> pair =
562            AggregationManager.generateSql(
563                groupingSetsList, compoundPredicateList);
564        final Locus locus =
565            new SqlStatement.StatementLocus(
566                Locus.peek().execution,
567                "Segment.load",
568                "Error while loading segment",
569                SqlStatementEvent.Purpose.CELL_SEGMENT,
570                cellRequestCount);
571
572        // When caching is enabled, we must register the SQL statement
573        // in the index. We don't want to cancel SQL statements that are shared
574        // across threads unless it is safe.
575        final Util.Functor1<Void, Statement> callbackWithCaching =
576            new Util.Functor1<Void, Statement>() {
577                public Void apply(final Statement stmt) {
578                    cacheMgr.execute(
579                        new SegmentCacheManager.Command<Void>() {
580                            public Void call() throws Exception {
581                                boolean atLeastOneActive = false;
582                                for (Segment seg
583                                    : groupingSetsList.getDefaultSegments())
584                                {
585                                    final SegmentCacheIndex index =
586                                        cacheMgr.getIndexRegistry()
587                                            .getIndex(seg.star);
588                                    // Make sure to check if the segment still
589                                    // exists in the index. It could have been
590                                    // removed by a cancellation request since
591                                    // then.
592                                    if (index.contains(seg.getHeader())) {
593                                        index.linkSqlStatement(
594                                            seg.getHeader(), stmt);
595                                        atLeastOneActive = true;
596                                    }
597                                    if (!atLeastOneActive) {
598                                        // There are no segments to load.
599                                        // Throw this so that the segment thread
600                                        // knows to stop.
601                                        throw new AbortException();
602                                    }
603                                }
604                                return null;
605                            }
606                            public Locus getLocus() {
607                              return locus;
608                            }
609                        });
610                    return null;
611                }
612        };
613
614        // When using no cache, we register the SQL statement directly
615        // with the execution instance for cleanup.
616        final Util.Functor1<Void, Statement> callbackNoCaching =
617                new Util.Functor1<Void, Statement>() {
618                    public Void apply(final Statement stmt) {
619                        locus.execution.registerStatement(locus, stmt);
620                        return null;
621                    }
622            };
623
624        try {
625            return RolapUtil.executeQuery(
626                star.getDataSource(),
627                pair.left,
628                pair.right,
629                0,
630                0,
631                locus,
632                -1,
633                -1,
634                // Only one of the two callbacks are required, depending if we
635                // cache the segments or not.
636                MondrianProperties.instance().DisableCaching.get()
637                    ? callbackNoCaching
638                    : callbackWithCaching);
639        } catch (Throwable t) {
640            if (Util.getMatchingCause(t, AbortException.class) != null) {
641                return null;
642            } else {
643                throw new MondrianException(
644                    "Failed to load segment form SQL",
645                    t);
646            }
647        }
648    }
649
650    RowList processData(
651        SqlStatement stmt,
652        final boolean[] axisContainsNull,
653        final SortedSet<Comparable>[] axisValueSets,
654        final GroupingSetsList groupingSetsList) throws SQLException
655    {
656        List<Segment> segments = groupingSetsList.getDefaultSegments();
657        int measureCount = segments.size();
658        ResultSet rawRows = loadData(stmt, groupingSetsList);
659        assert stmt != null;
660        final List<SqlStatement.Type> types = stmt.guessTypes();
661        int arity = axisValueSets.length;
662        final int groupingColumnStartIndex = arity + measureCount;
663
664        // If we're using grouping sets, the SQL query will have a number of
665        // indicator columns, and we roll these into a single BitSet column in
666        // the processed data set.
667        final List<SqlStatement.Type> processedTypes;
668        if (groupingSetsList.useGroupingSets()) {
669            processedTypes =
670                new ArrayList<SqlStatement.Type>(
671                    types.subList(0, groupingColumnStartIndex));
672            processedTypes.add(SqlStatement.Type.OBJECT);
673        } else {
674            processedTypes = types;
675        }
676        final RowList processedRows = new RowList(processedTypes, 100);
677
678        while (rawRows.next()) {
679            checkResultLimit(++stmt.rowCount);
680            processedRows.createRow();
681
682            // get the columns
683            int columnIndex = 0;
684            for (int axisIndex = 0; axisIndex < arity;
685                 axisIndex++, columnIndex++)
686            {
687                final SqlStatement.Type type = types.get(columnIndex);
688                switch (type) {
689                case OBJECT:
690                case STRING:
691                    Object o = rawRows.getObject(columnIndex + 1);
692                    if (o == null) {
693                        o = RolapUtil.sqlNullValue;
694                        if (!groupingSetsList.useGroupingSets()
695                            || !isAggregateNull(
696                                rawRows, groupingColumnStartIndex,
697                            groupingSetsList,
698                            axisIndex))
699                        {
700                            axisContainsNull[axisIndex] = true;
701                        }
702                    } else {
703                        // We assume that all values are Comparable. Boolean
704                        // wasn't Comparable until JDK 1.5, but we can live with
705                        // that bug because JDK 1.4 is no longer important.
706                        axisValueSets[axisIndex].add((Comparable) o);
707                    }
708                    processedRows.setObject(columnIndex, o);
709                    break;
710                case INT:
711                    final int intValue = rawRows.getInt(columnIndex + 1);
712                    if (intValue == 0 && rawRows.wasNull()) {
713                        if (!groupingSetsList.useGroupingSets()
714                            || !isAggregateNull(
715                                rawRows, groupingColumnStartIndex,
716                            groupingSetsList,
717                            axisIndex))
718                        {
719                            axisContainsNull[axisIndex] = true;
720                        }
721                        processedRows.setNull(columnIndex, true);
722                    } else {
723                        axisValueSets[axisIndex].add(intValue);
724                        processedRows.setInt(columnIndex, intValue);
725                    }
726                    break;
727                case LONG:
728                    final long longValue = rawRows.getLong(columnIndex + 1);
729                    if (longValue == 0 && rawRows.wasNull()) {
730                        if (!groupingSetsList.useGroupingSets()
731                            || !isAggregateNull(
732                                rawRows,
733                                groupingColumnStartIndex,
734                                groupingSetsList,
735                                axisIndex))
736                        {
737                            axisContainsNull[axisIndex] = true;
738                        }
739                        processedRows.setNull(columnIndex, true);
740                    } else {
741                        axisValueSets[axisIndex].add(longValue);
742                        processedRows.setLong(columnIndex, longValue);
743                    }
744                    break;
745                case DOUBLE:
746                    final double doubleValue =
747                        rawRows.getDouble(columnIndex + 1);
748                    if (doubleValue == 0 && rawRows.wasNull()) {
749                        if (!groupingSetsList.useGroupingSets()
750                            || !isAggregateNull(
751                                rawRows, groupingColumnStartIndex,
752                            groupingSetsList,
753                            axisIndex))
754                        {
755                            axisContainsNull[axisIndex] = true;
756                        }
757                    }
758                    axisValueSets[axisIndex].add(doubleValue);
759                    processedRows.setDouble(columnIndex, doubleValue);
760                    break;
761                default:
762                    throw Util.unexpected(type);
763                }
764            }
765
766            // pre-compute which measures are numeric
767            final boolean[] numeric = new boolean[measureCount];
768            int k = 0;
769            for (Segment segment : segments) {
770                numeric[k++] = segment.measure.getDatatype().isNumeric();
771            }
772
773            // get the measure
774            for (int i = 0; i < measureCount; i++, columnIndex++) {
775                final SqlStatement.Type type =
776                    types.get(columnIndex);
777                switch (type) {
778                case OBJECT:
779                case STRING:
780                    Object o = rawRows.getObject(columnIndex + 1);
781                    if (o == null) {
782                        o = Util.nullValue; // convert to placeholder
783                    } else if (numeric[i]) {
784                        if (o instanceof Double) {
785                            // nothing to do
786                        } else if (o instanceof Number) {
787                            o = ((Number) o).doubleValue();
788                        } else if (o instanceof byte[]) {
789                            // On MySQL 5.0 in German locale, values can come
790                            // out as byte arrays. Don't know why. Bug 1594119.
791                            o = Double.parseDouble(new String((byte[]) o));
792                        } else {
793                            o = Double.parseDouble(o.toString());
794                        }
795                    }
796                    processedRows.setObject(columnIndex, o);
797                    break;
798                case INT:
799                    final int intValue = rawRows.getInt(columnIndex + 1);
800                    processedRows.setInt(columnIndex, intValue);
801                    if (intValue == 0 && rawRows.wasNull()) {
802                        processedRows.setNull(columnIndex, true);
803                    }
804                    break;
805                case LONG:
806                    final long longValue = rawRows.getLong(columnIndex + 1);
807                    processedRows.setLong(columnIndex, longValue);
808                    if (longValue == 0 && rawRows.wasNull()) {
809                        processedRows.setNull(columnIndex, true);
810                    }
811                    break;
812                case DOUBLE:
813                    final double doubleValue =
814                        rawRows.getDouble(columnIndex + 1);
815                    processedRows.setDouble(columnIndex, doubleValue);
816                    if (doubleValue == 0 && rawRows.wasNull()) {
817                        processedRows.setNull(columnIndex, true);
818                    }
819                    break;
820                default:
821                    throw Util.unexpected(type);
822                }
823            }
824
825            if (groupingSetsList.useGroupingSets()) {
826                processedRows.setObject(
827                    columnIndex,
828                    getRollupBitKey(
829                        groupingSetsList.getRollupColumns().size(),
830                        rawRows, columnIndex));
831            }
832        }
833        return processedRows;
834    }
835
836    private void checkResultLimit(int currentCount) {
837        final int limit =
838            MondrianProperties.instance().ResultLimit.get();
839        if (limit > 0 && currentCount > limit) {
840            throw MondrianResource.instance()
841                .SegmentFetchLimitExceeded.ex(limit);
842        }
843    }
844
845    /**
846     * Generates bit key representing roll up columns
847     */
848    BitKey getRollupBitKey(int arity, ResultSet rowList, int k)
849        throws SQLException
850    {
851        BitKey groupingBitKey = BitKey.Factory.makeBitKey(arity);
852        for (int i = 0; i < arity; i++) {
853            int o = rowList.getInt(k + i + 1);
854            if (o == 1) {
855                groupingBitKey.set(i);
856            }
857        }
858        return groupingBitKey;
859    }
860
861    private boolean isAggregateNull(
862        ResultSet rowList,
863        int groupingColumnStartIndex,
864        GroupingSetsList groupingSetsList,
865        int axisIndex) throws SQLException
866    {
867        int groupingFunctionIndex =
868            groupingSetsList.findGroupingFunctionIndex(axisIndex);
869        if (groupingFunctionIndex == -1) {
870            // Not a rollup column
871            return false;
872        }
873        return rowList.getInt(
874            groupingColumnStartIndex + groupingFunctionIndex + 1) == 1;
875    }
876
877    ResultSet loadData(
878        SqlStatement stmt,
879        GroupingSetsList groupingSetsList)
880        throws SQLException
881    {
882        int arity = groupingSetsList.getDefaultColumns().length;
883        int measureCount = groupingSetsList.getDefaultSegments().size();
884        int groupingFunctionsCount = groupingSetsList.getRollupColumns().size();
885        List<SqlStatement.Type> types = stmt.guessTypes();
886        assert arity + measureCount + groupingFunctionsCount == types.size();
887
888        return stmt.getResultSet();
889    }
890
891    SortedSet<Comparable>[] getDistinctValueWorkspace(int arity) {
892        // Workspace to build up lists of distinct values for each axis.
893        SortedSet<Comparable>[] axisValueSets = new SortedSet[arity];
894        for (int i = 0; i < axisValueSets.length; i++) {
895            axisValueSets[i] =
896                Util.PreJdk15
897                    ? new TreeSet<Comparable>(BooleanComparator.INSTANCE)
898                    : new TreeSet<Comparable>();
899        }
900        return axisValueSets;
901    }
902
903    /**
904     * Decides whether to use a sparse representation for this segment, using
905     * the formula described
906     * {@link mondrian.olap.MondrianProperties#SparseSegmentCountThreshold
907     * here}.
908     *
909     * @param possibleCount Number of values in the space.
910     * @param actualCount   Actual number of values.
911     * @return Whether to use a sparse representation.
912     */
913    static boolean useSparse(
914        final double possibleCount,
915        final double actualCount)
916    {
917        final MondrianProperties properties = MondrianProperties.instance();
918        double densityThreshold =
919            properties.SparseSegmentDensityThreshold.get();
920        if (densityThreshold < 0) {
921            densityThreshold = 0;
922        }
923        if (densityThreshold > 1) {
924            densityThreshold = 1;
925        }
926        int countThreshold = properties.SparseSegmentCountThreshold.get();
927        if (countThreshold < 0) {
928            countThreshold = 0;
929        }
930        boolean sparse =
931            (possibleCount - countThreshold) * densityThreshold >
932                actualCount;
933        if (possibleCount < countThreshold) {
934            assert !sparse
935                : "Should never use sparse if count is less "
936                + "than threshold, possibleCount=" + possibleCount
937                + ", actualCount=" + actualCount
938                + ", countThreshold=" + countThreshold
939                + ", densityThreshold=" + densityThreshold;
940        }
941        if (possibleCount == actualCount) {
942            assert !sparse
943                : "Should never use sparse if result is 100% dense: "
944                + "possibleCount=" + possibleCount
945                + ", actualCount=" + actualCount
946                + ", countThreshold=" + countThreshold
947                + ", densityThreshold=" + densityThreshold;
948        }
949        return sparse;
950    }
951
952    /**
953     * This is a private abstraction wrapper to perform
954     * rollups. It allows us to rollup from a mix of segments
955     * coming from either the local cache or the external one.
956     */
957    abstract class SegmentRollupWrapper {
958        abstract BitKey getConstrainedColumnsBitKey();
959        abstract SegmentColumn[] getConstrainedColumns();
960        abstract SegmentDataset getDataset();
961        abstract Object[] getValuesForColumn(SegmentColumn cc);
962        abstract mondrian.spi.SegmentColumn getHeader();
963        public int hashCode() {
964            return getHeader().hashCode();
965        }
966        public boolean equals(Object obj) {
967            return getHeader().equals(obj);
968        }
969    }
970
971    /**
972     * Collection of rows, each with a set of columns of type Object, double, or
973     * int. Native types are not boxed.
974     */
975    protected static class RowList {
976        private final Column[] columns;
977        private int rowCount = 0;
978        private int capacity = 0;
979        private int currentRow = -1;
980
981        /**
982         * Creates a RowList.
983         *
984         * @param types Column types
985         */
986        RowList(List<SqlStatement.Type> types) {
987            this(types, 100);
988        }
989
990        /**
991         * Creates a RowList with a specified initial capacity.
992         *
993         * @param types Column types
994         * @param capacity Initial capacity
995         */
996        RowList(List<SqlStatement.Type> types, int capacity) {
997            this.columns = new Column[types.size()];
998            this.capacity = capacity;
999            for (int i = 0; i < columns.length; i++) {
1000                columns[i] = Column.forType(i, types.get(i), capacity);
1001            }
1002        }
1003
1004        void createRow() {
1005            currentRow = rowCount++;
1006            if (rowCount > capacity) {
1007                capacity *= 3;
1008                for (Column column : columns) {
1009                    column.resize(capacity);
1010                }
1011            }
1012        }
1013
1014        void setObject(int column, Object value) {
1015            columns[column].setObject(currentRow, value);
1016        }
1017
1018        void setDouble(int column, double value) {
1019            columns[column].setDouble(currentRow, value);
1020        }
1021
1022        void setInt(int column, int value) {
1023            columns[column].setInt(currentRow, value);
1024        }
1025
1026        void setLong(int column, long value) {
1027            columns[column].setLong(currentRow, value);
1028        }
1029
1030        public int size() {
1031            return rowCount;
1032        }
1033
1034        public void createRow(ResultSet resultSet) throws SQLException {
1035            createRow();
1036            for (Column column : columns) {
1037                column.populateFrom(currentRow, resultSet);
1038            }
1039        }
1040
1041        public List<SqlStatement.Type> getTypes() {
1042            return new AbstractList<SqlStatement.Type>() {
1043                public SqlStatement.Type get(int index) {
1044                    return columns[index].type;
1045                }
1046
1047                public int size() {
1048                    return columns.length;
1049                }
1050            };
1051        }
1052
1053        /**
1054         * Moves to before the first row.
1055         */
1056        public void first() {
1057            currentRow = -1;
1058        }
1059
1060        /**
1061         * Moves to after the last row.
1062         */
1063        public void last() {
1064            currentRow = rowCount;
1065        }
1066
1067        /**
1068         * Moves forward one row, or returns false if at the last row.
1069         *
1070         * @return whether moved forward
1071         */
1072        public boolean next() {
1073            if (currentRow < rowCount - 1) {
1074                ++currentRow;
1075                return true;
1076            }
1077            return false;
1078        }
1079
1080        /**
1081         * Moves backward one row, or returns false if at the first row.
1082         *
1083         * @return whether moved backward
1084         */
1085        public boolean previous() {
1086            if (currentRow > 0) {
1087                --currentRow;
1088                return true;
1089            }
1090            return false;
1091        }
1092
1093        /**
1094         * Returns the object in the given column of the current row.
1095         *
1096         * @param columnIndex Column index
1097         * @return Value of the column
1098         */
1099        public Object getObject(int columnIndex) {
1100            return columns[columnIndex].getObject(currentRow);
1101        }
1102
1103        public int getInt(int columnIndex) {
1104            return columns[columnIndex].getInt(currentRow);
1105        }
1106
1107        public double getDouble(int columnIndex) {
1108            return columns[columnIndex].getDouble(currentRow);
1109        }
1110
1111        public boolean isNull(int columnIndex) {
1112            return columns[columnIndex].isNull(currentRow);
1113        }
1114
1115        public void setNull(int columnIndex, boolean b) {
1116            columns[columnIndex].setNull(currentRow, b);
1117        }
1118
1119        static abstract class Column {
1120            final int ordinal;
1121            final SqlStatement.Type type;
1122
1123            protected Column(int ordinal, SqlStatement.Type type) {
1124                this.ordinal = ordinal;
1125                this.type = type;
1126            }
1127
1128            static Column forType(
1129                int ordinal,
1130                SqlStatement.Type type,
1131                int capacity)
1132            {
1133                switch (type) {
1134                case OBJECT:
1135                case STRING:
1136                    return new ObjectColumn(ordinal, type, capacity);
1137                case INT:
1138                    return new IntColumn(ordinal, type, capacity);
1139                case LONG:
1140                    return new LongColumn(ordinal, type, capacity);
1141                case DOUBLE:
1142                    return new DoubleColumn(ordinal, type, capacity);
1143                default:
1144                    throw Util.unexpected(type);
1145                }
1146            }
1147
1148            public abstract void resize(int newSize);
1149
1150            public void setObject(int row, Object value) {
1151                throw new UnsupportedOperationException();
1152            }
1153
1154            public void setDouble(int row, double value) {
1155                throw new UnsupportedOperationException();
1156            }
1157
1158            public void setInt(int row, int value) {
1159                throw new UnsupportedOperationException();
1160            }
1161
1162            public void setLong(int row, long value) {
1163                throw new UnsupportedOperationException();
1164            }
1165
1166            public void setNull(int row, boolean b) {
1167                throw new UnsupportedOperationException();
1168            }
1169
1170            public abstract void populateFrom(int row, ResultSet resultSet)
1171                throws SQLException;
1172
1173            public Object getObject(int row) {
1174                throw new UnsupportedOperationException();
1175            }
1176
1177            public int getInt(int row) {
1178                throw new UnsupportedOperationException();
1179            }
1180
1181            public double getDouble(int row) {
1182                throw new UnsupportedOperationException();
1183            }
1184
1185            protected abstract int getCapacity();
1186
1187            public abstract boolean isNull(int row);
1188        }
1189
1190        static class ObjectColumn extends Column {
1191            private Object[] objects;
1192
1193            ObjectColumn(int ordinal, SqlStatement.Type type, int size) {
1194                super(ordinal, type);
1195                objects = new Object[size];
1196            }
1197
1198            protected int getCapacity() {
1199                return objects.length;
1200            }
1201
1202            public boolean isNull(int row) {
1203                return objects[row] == null;
1204            }
1205
1206            public void resize(int newSize) {
1207                objects = Util.copyOf(objects, newSize);
1208            }
1209
1210            public void populateFrom(int row, ResultSet resultSet)
1211                throws SQLException
1212            {
1213                objects[row] = resultSet.getObject(ordinal + 1);
1214            }
1215
1216            public void setObject(int row, Object value) {
1217                objects[row] = value;
1218            }
1219
1220            public Object getObject(int row) {
1221                return objects[row];
1222            }
1223        }
1224
1225        static abstract class NativeColumn extends Column {
1226            protected BitSet nullIndicators;
1227
1228            NativeColumn(int ordinal, SqlStatement.Type type) {
1229                super(ordinal, type);
1230            }
1231
1232            public void setNull(int row, boolean b) {
1233                getNullIndicators().set(row, b);
1234            }
1235
1236            protected BitSet getNullIndicators() {
1237                if (nullIndicators == null) {
1238                    nullIndicators = new BitSet(getCapacity());
1239                }
1240                return nullIndicators;
1241            }
1242        }
1243
1244        static class IntColumn extends NativeColumn {
1245            private int[] ints;
1246
1247            IntColumn(int ordinal, SqlStatement.Type type, int size) {
1248                super(ordinal, type);
1249                ints = new int[size];
1250            }
1251
1252            public void resize(int newSize) {
1253                ints = Util.copyOf(ints, newSize);
1254            }
1255
1256            public void populateFrom(int row, ResultSet resultSet)
1257                throws SQLException
1258            {
1259                int i = ints[row] = resultSet.getInt(ordinal + 1);
1260                if (i == 0) {
1261                    getNullIndicators().set(row, resultSet.wasNull());
1262                }
1263            }
1264
1265            public void setInt(int row, int value) {
1266                ints[row] = value;
1267            }
1268
1269            public int getInt(int row) {
1270                return ints[row];
1271            }
1272
1273            public boolean isNull(int row) {
1274                return ints[row] == 0
1275                   && nullIndicators != null
1276                   && nullIndicators.get(row);
1277            }
1278
1279            protected int getCapacity() {
1280                return ints.length;
1281            }
1282
1283            public Integer getObject(int row) {
1284                return isNull(row) ? null : ints[row];
1285            }
1286        }
1287
1288        static class LongColumn extends NativeColumn {
1289            private long[] longs;
1290
1291            LongColumn(int ordinal, SqlStatement.Type type, int size) {
1292                super(ordinal, type);
1293                longs = new long[size];
1294            }
1295
1296            public void resize(int newSize) {
1297                longs = Util.copyOf(longs, newSize);
1298            }
1299
1300            public void populateFrom(int row, ResultSet resultSet)
1301                throws SQLException
1302            {
1303                long i = longs[row] = resultSet.getLong(ordinal + 1);
1304                if (i == 0) {
1305                    getNullIndicators().set(row, resultSet.wasNull());
1306                }
1307            }
1308
1309            public void setLong(int row, long value) {
1310                longs[row] = value;
1311            }
1312
1313            public long getLong(int row) {
1314                return longs[row];
1315            }
1316
1317            public boolean isNull(int row) {
1318                return longs[row] == 0
1319                   && nullIndicators != null
1320                   && nullIndicators.get(row);
1321            }
1322
1323            protected int getCapacity() {
1324                return longs.length;
1325            }
1326
1327            public Long getObject(int row) {
1328                return isNull(row) ? null : longs[row];
1329            }
1330        }
1331
1332        static class DoubleColumn extends NativeColumn {
1333            private double[] doubles;
1334
1335            DoubleColumn(int ordinal, SqlStatement.Type type, int size) {
1336                super(ordinal, type);
1337                doubles = new double[size];
1338            }
1339
1340            public void resize(int newSize) {
1341                doubles = Util.copyOf(doubles, newSize);
1342            }
1343
1344            public void populateFrom(int row, ResultSet resultSet)
1345                throws SQLException
1346            {
1347                double d = doubles[row] = resultSet.getDouble(ordinal + 1);
1348                if (d == 0d) {
1349                    getNullIndicators().set(row, resultSet.wasNull());
1350                }
1351            }
1352
1353            public void setDouble(int row, double value) {
1354                doubles[row] = value;
1355            }
1356
1357            public double getDouble(int row) {
1358                return doubles[row];
1359            }
1360
1361            protected int getCapacity() {
1362                return doubles.length;
1363            }
1364
1365            public boolean isNull(int row) {
1366                return doubles[row] == 0d
1367                   && nullIndicators != null
1368                   && nullIndicators.get(row);
1369            }
1370
1371            public Double getObject(int row) {
1372                return isNull(row) ? null : doubles[row];
1373            }
1374        }
1375
1376        public interface Handler {
1377        }
1378    }
1379
1380    private static class BooleanComparator
1381        implements Comparator<Object>, Serializable
1382    {
1383        public static final BooleanComparator INSTANCE =
1384            new BooleanComparator();
1385
1386        private BooleanComparator() {
1387            if (Util.PreJdk15) {
1388                // This class exists to work around the fact that Boolean is not
1389                // Comparable until JDK 1.5.
1390                assert !(Comparable.class.isAssignableFrom(Boolean.class));
1391            } else {
1392                assert Comparable.class.isAssignableFrom(Boolean.class);
1393            }
1394        }
1395
1396        public int compare(Object o1, Object o2) {
1397            if (o1 instanceof Boolean) {
1398                boolean b1 = (Boolean) o1;
1399                if (o2 instanceof Boolean) {
1400                    boolean b2 = (Boolean) o2;
1401                    return b1 == b2
1402                        ? 0
1403                        : (b1 ? 1 : -1);
1404                } else {
1405                    return -1;
1406                }
1407            } else {
1408                return ((Comparable) o1).compareTo(o2);
1409            }
1410        }
1411    }
1412}
1413
1414// End SegmentLoader.java