001/*
002// This software is subject to the terms of the Eclipse Public License v1.0
003// Agreement, available at the following URL:
004// http://www.eclipse.org/legal/epl-v10.html.
005// You must accept the terms of that agreement to use this software.
006//
007// Copyright (C) 2001-2005 Julian Hyde
008// Copyright (C) 2005-2013 Pentaho and others
009// All Rights Reserved.
010*/
011package mondrian.rolap;
012
013import mondrian.calc.*;
014import mondrian.calc.impl.*;
015import mondrian.mdx.*;
016import mondrian.olap.*;
017import mondrian.olap.fun.*;
018import mondrian.olap.fun.VisualTotalsFunDef.VisualTotalMember;
019import mondrian.olap.type.ScalarType;
020import mondrian.olap.type.SetType;
021import mondrian.resource.MondrianResource;
022import mondrian.rolap.agg.AggregationManager;
023import mondrian.rolap.agg.CellRequestQuantumExceededException;
024import mondrian.server.Execution;
025import mondrian.server.Locus;
026import mondrian.spi.CellFormatter;
027import mondrian.util.*;
028
029import org.apache.log4j.Logger;
030
031import java.util.*;
032
033
034/**
035 * A <code>RolapResult</code> is the result of running a query.
036 *
037 * @author jhyde
038 * @since 10 August, 2001
039 */
040public class RolapResult extends ResultBase {
041
042    static final Logger LOGGER = Logger.getLogger(ResultBase.class);
043
044    private RolapEvaluator evaluator;
045    RolapEvaluator slicerEvaluator;
046    private final CellKey point;
047
048    private CellInfoContainer cellInfos;
049    private FastBatchingCellReader batchingReader;
050    private final CellReader aggregatingReader;
051    private Modulos modulos = null;
052    private final int maxEvalDepth =
053            MondrianProperties.instance().MaxEvalDepth.get();
054
055    private final Map<Integer, Boolean> positionsHighCardinality =
056        new HashMap<Integer, Boolean>();
057    private final Map<Integer, TupleCursor> positionsIterators =
058        new HashMap<Integer, TupleCursor>();
059    private final Map<Integer, Integer> positionsIndexes =
060        new HashMap<Integer, Integer>();
061    private final Map<Integer, List<List<Member>>> positionsCurrent =
062        new HashMap<Integer, List<List<Member>>>();
063
064    /**
065     * Creates a RolapResult.
066     *
067     * @param execution Execution of a statement
068     * @param execute Whether to execute the query
069     */
070    RolapResult(
071        final Execution execution,
072        boolean execute)
073    {
074        super(execution, null);
075
076        this.point = CellKey.Generator.newCellKey(axes.length);
077        final AggregationManager aggMgr =
078            execution.getMondrianStatement()
079                .getMondrianConnection()
080                .getServer().getAggregationManager();
081        this.aggregatingReader = aggMgr.getCacheCellReader();
082        final int expDeps =
083            MondrianProperties.instance().TestExpDependencies.get();
084        if (expDeps > 0) {
085            this.evaluator = new RolapDependencyTestingEvaluator(this, expDeps);
086        } else {
087            final RolapEvaluatorRoot root =
088                new RolapResultEvaluatorRoot(this);
089            if (statement.getProfileHandler() != null) {
090                this.evaluator = new RolapProfilingEvaluator(root);
091            } else {
092                this.evaluator = new RolapEvaluator(root);
093            }
094        }
095        RolapCube cube = (RolapCube) query.getCube();
096        this.batchingReader =
097            new FastBatchingCellReader(execution, cube, aggMgr);
098
099        this.cellInfos =
100            (query.axes.length > 4)
101                ? new CellInfoMap(point)
102                : new CellInfoPool(query.axes.length);
103
104        if (!execute) {
105            return;
106        }
107
108        boolean normalExecution = true;
109        try {
110            // This call to clear the cube's cache only has an
111            // effect if caching has been disabled, otherwise
112            // nothing happens.
113            // Clear the local cache before a query has run
114            cube.clearCachedAggregations();
115
116            /////////////////////////////////////////////////////////////////
117            //
118            // Evaluation Algorithm
119            //
120            // There are three basic steps to the evaluation algorithm:
121            // 1) Determine all Members for each axis but do not save
122            // information (do not build the RolapAxis),
123            // 2) Save all Members for each axis (build RolapAxis).
124            // 3) Evaluate and store each Cell determined by the Members
125            // of the axes.
126            // Step 1 converges on the stable set of Members pre axis.
127            // Steps 1 and 2 make sure that the data has been loaded.
128            //
129            // More detail follows.
130            //
131            // Explicit and Implicit Members:
132            // A Member is said to be 'explicit' if it appears on one of
133            // the Axes (one of the RolapAxis Position List of Members).
134            // A Member is 'implicit' if it is in the query but does not
135            // end up on any Axes (its usage, for example, is in a function).
136            // When for a Dimension none of its Members are explicit in the
137            // query, then the default Member is used which is like putting
138            // the Member in the Slicer.
139            //
140            // Special Dimensions:
141            // There are 2 special dimensions.
142            // The first is the Time dimension. If in a schema there is
143            // no ALL Member, then Whatever happens to be the default
144            // Member is used if Time Members are not explicitly set
145            // in the query.
146            // The second is the Measures dimension. This dimension
147            // NEVER has an ALL Member. A cube's default Measure is set
148            // by convention - its simply the first Measure defined in the
149            // cube.
150            //
151            // First a RolapEvaluator is created. During its creation,
152            // it gets a Member from each Hierarchy. Each Member is the
153            // default Member of the Hierarchy. For most Hierarchies this
154            // Member is the ALL Member, but there are cases where 1)
155            // a Hierarchy does not have an ALL Member or 2) the Hierarchy
156            // has an ALL Member but that Member is not the default Member.
157            // In these cases, the default Member is still used, but its
158            // use can cause evaluation issues (seemingly strange evaluation
159            // results).
160            //
161            // Next, load all root Members for Hierarchies that have no ALL
162            // Member and load ALL Members that are not the default Member.
163            //
164            // Determine the Members of the Slicer axis (Step 1 above).  Any
165            // Members found are added to the AxisMember object. If one of these
166            // Members happens to be a Measure, then the Slicer is explicitly
167            // specifying the query's Measure and this should be put into the
168            // evaluator's context (replacing the default Measure which just
169            // happens to be the first Measure defined in the cube).  Other
170            // Members found in the AxisMember object are also placed into the
171            // evaluator's context since these also are explicitly specified.
172            // Also, any other Members in the AxisMember object which have the
173            // same Hierarchy as Members in the list of root Members for
174            // Hierarchies that have no ALL Member, replace those Members - they
175            // Slicer has explicitly determined which ones to use. The
176            // AxisMember object is now cleared.
177            // The Slicer does not depend upon the other Axes, but the other
178            // Axes depend upon both the Slicer and each other.
179            //
180            // The AxisMember object also checks if the number of Members
181            // exceeds the ResultLimit property throwing a
182            // TotalMembersLimitExceeded Exception if it does.
183            //
184            // For all non-Slicer axes, the Members are determined (Step 1
185            // above). If a Measure is found in the AxisMember, then an
186            // Axis is explicitly specifying a Measure.
187            // If any Members in the AxisMember object have the same Hierarchy
188            // as a Member in the set of root Members for Hierarchies that have
189            // no ALL Member, then replace those root Members with the Member
190            // from the AxisMember object. In this case, again, a Member
191            // was explicitly specified in an Axis. If this replacement
192            // occurs, then one must redo this step with the new Members.
193            //
194            // Now Step 3 above is done. First to the Slicer Axis and then
195            // to the other Axes. Here the Axes are actually generated.
196            // If a Member of an Axis is an Calculated Member (and the
197            // Calculated Member is not a Member of the Measure Hierarchy),
198            // then find the Dimension associated with the Calculated
199            // Member and remove Members with the same Dimension in the set of
200            // root Members for Hierarchies that have no ALL Member.
201            // This is done because via the Calculated Member the Member
202            // was implicitly specified in the query. If this removal occurs,
203            // then the Axes must be re-evaluated repeating Step 3.
204            //
205            /////////////////////////////////////////////////////////////////
206
207
208            // The AxisMember object is used to hold Members that are found
209            // during Step 1 when the Axes are determined.
210            final AxisMemberList axisMembers = new AxisMemberList();
211
212
213            // list of ALL Members that are not default Members
214            final List<Member> nonDefaultAllMembers = new ArrayList<Member>();
215
216            // List of Members of Hierarchies that do not have an ALL Member
217            List<List<Member>> nonAllMembers = new ArrayList<List<Member>>();
218
219            // List of Measures
220            final List<Member> measureMembers = new ArrayList<Member>();
221
222            // load all root Members for Hierarchies that have no ALL
223            // Member and load ALL Members that are not the default Member.
224            // Also, all Measures are are gathered.
225            loadSpecialMembers(
226                nonDefaultAllMembers, nonAllMembers, measureMembers);
227
228            // clear evaluation cache
229            query.clearEvalCache();
230
231            // Save, may be needed by some Expression Calc's
232            query.putEvalCache("ALL_MEMBER_LIST", nonDefaultAllMembers);
233
234
235            final List<List<Member>> emptyNonAllMembers =
236                Collections.emptyList();
237
238            // Initial evaluator, to execute slicer.
239            slicerEvaluator = evaluator.push();
240
241            /////////////////////////////////////////////////////////////////
242            // Determine Slicer
243            //
244            axisMembers.setSlicer(true);
245            loadMembers(
246                emptyNonAllMembers,
247                evaluator,
248                query.getSlicerAxis(),
249                query.slicerCalc,
250                axisMembers);
251            axisMembers.setSlicer(false);
252
253            // Save unadulterated context for the next time we need to evaluate
254            // the slicer.
255            final RolapEvaluator savedEvaluator = evaluator.push();
256
257            if (!axisMembers.isEmpty()) {
258                for (Member m : axisMembers) {
259                    if (m == null) {
260                        break;
261                    }
262                    evaluator.setSlicerContext(m);
263                    if (m.isMeasure()) {
264                        // A Measure was explicitly declared in the
265                        // Slicer, don't need to worry about Measures
266                        // for this query.
267                        measureMembers.clear();
268                    }
269                }
270                replaceNonAllMembers(nonAllMembers, axisMembers);
271                axisMembers.clearMembers();
272            }
273
274            // Save evaluator that has slicer as its context.
275            slicerEvaluator = evaluator.push();
276
277            /////////////////////////////////////////////////////////////////
278            // Determine Axes
279            //
280            boolean changed = false;
281
282            // reset to total member count
283            axisMembers.clearTotalCellCount();
284
285            for (int i = 0; i < axes.length; i++) {
286                final QueryAxis axis = query.axes[i];
287                final Calc calc = query.axisCalcs[i];
288                loadMembers(
289                    emptyNonAllMembers, evaluator, axis, calc, axisMembers);
290            }
291
292            if (!axisMembers.isEmpty()) {
293                for (Member m : axisMembers) {
294                    if (m.isMeasure()) {
295                        // A Measure was explicitly declared on an
296                        // axis, don't need to worry about Measures
297                        // for this query.
298                        measureMembers.clear();
299                    }
300                }
301                changed = replaceNonAllMembers(nonAllMembers, axisMembers);
302                axisMembers.clearMembers();
303            }
304
305            if (changed) {
306                // only count number of members, do not collect any
307                axisMembers.countOnly(true);
308                // reset to total member count
309                axisMembers.clearTotalCellCount();
310
311                final int savepoint = evaluator.savepoint();
312                try {
313                    for (int i = 0; i < axes.length; i++) {
314                        final QueryAxis axis = query.axes[i];
315                        final Calc calc = query.axisCalcs[i];
316                        loadMembers(
317                            nonAllMembers,
318                            evaluator,
319                            axis, calc, axisMembers);
320                        evaluator.restore(savepoint);
321                    }
322                } finally {
323                    evaluator.restore(savepoint);
324                }
325            }
326
327            // throws exception if number of members exceeds limit
328            axisMembers.checkLimit();
329            Axis savedSlicerAxis;
330            /////////////////////////////////////////////////////////////////
331            // Execute Slicer
332            //
333            RolapEvaluator slicerEvaluator;
334            do {
335                TupleIterable tupleIterable =
336                    evalExecute(
337                        nonAllMembers,
338                        nonAllMembers.size() - 1,
339                        savedEvaluator,
340                        query.getSlicerAxis(),
341                        query.slicerCalc);
342                // Materialize the iterable as a list. Although it may take
343                // memory, we need the first member below, and besides, slicer
344                // axes are generally small.
345                TupleList tupleList =
346                    TupleCollections.materialize(tupleIterable, true);
347
348                this.slicerAxis = new RolapAxis(tupleList);
349                // the slicerAxis may be overwritten during slicer execution
350                // if there is a compound slicer.  Save it so that it can be
351                // reverted before completing result construction.
352                savedSlicerAxis = this.slicerAxis;
353
354                // Use the context created by the slicer for the other
355                // axes.  For example, "select filter([Customers], [Store
356                // Sales] > 100) on columns from Sales where
357                // ([Time].[1998])" should show customers whose 1998 (not
358                // total) purchases exceeded 100.
359                slicerEvaluator = this.evaluator;
360                if (tupleList.size() > 1) {
361                    tupleList =
362                        AggregateFunDef.AggregateCalc.optimizeTupleList(
363                            slicerEvaluator,
364                            tupleList,
365                            false);
366
367                    final Calc valueCalc =
368                        new ValueCalc(
369                            new DummyExp(new ScalarType()));
370                    final TupleList tupleList1 = tupleList;
371
372
373                    final Calc calc =
374                        new GenericCalc(
375                            new DummyExp(query.slicerCalc.getType()))
376                        {
377                            public Object evaluate(Evaluator evaluator) {
378                                return AggregateFunDef.AggregateCalc.aggregate(
379                                    valueCalc, evaluator, tupleList1);
380                            }
381                        };
382                    final List<RolapHierarchy> hierarchyList =
383                        new AbstractList<RolapHierarchy>() {
384                            final List<Member> pos0 = tupleList1.get(0);
385
386                            public RolapHierarchy get(int index) {
387                                return ((RolapMember) pos0.get(index))
388                                    .getHierarchy();
389                            }
390
391                            public int size() {
392                                return pos0.size();
393                            }
394                        };
395
396                    // replace the slicer set with a placeholder to avoid
397                    // interaction between the aggregate calc we just created
398                    // and any calculated members that might be present in
399                    // the slicer.
400                    // Arbitrarily picks the first dim of the first tuple
401                    // to use as placeholder.
402                    Member placeholder = setPlaceholderSlicerAxis(
403                        (RolapMember)tupleList.get(0).get(0), calc);
404                    evaluator.setContext(placeholder);
405                }
406            } while (phase());
407
408            /////////////////////////////////////////////////////////////////
409            // Execute Axes
410            //
411            final int savepoint = evaluator.savepoint();
412            do {
413                try {
414                    boolean redo;
415                    do {
416                        evaluator.restore(savepoint);
417                        redo = false;
418                        for (int i = 0; i < axes.length; i++) {
419                            QueryAxis axis = query.axes[i];
420                            final Calc calc = query.axisCalcs[i];
421                            TupleIterable tupleIterable =
422                                evalExecute(
423                                    nonAllMembers,
424                                    nonAllMembers.size() - 1,
425                                    evaluator,
426                                    axis,
427                                    calc);
428
429                            if (!nonAllMembers.isEmpty()) {
430                                final TupleIterator tupleIterator =
431                                    tupleIterable.tupleIterator();
432                                if (tupleIterator.hasNext()) {
433                                    List<Member> tuple0 = tupleIterator.next();
434                                    // Only need to process the first tuple on
435                                    // the axis.
436                                    for (Member m : tuple0) {
437                                        if (m.isCalculated()) {
438                                            CalculatedMeasureVisitor visitor =
439                                                new CalculatedMeasureVisitor();
440                                            m.getExpression().accept(visitor);
441                                            Dimension dimension =
442                                                visitor.dimension;
443                                            if (removeDimension(
444                                                    dimension, nonAllMembers))
445                                            {
446                                                redo = true;
447                                            }
448                                        }
449                                    }
450                                }
451                            }
452                            this.axes[i] =
453                                new RolapAxis(
454                                    TupleCollections.materialize(
455                                        tupleIterable, false));
456                        }
457                    } while (redo);
458                } catch (CellRequestQuantumExceededException e) {
459                    // Safe to ignore. Need to call 'phase' and loop again.
460                }
461            } while (phase());
462
463            evaluator.restore(savepoint);
464
465            // Get value for each Cell
466            final Locus locus = new Locus(execution, null, "Loading cells");
467            Locus.push(locus);
468            try {
469                executeBody(slicerEvaluator, query, new int[axes.length]);
470            } finally {
471                Locus.pop(locus);
472            }
473
474            // If you are very close to running out of memory due to
475            // the number of CellInfo's in cellInfos, then calling this
476            // may cause the out of memory one is trying to aviod.
477            // On the other hand, calling this can reduce the size of
478            // the ObjectPool's internal storage by half (but, of course,
479            // it will not reduce the size of the stored objects themselves).
480            // Only call this if there are lots of CellInfo.
481            if (this.cellInfos.size() > 10000) {
482                this.cellInfos.trimToSize();
483            }
484            // revert the slicer axis so that the original slicer
485            // can be included in the result.
486            this.slicerAxis  = savedSlicerAxis;
487        } catch (ResultLimitExceededException ex) {
488            // If one gets a ResultLimitExceededException, then
489            // don't count on anything being worth caching.
490            normalExecution = false;
491
492            // De-reference data structures that might be holding
493            // partial results but surely are taking up memory.
494            evaluator = null;
495            slicerEvaluator = null;
496            cellInfos = null;
497            batchingReader = null;
498            for (int i = 0; i < axes.length; i++) {
499                axes[i] = null;
500            }
501            slicerAxis = null;
502
503            query.clearEvalCache();
504
505            throw ex;
506        } finally {
507            if (normalExecution) {
508                // Expression cache duration is for each query. It is time to
509                // clear out the whole expression cache at the end of a query.
510                evaluator.clearExpResultCache(true);
511            }
512            if (LOGGER.isDebugEnabled()) {
513                LOGGER.debug("RolapResult<init>: " + Util.printMemory());
514            }
515        }
516    }
517
518    /**
519     * Sets slicerAxis to a dummy placeholder RolapAxis containing
520     * a single item TupleList with the null member of hierarchy.
521     * This is used with compound slicer evaluation to avoid the slicer
522     * tuple list from interacting with the aggregate calc which rolls up
523     * the set.  This member will contain the AggregateCalc which rolls
524     * up the set on the slicer.
525     */
526    private Member setPlaceholderSlicerAxis(
527        final RolapMember member, final Calc calc)
528    {
529        ValueFormatter formatter;
530        if (member.getDimension().isMeasures()) {
531            formatter = ((RolapMeasure)member).getFormatter();
532        } else {
533            formatter = null;
534        }
535
536        CompoundSlicerRolapMember placeholderMember =
537            new CompoundSlicerRolapMember(
538                (RolapMember)member.getHierarchy().getNullMember(),
539                calc, formatter);
540
541
542        placeholderMember.setProperty(
543            Property.FORMAT_STRING.getName(),
544            member.getPropertyValue(Property.FORMAT_STRING.getName()));
545        placeholderMember.setProperty(
546            Property.FORMAT_EXP_PARSED.getName(),
547            member.getPropertyValue(Property.FORMAT_EXP_PARSED.getName()));
548
549        TupleList dummyList = TupleCollections.createList(1);
550        dummyList.addTuple(placeholderMember);
551
552        this.slicerAxis = new RolapAxis(dummyList);
553        return placeholderMember;
554    }
555
556    private boolean phase() {
557        if (batchingReader.isDirty()) {
558            execution.tracePhase(
559                batchingReader.getHitCount(),
560                batchingReader.getMissCount(),
561                batchingReader.getPendingCount());
562
563            return batchingReader.loadAggregations();
564        } else {
565            return false;
566        }
567    }
568
569    @Override
570    public void close() {
571        super.close();
572    }
573
574    protected boolean removeDimension(
575        Dimension dimension,
576        List<List<Member>> memberLists)
577    {
578        for (int i = 0; i < memberLists.size(); i++) {
579            List<Member> memberList = memberLists.get(i);
580            if (memberList.get(0).getDimension().equals(dimension)) {
581                memberLists.remove(i);
582                return true;
583            }
584        }
585        return false;
586    }
587
588    public final Execution getExecution() {
589        return execution;
590    }
591
592    private static class CalculatedMeasureVisitor
593        extends MdxVisitorImpl
594    {
595        Dimension dimension;
596
597        CalculatedMeasureVisitor() {
598        }
599
600        public Object visit(DimensionExpr dimensionExpr) {
601            dimension = dimensionExpr.getDimension();
602            return null;
603        }
604
605        public Object visit(HierarchyExpr hierarchyExpr) {
606            Hierarchy hierarchy = hierarchyExpr.getHierarchy();
607            dimension = hierarchy.getDimension();
608            return null;
609        }
610
611        public Object visit(MemberExpr memberExpr)  {
612            Member member = memberExpr.getMember();
613            dimension = member.getHierarchy().getDimension();
614            return null;
615        }
616    }
617
618    protected boolean replaceNonAllMembers(
619        List<List<Member>> nonAllMembers,
620        AxisMemberList axisMembers)
621    {
622        boolean changed = false;
623        List<Member> mList = new ArrayList<Member>();
624        for (ListIterator<List<Member>> it = nonAllMembers.listIterator();
625                it.hasNext();)
626        {
627            List<Member> ms = it.next();
628            Hierarchy h = ms.get(0).getHierarchy();
629            mList.clear();
630            for (Member m : axisMembers) {
631                if (m.getHierarchy().equals(h)) {
632                    mList.add(m);
633                }
634            }
635            if (! mList.isEmpty()) {
636                changed = true;
637                it.set(new ArrayList<Member>(mList));
638            }
639        }
640        return changed;
641    }
642
643    protected void loadMembers(
644        List<List<Member>> nonAllMembers,
645        RolapEvaluator evaluator,
646        QueryAxis axis,
647        Calc calc,
648        AxisMemberList axisMembers)
649    {
650        int attempt = 0;
651        evaluator.setCellReader(batchingReader);
652        while (true) {
653            axisMembers.clearAxisCount();
654            final int savepoint = evaluator.savepoint();
655            try {
656                evalLoad(
657                    nonAllMembers,
658                    nonAllMembers.size() - 1,
659                    evaluator,
660                    axis,
661                    calc,
662                    axisMembers);
663            } catch (CellRequestQuantumExceededException e) {
664                // Safe to ignore. Need to call 'phase' and loop again.
665                // Decrement count because it wasn't a recursive formula that
666                // caused the iteration.
667                --attempt;
668            } finally {
669                evaluator.restore(savepoint);
670            }
671
672            if (!phase()) {
673                break;
674            } else {
675                // Clear invalid expression result so that the next evaluation
676                // will pick up the newly loaded aggregates.
677                evaluator.clearExpResultCache(false);
678            }
679
680            if (attempt++ > maxEvalDepth) {
681                throw Util.newInternal(
682                    "Failed to load all aggregations after "
683                    + maxEvalDepth
684                    + " passes; there's probably a cycle");
685            }
686        }
687    }
688
689    void evalLoad(
690        List<List<Member>> nonAllMembers,
691        int cnt,
692        Evaluator evaluator,
693        QueryAxis axis,
694        Calc calc,
695        AxisMemberList axisMembers)
696    {
697        final int savepoint = evaluator.savepoint();
698        try {
699            if (cnt < 0) {
700                executeAxis(evaluator, axis, calc, false, axisMembers);
701            } else {
702                for (Member m : nonAllMembers.get(cnt)) {
703                    evaluator.setContext(m);
704                    evalLoad(
705                        nonAllMembers, cnt - 1, evaluator,
706                        axis, calc, axisMembers);
707                }
708            }
709        } finally {
710            evaluator.restore(savepoint);
711        }
712    }
713
714    TupleIterable evalExecute(
715        List<List<Member>> nonAllMembers,
716        int cnt,
717        RolapEvaluator evaluator,
718        QueryAxis queryAxis,
719        Calc calc)
720    {
721        final int savepoint = evaluator.savepoint();
722        final int arity = calc == null ? 0 : calc.getType().getArity();
723        if (cnt < 0) {
724            try {
725                final TupleIterable axis =
726                    executeAxis(evaluator, queryAxis, calc, true, null);
727                return axis;
728            } finally {
729                evaluator.restore(savepoint);
730            }
731            // No need to clear expression cache here as no new aggregates are
732            // loaded(aggregatingReader reads from cache).
733        } else {
734            try {
735                TupleList axisResult = TupleCollections.emptyList(arity);
736                for (Member m : nonAllMembers.get(cnt)) {
737                    evaluator.setContext(m);
738                    TupleIterable axis =
739                        evalExecute(
740                            nonAllMembers, cnt - 1,
741                            evaluator, queryAxis, calc);
742                    boolean ordered = false;
743                    if (queryAxis != null) {
744                        ordered = queryAxis.isOrdered();
745                    }
746                    axisResult = mergeAxes(axisResult, axis, ordered);
747                }
748                return axisResult;
749            } finally {
750                evaluator.restore(savepoint);
751            }
752        }
753    }
754
755    /**
756     * Finds all root Members 1) whose Hierarchy does not have an ALL
757     * Member, 2) whose default Member is not the ALL Member and 3)
758     * all Measures.
759     *
760     * @param nonDefaultAllMembers  List of all root Members for Hierarchies
761     * whose default Member is not the ALL Member.
762     * @param nonAllMembers List of root Members for Hierarchies that have no
763     * ALL Member.
764     * @param measureMembers  List all Measures
765     */
766    protected void loadSpecialMembers(
767        List<Member> nonDefaultAllMembers,
768        List<List<Member>> nonAllMembers,
769        List<Member> measureMembers)
770    {
771        SchemaReader schemaReader = evaluator.getSchemaReader();
772        Member[] evalMembers = evaluator.getMembers();
773        for (Member em : evalMembers) {
774            if (em.isCalculated()) {
775                continue;
776            }
777            Hierarchy h = em.getHierarchy();
778            Dimension d = h.getDimension();
779            if (d.getDimensionType() == DimensionType.TimeDimension) {
780                continue;
781            }
782            if (!em.isAll()) {
783                List<Member> rootMembers =
784                    schemaReader.getHierarchyRootMembers(h);
785                if (em.isMeasure()) {
786                    for (Member mm : rootMembers) {
787                        measureMembers.add(mm);
788                    }
789                } else {
790                    if (h.hasAll()) {
791                        for (Member m : rootMembers) {
792                            if (m.isAll()) {
793                                nonDefaultAllMembers.add(m);
794                                break;
795                            }
796                        }
797                    } else {
798                        nonAllMembers.add(rootMembers);
799                    }
800                }
801            }
802        }
803    }
804
805    protected Logger getLogger() {
806        return LOGGER;
807    }
808
809    public final RolapCube getCube() {
810        return evaluator.getCube();
811    }
812
813    // implement Result
814    public Axis[] getAxes() {
815        return axes;
816    }
817
818    /**
819     * Get the Cell for the given Cell position.
820     *
821     * @param pos Cell position.
822     * @return the Cell associated with the Cell position.
823     */
824    public Cell getCell(int[] pos) {
825        if (pos.length != point.size()) {
826            throw Util.newError(
827                "coordinates should have dimension " + point.size());
828        }
829
830        for (int i = 0; i < pos.length; i++) {
831            if (positionsHighCardinality.get(i)) {
832                final Locus locus = new Locus(execution, null, "Loading cells");
833                Locus.push(locus);
834                try {
835                    executeBody(evaluator, statement.getQuery(), pos);
836                } finally {
837                    Locus.pop(locus);
838                }
839                break;
840            }
841        }
842
843        CellInfo ci = cellInfos.lookup(pos);
844        if (ci.value == null) {
845            for (int i = 0; i < pos.length; i++) {
846                int po = pos[i];
847                if (po < 0 || po >= axes[i].getPositions().size()) {
848                    throw Util.newError("coordinates out of range");
849                }
850            }
851            ci.value = Util.nullValue;
852        }
853
854        return new RolapCell(this, pos.clone(), ci);
855    }
856
857    private TupleIterable executeAxis(
858        Evaluator evaluator,
859        QueryAxis queryAxis,
860        Calc axisCalc,
861        boolean construct,
862        AxisMemberList axisMembers)
863    {
864        if (queryAxis == null) {
865            // Create an axis containing one position with no members (not
866            // the same as an empty axis).
867            return new DelegatingTupleList(
868                0,
869                Collections.singletonList(Collections.<Member>emptyList()));
870        }
871        final int savepoint = evaluator.savepoint();
872        try {
873            evaluator.setNonEmpty(queryAxis.isNonEmpty());
874            evaluator.setEvalAxes(true);
875            final TupleIterable iterable =
876                ((IterCalc) axisCalc).evaluateIterable(evaluator);
877            if (axisCalc.getClass().getName().indexOf("OrderFunDef") != -1) {
878                queryAxis.setOrdered(true);
879            }
880            if (iterable instanceof TupleList) {
881                TupleList list = (TupleList) iterable;
882                if (construct) {
883                } else if (axisMembers != null) {
884                    axisMembers.mergeTupleList(list);
885                }
886            } else {
887                // Iterable
888                TupleCursor cursor = iterable.tupleCursor();
889                if (construct) {
890                } else if (axisMembers != null) {
891                    axisMembers.mergeTupleIter(cursor);
892                }
893            }
894            return iterable;
895        } finally {
896            evaluator.restore(savepoint);
897        }
898    }
899
900    private void executeBody(
901        RolapEvaluator evaluator,
902        Query query,
903        final int[] pos)
904    {
905        // Compute the cells several times. The first time, use a dummy
906        // evaluator which collects requests.
907        int count = 0;
908        final int savepoint = evaluator.savepoint();
909        while (true) {
910            evaluator.setCellReader(batchingReader);
911            try {
912                executeStripe(query.axes.length - 1, evaluator, pos);
913            } catch (CellRequestQuantumExceededException e) {
914                // Safe to ignore. Need to call 'phase' and loop again.
915                // Decrement count because it wasn't a recursive formula that
916                // caused the iteration.
917                --count;
918            }
919            evaluator.restore(savepoint);
920
921            // Retrieve the aggregations collected.
922            //
923            if (!phase()) {
924                // We got all of the cells we needed, so the result must be
925                // correct.
926                return;
927            } else {
928                // Clear invalid expression result so that the next evaluation
929                // will pick up the newly loaded aggregates.
930                evaluator.clearExpResultCache(false);
931            }
932
933            if (count++ > maxEvalDepth) {
934                if (evaluator instanceof RolapDependencyTestingEvaluator) {
935                    // The dependency testing evaluator can trigger new
936                    // requests every cycle. So let is run as normal for
937                    // the first N times, then run it disabled.
938                    ((RolapDependencyTestingEvaluator.DteRoot)
939                        evaluator.root).disabled = true;
940                    if (count > maxEvalDepth * 2) {
941                        throw Util.newInternal(
942                            "Query required more than " + count
943                            + " iterations");
944                    }
945                } else {
946                    throw Util.newInternal(
947                        "Query required more than " + count + " iterations");
948                }
949            }
950
951            cellInfos.clear();
952        }
953    }
954
955    boolean isDirty() {
956        return batchingReader.isDirty();
957    }
958
959    /**
960     * Evaluates an expression. Intended for evaluating named sets.
961     *
962     * <p>Does not modify the contents of the evaluator.
963     *
964     * @param calc Compiled expression
965     * @param slicerEvaluator Evaluation context for slicers
966     * @param contextEvaluator Evaluation context (optional)
967     * @return Result
968     */
969    Object evaluateExp(
970        Calc calc,
971        RolapEvaluator slicerEvaluator,
972        Evaluator contextEvaluator)
973    {
974        int attempt = 0;
975
976        RolapEvaluator evaluator = slicerEvaluator.push();
977        if (contextEvaluator != null && contextEvaluator.isEvalAxes()) {
978            evaluator.setEvalAxes(true);
979            evaluator.setContext(contextEvaluator.getMembers());
980        }
981
982        final int savepoint = evaluator.savepoint();
983        boolean dirty = batchingReader.isDirty();
984        try {
985            while (true) {
986                evaluator.restore(savepoint);
987
988                evaluator.setCellReader(batchingReader);
989                Object preliminaryValue = calc.evaluate(evaluator);
990
991                if (preliminaryValue instanceof TupleIterable) {
992                    // During the preliminary phase, we have to materialize the
993                    // tuple lists or the evaluation lower down won't take into
994                    // account all the tuples.
995                    TupleIterable iterable = (TupleIterable) preliminaryValue;
996                    final TupleCursor cursor = iterable.tupleCursor();
997                    while (cursor.forward()) {
998                        // ignore
999                    }
1000                }
1001
1002                if (!phase()) {
1003                    break;
1004                } else {
1005                    // Clear invalid expression result so that the next
1006                    // evaluation will pick up the newly loaded aggregates.
1007                    evaluator.clearExpResultCache(false);
1008                }
1009
1010                if (attempt++ > maxEvalDepth) {
1011                    throw Util.newInternal(
1012                        "Failed to load all aggregations after "
1013                        + maxEvalDepth + "passes; there's probably a cycle");
1014                }
1015            }
1016
1017            // If there were pending reads when we entered, some of the other
1018            // expressions may have been evaluated incorrectly. Set the
1019            // reader's 'dirty' flag so that the caller knows that it must
1020            // re-evaluate them.
1021            if (dirty) {
1022                batchingReader.setDirty(true);
1023            }
1024
1025            evaluator.restore(savepoint);
1026            evaluator.setCellReader(aggregatingReader);
1027            final Object o = calc.evaluate(evaluator);
1028            return o;
1029        } finally {
1030            evaluator.restore(savepoint);
1031        }
1032    }
1033
1034    private void executeStripe(
1035        int axisOrdinal,
1036        RolapEvaluator revaluator,
1037        final int[] pos)
1038    {
1039        if (axisOrdinal < 0) {
1040            RolapAxis axis = (RolapAxis) slicerAxis;
1041            TupleList tupleList = axis.getTupleList();
1042            final Iterator<List<Member>> tupleIterator = tupleList.iterator();
1043            if (tupleIterator.hasNext()) {
1044                final List<Member> members = tupleIterator.next();
1045                execution.checkCancelOrTimeout();
1046                final int savepoint = revaluator.savepoint();
1047                revaluator.setContext(members);
1048                Object o;
1049                try {
1050                    o = revaluator.evaluateCurrent();
1051                } catch (MondrianEvaluationException e) {
1052                    LOGGER.warn("Mondrian: exception in executeStripe.", e);
1053                    o = e;
1054                } finally {
1055                    revaluator.restore(savepoint);
1056                }
1057
1058                CellInfo ci = null;
1059
1060                // Get the Cell's format string and value formatting
1061                // Object.
1062                try {
1063                    // This code is a combination of the code found in
1064                    // the old RolapResult
1065                    // <code>getCellNoDefaultFormatString</code> method and
1066                    // the old RolapCell <code>getFormattedValue</code> method.
1067
1068                    // Create a CellInfo object for the given position
1069                    // integer array.
1070                    ci = cellInfos.create(point.getOrdinals());
1071
1072                    String cachedFormatString = null;
1073
1074                    // Determine if there is a CellFormatter registered for
1075                    // the current Cube's Measure's Dimension. If so,
1076                    // then find or create a CellFormatterValueFormatter
1077                    // for it. If not, then find or create a Locale based
1078                    // FormatValueFormatter.
1079                    final RolapCube cube = getCube();
1080                    Hierarchy measuresHierarchy =
1081                        cube.getMeasuresHierarchy();
1082                    RolapMeasure m =
1083                        (RolapMeasure) revaluator.getContext(measuresHierarchy);
1084                    ValueFormatter valueFormatter = m.getFormatter();
1085                    if (valueFormatter == null) {
1086                        cachedFormatString = revaluator.getFormatString();
1087                        Locale locale =
1088                            statement.getMondrianConnection().getLocale();
1089                        valueFormatter = formatValueFormatters.get(locale);
1090                        if (valueFormatter == null) {
1091                            valueFormatter = new FormatValueFormatter(locale);
1092                            formatValueFormatters.put(locale, valueFormatter);
1093                        }
1094                    }
1095
1096                    ci.formatString = cachedFormatString;
1097                    ci.valueFormatter = valueFormatter;
1098                } catch (ResultLimitExceededException e) {
1099                    // Do NOT ignore a ResultLimitExceededException!!!
1100                    throw e;
1101                } catch (CellRequestQuantumExceededException e) {
1102                    // We need to throw this so another phase happens.
1103                    throw e;
1104                } catch (MondrianEvaluationException e) {
1105                    // ignore but warn
1106                    LOGGER.warn("Mondrian: exception in executeStripe.", e);
1107                } catch (Error e) {
1108                    // Errors indicate fatal JVM problems; do not discard
1109                    throw e;
1110                } catch (Throwable e) {
1111                    LOGGER.warn("Mondrian: exception in executeStripe.", e);
1112                    Util.discard(e);
1113                }
1114
1115                if (o != RolapUtil.valueNotReadyException) {
1116                    ci.value = o;
1117                }
1118            }
1119        } else {
1120            RolapAxis axis = (RolapAxis) axes[axisOrdinal];
1121            TupleList tupleList = axis.getTupleList();
1122            Util.discard(tupleList.size()); // force materialize
1123            if (isAxisHighCardinality(axisOrdinal, tupleList)) {
1124                final int limit =
1125                    MondrianProperties.instance().HighCardChunkSize.get();
1126                if (positionsIterators.get(axisOrdinal) == null) {
1127                    final TupleCursor tupleCursor = tupleList.tupleCursor();
1128                    positionsIterators.put(axisOrdinal, tupleCursor);
1129                    positionsIndexes.put(axisOrdinal, 0);
1130                    final List<List<Member>> subPositions =
1131                        new ArrayList<List<Member>>();
1132                    for (int i = 0; i < limit && tupleCursor.forward(); i++) {
1133                        subPositions.add(tupleCursor.current());
1134                    }
1135                    positionsCurrent.put(axisOrdinal, subPositions);
1136                }
1137                final TupleCursor tupleCursor =
1138                    positionsIterators.get(axisOrdinal);
1139                final int positionIndex = positionsIndexes.get(axisOrdinal);
1140                List<List<Member>> subTuples =
1141                    positionsCurrent.get(axisOrdinal);
1142
1143                if (subTuples == null) {
1144                    return;
1145                }
1146
1147                int pi;
1148                if (pos[axisOrdinal] > positionIndex + subTuples.size() - 1
1149                        && subTuples.size() == limit)
1150                {
1151                    pi = positionIndex + subTuples.size();
1152                    positionsIndexes.put(
1153                        axisOrdinal, positionIndex + subTuples.size());
1154                    subTuples.subList(0, subTuples.size()).clear();
1155                    for (int i = 0; i < limit && tupleCursor.forward(); i++) {
1156                        subTuples.add(tupleCursor.current());
1157                    }
1158                    positionsCurrent.put(axisOrdinal, subTuples);
1159                } else {
1160                    pi = positionIndex;
1161                }
1162                for (final List<Member> tuple : subTuples) {
1163                    point.setAxis(axisOrdinal, pi);
1164                    final int savepoint = revaluator.savepoint();
1165                    try {
1166                        revaluator.setContext(tuple);
1167                        execution.checkCancelOrTimeout();
1168                        executeStripe(axisOrdinal - 1, revaluator, pos);
1169                    } finally {
1170                        revaluator.restore(savepoint);
1171                    }
1172                    pi++;
1173                }
1174            } else {
1175                for (List<Member> tuple : tupleList) {
1176                    List<Member> measures =
1177                        new ArrayList<Member>(
1178                            statement.getQuery().getMeasuresMembers());
1179                    for (Member measure : measures) {
1180                        if (measure instanceof RolapBaseCubeMeasure) {
1181                            RolapBaseCubeMeasure baseCubeMeasure =
1182                                (RolapBaseCubeMeasure) measure;
1183                            if (baseCubeMeasure.getAggregator()
1184                                == RolapAggregator.DistinctCount)
1185                            {
1186                                processDistinctMeasureExpr(
1187                                    tuple, baseCubeMeasure);
1188                            }
1189                        }
1190                    }
1191                }
1192
1193                int tupleIndex = 0;
1194                for (final List<Member> tuple : tupleList) {
1195                    point.setAxis(axisOrdinal, tupleIndex);
1196                    final int savepoint = revaluator.savepoint();
1197                    try {
1198                        revaluator.setContext(tuple);
1199                        execution.checkCancelOrTimeout();
1200                        executeStripe(axisOrdinal - 1, revaluator, pos);
1201                    } finally {
1202                        revaluator.restore(savepoint);
1203                    }
1204                    tupleIndex++;
1205                }
1206            }
1207        }
1208    }
1209
1210    private boolean isAxisHighCardinality(
1211        int axisOrdinal,
1212        TupleList tupleList)
1213    {
1214        Boolean highCardinality =
1215            positionsHighCardinality.get(axisOrdinal);
1216        if (highCardinality == null) {
1217            highCardinality = false;
1218            //noinspection LoopStatementThatDoesntLoop
1219            for (List<Member> tuple : tupleList) {
1220                if (!tuple.isEmpty()) {
1221                    highCardinality =
1222                        tuple.get(0).getDimension().isHighCardinality();
1223                }
1224                break;
1225            }
1226            positionsHighCardinality.put(axisOrdinal, highCardinality);
1227        }
1228        return highCardinality;
1229    }
1230
1231    /**
1232     * Distinct counts are aggregated separately from other measures.
1233     * We need to apply filters to each level in the query.
1234     *
1235     * <p>Replace VisualTotalMember expressions with new expressions
1236     * where all leaf level members are included.</p>
1237     *
1238     * <p>Example.
1239     * For MDX query:
1240     *
1241     * <blockquote><pre>
1242     * WITH SET [XL_Row_Dim_0] AS
1243     *         VisualTotals(
1244     *           Distinct(
1245     *             Hierarchize(
1246     *               {Ascendants([Store].[All Stores].[USA].[CA]),
1247     *                Descendants([Store].[All Stores].[USA].[CA])})))
1248     *        select NON EMPTY
1249     *          Hierarchize(
1250     *            Intersect(
1251     *              {DrilldownLevel({[Store].[All Stores]})},
1252     *              [XL_Row_Dim_0])) ON COLUMNS
1253     *        from [HR]
1254     *        where [Measures].[Number of Employees]</pre></blockquote>
1255     *
1256     * <p>For member [Store].[All Stores],
1257     * we replace aggregate expression
1258     *
1259     * <blockquote><pre>
1260     * Aggregate({[Store].[All Stores].[USA]})
1261     * </pre></blockquote>
1262     *
1263     * with
1264     *
1265     * <blockquote><pre>
1266     * Aggregate({[Store].[All Stores].[USA].[CA].[Alameda].[HQ],
1267     *               [Store].[All Stores].[USA].[CA].[Beverly Hills].[Store 6],
1268     *               [Store].[All Stores].[USA].[CA].[Los Angeles].[Store 7],
1269     *               [Store].[All Stores].[USA].[CA].[San Diego].[Store 24],
1270     *               [Store].[All Stores].[USA].[CA].[San Francisco].[Store 14]
1271     *              })
1272     * </pre></blockquote>
1273     *
1274     * <p>TODO:
1275     * Can be optimized. For that particular query
1276     * we don't need to go to the lowest level.
1277     * We can simply replace it with:
1278     * <pre>Aggregate({[Store].[All Stores].[USA].[CA]})</pre>
1279     * Because all children of [Store].[All Stores].[USA].[CA] are included.</p>
1280     */
1281    private List<Member> processDistinctMeasureExpr(
1282        List<Member> tuple,
1283        RolapBaseCubeMeasure measure)
1284    {
1285        for (Member member : tuple) {
1286            if (!(member instanceof VisualTotalMember)) {
1287                continue;
1288            }
1289            evaluator.setContext(measure);
1290            List<Member> exprMembers = new ArrayList<Member>();
1291            processMemberExpr(member, exprMembers);
1292            ((VisualTotalMember) member).setExpression(evaluator, exprMembers);
1293        }
1294        return tuple;
1295    }
1296
1297    private static void processMemberExpr(Object o, List<Member> exprMembers) {
1298        if (o instanceof Member && o instanceof RolapCubeMember) {
1299            exprMembers.add((Member) o);
1300        } else if (o instanceof VisualTotalMember) {
1301            VisualTotalMember member = (VisualTotalMember) o;
1302            Exp exp = member.getExpression();
1303            processMemberExpr(exp, exprMembers);
1304        } else if (o instanceof Exp && !(o instanceof MemberExpr)) {
1305            Exp exp = (Exp)o;
1306            ResolvedFunCall funCall = (ResolvedFunCall)exp;
1307            Exp[] exps = funCall.getArgs();
1308            processMemberExpr(exps, exprMembers);
1309        } else if (o instanceof Exp[]) {
1310            Exp[] exps = (Exp[]) o;
1311            for (Exp exp : exps) {
1312                processMemberExpr(exp, exprMembers);
1313            }
1314        } else if (o instanceof MemberExpr) {
1315            MemberExpr memberExp = (MemberExpr) o;
1316            Member member = memberExp.getMember();
1317            processMemberExpr(member, exprMembers);
1318        }
1319    }
1320
1321    /**
1322     * Converts a set of cell coordinates to a cell ordinal.
1323     *
1324     * <p>This method can be expensive, because the ordinal is computed from the
1325     * length of the axes, and therefore the axes need to be instantiated.
1326     */
1327    int getCellOrdinal(int[] pos) {
1328        if (modulos == null) {
1329            makeModulos();
1330        }
1331        return modulos.getCellOrdinal(pos);
1332    }
1333
1334    /**
1335     * Instantiates the calculator to convert cell coordinates to a cell ordinal
1336     * and vice versa.
1337     *
1338     * <p>To create the calculator, any axis that is based upon an Iterable is
1339     * converted into a List - thus increasing memory usage.
1340     */
1341    protected void makeModulos() {
1342        modulos = Modulos.Generator.create(axes);
1343    }
1344
1345    /**
1346     * Called only by RolapCell. Use this when creating an Evaluator
1347     * is not required.
1348     *
1349     * @param pos Coordinates of cell
1350     * @return Members which form the context of the given cell
1351     */
1352    RolapMember[] getCellMembers(int[] pos) {
1353        RolapMember[] members = (RolapMember[]) evaluator.getMembers().clone();
1354        for (int i = 0; i < pos.length; i++) {
1355            Position position = axes[i].getPositions().get(pos[i]);
1356            for (Member member : position) {
1357                RolapMember m = (RolapMember) member;
1358                int ordinal = m.getHierarchy().getOrdinalInCube();
1359                members[ordinal] = m;
1360            }
1361        }
1362        return members;
1363    }
1364
1365    Evaluator getRootEvaluator() {
1366        return evaluator;
1367    }
1368
1369    Evaluator getEvaluator(int[] pos) {
1370        // Set up evaluator's context, so that context-dependent format
1371        // strings work properly.
1372        Evaluator cellEvaluator = evaluator.push();
1373        populateEvaluator(cellEvaluator, pos);
1374        return cellEvaluator;
1375    }
1376
1377    void populateEvaluator(Evaluator evaluator, int[] pos) {
1378        for (int i = -1; i < axes.length; i++) {
1379            Axis axis;
1380            int index;
1381            if (i < 0) {
1382                axis = slicerAxis;
1383                if (axis.getPositions().isEmpty()) {
1384                    continue;
1385                }
1386                index = 0;
1387            } else {
1388                axis = axes[i];
1389                index = pos[i];
1390            }
1391            Position position = axis.getPositions().get(index);
1392            evaluator.setContext(position);
1393        }
1394    }
1395
1396    /**
1397     * Collection of members found on an axis.
1398     *
1399     * <p>The behavior depends on the mode (i.e. the kind of axis).
1400     * If it collects, it generally eliminates duplicates. It also has a mode
1401     * where it only counts members, does not collect them.</p>
1402     *
1403     * <p>This class does two things. First it collects all Members
1404     * found during the Member-Determination phase.
1405     * Second, it counts how many Members are on each axis and
1406     * forms the product, the totalCellCount which is checked against
1407     * the ResultLimit property value.</p>
1408     */
1409    private static class AxisMemberList implements Iterable<Member> {
1410        private final List<Member> members;
1411        private final int limit;
1412        private boolean isSlicer;
1413        private int totalCellCount;
1414        private int axisCount;
1415        private boolean countOnly;
1416
1417        AxisMemberList() {
1418            this.countOnly = false;
1419            this.members = new ConcatenableList<Member>();
1420            this.totalCellCount = 1;
1421            this.axisCount = 0;
1422            // Now that the axes are evaluated, make sure that the number of
1423            // cells does not exceed the result limit.
1424            this.limit = MondrianProperties.instance().ResultLimit.get();
1425        }
1426
1427        public Iterator<Member> iterator() {
1428            return members.iterator();
1429        }
1430
1431        void setSlicer(final boolean isSlicer) {
1432            this.isSlicer = isSlicer;
1433        }
1434
1435        boolean isEmpty() {
1436            return this.members.isEmpty();
1437        }
1438
1439        void countOnly(boolean countOnly) {
1440            this.countOnly = countOnly;
1441        }
1442
1443        void checkLimit() {
1444            if (this.limit > 0) {
1445                this.totalCellCount *= this.axisCount;
1446                if (this.totalCellCount > this.limit) {
1447                    throw MondrianResource.instance().TotalMembersLimitExceeded
1448                        .ex(
1449                            this.totalCellCount,
1450                            this.limit);
1451                }
1452                this.axisCount = 0;
1453            }
1454        }
1455
1456        void clearAxisCount() {
1457            this.axisCount = 0;
1458        }
1459
1460        void clearTotalCellCount() {
1461            this.totalCellCount = 1;
1462        }
1463
1464        void clearMembers() {
1465            this.members.clear();
1466            this.axisCount = 0;
1467            this.totalCellCount = 1;
1468        }
1469
1470        void mergeTupleList(TupleList list) {
1471            mergeTupleIter(list.tupleCursor());
1472        }
1473
1474        private void mergeTupleIter(TupleCursor cursor) {
1475            while (cursor.forward()) {
1476                mergeTuple(cursor);
1477            }
1478        }
1479
1480        private Member getTopParent(Member m) {
1481            while (true) {
1482                Member parent = m.getParentMember();
1483                if (parent == null) {
1484                    return m;
1485                }
1486                m = parent;
1487            }
1488        }
1489
1490        private void mergeTuple(final TupleCursor cursor) {
1491            final int arity = cursor.getArity();
1492            for (int i = 0; i < arity; i++) {
1493                mergeMember(cursor.member(i));
1494            }
1495        }
1496
1497        private void mergeMember(final Member member) {
1498            this.axisCount++;
1499            if (! countOnly) {
1500                if (isSlicer) {
1501                    if (! members.contains(member)) {
1502                        members.add(member);
1503                    }
1504                } else {
1505                    if (member.isNull()) {
1506                        return;
1507                    } else if (member.isMeasure()) {
1508                        return;
1509                    } else if (member.isCalculated()) {
1510                        return;
1511                    } else if (member.isAll()) {
1512                        return;
1513                    }
1514                    Member topParent = getTopParent(member);
1515                    if (! this.members.contains(topParent)) {
1516                        this.members.add(topParent);
1517                    }
1518                }
1519            }
1520        }
1521    }
1522
1523    /**
1524     * Extension to {@link RolapEvaluatorRoot} which is capable
1525     * of evaluating sets and named sets.<p/>
1526     *
1527     * A given set is only evaluated once each time a query is executed; the
1528     * result is added to the {@link #namedSetEvaluators} cache on first execution
1529     * and re-used.<p/>
1530     *
1531     * <p>Named sets are always evaluated in the context of the slicer.<p/>
1532     */
1533    protected static class RolapResultEvaluatorRoot
1534        extends RolapEvaluatorRoot
1535    {
1536        /**
1537         * Maps the names of sets to their values. Populated on demand.
1538         */
1539        private final Map<String, RolapSetEvaluator> setEvaluators =
1540            new HashMap<String, RolapSetEvaluator>();
1541        private final Map<String, RolapNamedSetEvaluator> namedSetEvaluators =
1542            new HashMap<String, RolapNamedSetEvaluator>();
1543
1544        final RolapResult result;
1545        private static final Object CycleSentinel = new Object();
1546        private static final Object NullSentinel = new Object();
1547
1548        public RolapResultEvaluatorRoot(RolapResult result) {
1549            super(result.execution);
1550            this.result = result;
1551        }
1552
1553        protected Evaluator.NamedSetEvaluator evaluateNamedSet(
1554            final NamedSet namedSet,
1555            boolean create)
1556        {
1557            final String name = namedSet.getNameUniqueWithinQuery();
1558            RolapNamedSetEvaluator value;
1559            if (namedSet.isDynamic() && !create) {
1560                value = null;
1561            } else {
1562                value = namedSetEvaluators.get(name);
1563            }
1564            if (value == null) {
1565                value = new RolapNamedSetEvaluator(this, namedSet);
1566                namedSetEvaluators.put(name, value);
1567            }
1568            return value;
1569        }
1570
1571        protected Evaluator.SetEvaluator evaluateSet(
1572            final Exp exp,
1573            boolean create)
1574        {
1575            // Sanity check: This expression HAS to return a set.
1576            if (! (exp.getType() instanceof SetType)) {
1577                throw Util.newInternal(
1578                    "Trying to evaluate set but expression does not return a set");
1579            }
1580
1581
1582            // Should be acceptable to use the string representation of the
1583            // expression as the name
1584            final String name = exp.toString();
1585            RolapSetEvaluator value;
1586
1587            // pedro, 20120914 - I don't quite understand the !create, I was
1588            // kind'a expecting the opposite here. But I'll maintain the same
1589            // logic
1590            if (!create) {
1591                value = null;
1592            } else {
1593                value = setEvaluators.get(name);
1594            }
1595            if (value == null) {
1596                value = new RolapSetEvaluator(this, exp);
1597                setEvaluators.put(name, value);
1598            }
1599            return value;
1600        }
1601
1602        public Object getParameterValue(ParameterSlot slot) {
1603            if (slot.isParameterSet()) {
1604                return slot.getParameterValue();
1605            }
1606
1607            // Look in other places for the value. Which places we look depends
1608            // on the scope of the parameter.
1609            Parameter.Scope scope = slot.getParameter().getScope();
1610            switch (scope) {
1611            case System:
1612                // TODO: implement system params
1613
1614                // fall through
1615            case Schema:
1616                // TODO: implement schema params
1617
1618                // fall through
1619            case Connection:
1620                // if it's set in the session, return that value
1621
1622                // fall through
1623            case Statement:
1624                break;
1625
1626            default:
1627                throw Util.badValue(scope);
1628            }
1629
1630            // Not set in any accessible scope. Evaluate the default value,
1631            // then cache it.
1632            Object liftedValue = slot.getCachedDefaultValue();
1633            Object value;
1634            if (liftedValue != null) {
1635                if (liftedValue == CycleSentinel) {
1636                    throw MondrianResource.instance()
1637                        .CycleDuringParameterEvaluation.ex(
1638                            slot.getParameter().getName());
1639                }
1640                if (liftedValue == NullSentinel) {
1641                    value = null;
1642                } else {
1643                    value = liftedValue;
1644                }
1645                return value;
1646            }
1647            // Set value to a sentinel, so we can detect cyclic evaluation.
1648            slot.setCachedDefaultValue(CycleSentinel);
1649            value =
1650                result.evaluateExp(
1651                    slot.getDefaultValueCalc(), result.slicerEvaluator, null);
1652            if (value == null) {
1653                liftedValue = NullSentinel;
1654            } else {
1655                liftedValue = value;
1656            }
1657            slot.setCachedDefaultValue(liftedValue);
1658            return value;
1659        }
1660    }
1661
1662    /**
1663     * Formatter to convert values into formatted strings.
1664     *
1665     * <p>Every Cell has a value, a format string (or CellFormatter) and a
1666     * formatted value string.
1667     * There are a wide range of possible values (pick a Double, any
1668     * Double - its a value). Because there are lots of possible values,
1669     * there are also lots of possible formatted value strings. On the
1670     * other hand, there are only a very small number of format strings
1671     * and CellFormatter's. These formatters are to be cached
1672     * in a synchronized HashMaps in order to limit how many copies
1673     * need to be kept around.
1674     *
1675     * <p>
1676     * There are two implementations of the ValueFormatter interface:<ul>
1677     * <li>{@link CellFormatterValueFormatter}, which formats using a
1678     * user-registered {@link CellFormatter}; and
1679     * <li> {@link FormatValueFormatter}, which takes the {@link Locale} object.
1680     * </ul>
1681     */
1682    interface ValueFormatter {
1683        /**
1684         * Formats a value according to a format string.
1685         *
1686         * @param value Value
1687         * @param formatString Format string
1688         * @return Formatted value
1689         */
1690        String format(Object value, String formatString);
1691
1692        /**
1693         * Formatter that always returns the empty string.
1694         */
1695        public static final ValueFormatter EMPTY = new ValueFormatter() {
1696            public String format(Object value, String formatString) {
1697                return "";
1698            }
1699        };
1700    }
1701
1702    /**
1703     * A CellFormatterValueFormatter uses a user-defined {@link CellFormatter}
1704     * to format values.
1705     */
1706    static class CellFormatterValueFormatter implements ValueFormatter {
1707        final CellFormatter cf;
1708
1709        /**
1710         * Creates a CellFormatterValueFormatter
1711         *
1712         * @param cf Cell formatter
1713         */
1714        CellFormatterValueFormatter(CellFormatter cf) {
1715            this.cf = cf;
1716        }
1717        public String format(Object value, String formatString) {
1718            return cf.formatCell(value);
1719        }
1720    }
1721
1722    /**
1723     * A FormatValueFormatter takes a {@link Locale}
1724     * as a parameter and uses it to get the {@link mondrian.util.Format}
1725     * to be used in formatting an Object value with a
1726     * given format string.
1727     */
1728    static class FormatValueFormatter implements ValueFormatter {
1729        final Locale locale;
1730
1731        /**
1732         * Creates a FormatValueFormatter.
1733         *
1734         * @param locale Locale
1735         */
1736        FormatValueFormatter(Locale locale) {
1737            this.locale = locale;
1738        }
1739
1740        public String format(Object value, String formatString) {
1741            if (value == Util.nullValue) {
1742                value = null;
1743            }
1744            if (value instanceof Throwable) {
1745                return "#ERR: " + value.toString();
1746            }
1747            Format format = getFormat(formatString);
1748            return format.format(value);
1749        }
1750
1751        private Format getFormat(String formatString) {
1752            return Format.get(formatString, locale);
1753        }
1754    }
1755
1756    /**
1757     * Synchronized Map from Locale to ValueFormatter. It is expected that
1758     * there will be only a small number of Locale's.
1759     * Should these be a WeakHashMap?
1760     */
1761    protected static final Map<Locale, ValueFormatter>
1762        formatValueFormatters =
1763            Collections.synchronizedMap(new HashMap<Locale, ValueFormatter>());
1764
1765    /**
1766     * A CellInfo contains all of the information that a Cell requires.
1767     * It is placed in the cellInfos map during evaluation and
1768     * serves as a constructor parameter for {@link RolapCell}.
1769     *
1770     * <p>During the evaluation stage they are mutable but after evaluation has
1771     * finished they are not changed.
1772     */
1773    static class CellInfo {
1774        Object value;
1775        String formatString;
1776        ValueFormatter valueFormatter;
1777        long key;
1778
1779        /**
1780         * Creates a CellInfo representing the position of a cell.
1781         *
1782         * @param key Ordinal representing the position of a cell
1783         */
1784        CellInfo(long key) {
1785            this(key, null, null, ValueFormatter.EMPTY);
1786        }
1787
1788        /**
1789         * Creates a CellInfo with position, value, format string and formatter
1790         * of a cell.
1791         *
1792         * @param key Ordinal representing the position of a cell
1793         * @param value Value of cell, or null if not yet known
1794         * @param formatString Format string of cell, or null
1795         * @param valueFormatter Formatter for cell, or null
1796         */
1797        CellInfo(
1798            long key,
1799            Object value,
1800            String formatString,
1801            ValueFormatter valueFormatter)
1802        {
1803            this.key = key;
1804            this.value = value;
1805            this.formatString = formatString;
1806            this.valueFormatter = valueFormatter;
1807        }
1808
1809        public int hashCode() {
1810            // Combine the upper 32 bits of the key with the lower 32 bits.
1811            // We used to use 'key ^ (key >>> 32)' but that was bad, because
1812            // CellKey.Two encodes (i, j) as
1813            // (i * Integer.MAX_VALUE + j), which is practically the same as
1814            // (i << 32, j). If i and j were
1815            // both k bits long, all of the hashcodes were k bits long too!
1816            return (int) (key ^ (key >>> 11) ^ (key >>> 24));
1817        }
1818
1819        public boolean equals(Object o) {
1820            if (o instanceof CellInfo) {
1821                CellInfo that = (CellInfo) o;
1822                return that.key == this.key;
1823            } else {
1824                return false;
1825            }
1826        }
1827
1828        /**
1829         * Returns the formatted value of the Cell
1830         * @return formatted value of the Cell
1831         */
1832        String getFormatValue() {
1833            return valueFormatter.format(value, formatString);
1834        }
1835    }
1836
1837    /**
1838     * API for the creation and
1839     * lookup of {@link CellInfo} objects. There are two implementations,
1840     * one that uses a Map for storage and the other uses an ObjectPool.
1841     */
1842    interface CellInfoContainer {
1843        /**
1844         * Returns the number of CellInfo objects in this container.
1845         * @return  the number of CellInfo objects.
1846         */
1847        int size();
1848        /**
1849         * Reduces the size of the internal data structures needed to
1850         * support the current entries. This should be called after
1851         * all CellInfo objects have been added to container.
1852         */
1853        void trimToSize();
1854        /**
1855         * Removes all CellInfo objects from container. Does not
1856         * change the size of the internal data structures.
1857         */
1858        void clear();
1859        /**
1860         * Creates a new CellInfo object, adds it to the container
1861         * a location <code>pos</code> and returns it.
1862         *
1863         * @param pos where to store CellInfo object.
1864         * @return the newly create CellInfo object.
1865         */
1866        CellInfo create(int[] pos);
1867        /**
1868         * Gets the CellInfo object at the location <code>pos</code>.
1869         *
1870         * @param pos where to find the CellInfo object.
1871         * @return the CellInfo found or null.
1872         */
1873        CellInfo lookup(int[] pos);
1874    }
1875
1876    /**
1877     * Implementation of {@link CellInfoContainer} which uses a {@link Map} to
1878     * store CellInfo Objects.
1879     *
1880     * <p>Note that the CellKey point instance variable is the same
1881     * Object (NOT a copy) that is used and modified during
1882     * the recursive calls to executeStripe - the
1883     * <code>create</code> method relies on this fact.
1884     */
1885    static class CellInfoMap implements CellInfoContainer {
1886        private final Map<CellKey, CellInfo> cellInfoMap;
1887        private final CellKey point;
1888
1889        /**
1890         * Creates a CellInfoMap
1891         *
1892         * @param point Cell position
1893         */
1894        CellInfoMap(CellKey point) {
1895            this.point = point;
1896            this.cellInfoMap = new HashMap<CellKey, CellInfo>();
1897        }
1898        public int size() {
1899            return this.cellInfoMap.size();
1900        }
1901        public void trimToSize() {
1902            // empty
1903        }
1904        public void clear() {
1905            this.cellInfoMap.clear();
1906        }
1907        public CellInfo create(int[] pos) {
1908            CellKey key = this.point.copy();
1909            CellInfo ci = this.cellInfoMap.get(key);
1910            if (ci == null) {
1911                ci = new CellInfo(0);
1912                this.cellInfoMap.put(key, ci);
1913            }
1914            return ci;
1915        }
1916        public CellInfo lookup(int[] pos) {
1917            CellKey key = CellKey.Generator.newCellKey(pos);
1918            return this.cellInfoMap.get(key);
1919        }
1920    }
1921
1922    /**
1923     * Implementation of {@link CellInfoContainer} which uses an
1924     * {@link ObjectPool} to store {@link CellInfo} Objects.
1925     *
1926     * <p>There is an inner interface (<code>CellKeyMaker</code>) and
1927     * implementations for 0 through 4 axes that convert the Cell
1928     * position integer array into a long.
1929     *
1930     * <p>
1931     * It should be noted that there is an alternate approach.
1932     * As the <code>executeStripe</code>
1933     * method is recursively called, at each call it is known which
1934     * axis is being iterated across and it is known whether or
1935     * not the Position object for that axis is a List or just
1936     * an Iterable. It it is a List, then one knows the real
1937     * size of the axis. If it is an Iterable, then one has to
1938     * use one of the MAX_AXIS_SIZE values. Given that this information
1939     * is available when one recursives down to the next
1940     * <code>executeStripe</code> call, the Cell ordinal, the position
1941     * integer array could converted to an <code>long</code>, could
1942     * be generated on the call stack!! Just a thought for the future.
1943     */
1944    static class CellInfoPool implements CellInfoContainer {
1945        /**
1946         * The maximum number of Members, 2,147,483,647, that can be any given
1947         * Axis when the number of Axes is 2.
1948         */
1949        protected static final long MAX_AXIS_SIZE_2 = 2147483647;
1950        /**
1951         * The maximum number of Members, 2,000,000, that can be any given
1952         * Axis when the number of Axes is 3.
1953         */
1954        protected static final long MAX_AXIS_SIZE_3 = 2000000;
1955        /**
1956         * The maximum number of Members, 50,000, that can be any given
1957         * Axis when the number of Axes is 4.
1958         */
1959        protected static final long MAX_AXIS_SIZE_4 = 50000;
1960
1961        /**
1962         * Implementations of CellKeyMaker convert the Cell
1963         * position integer array to a <code>long</code>.
1964         *
1965         * <p>Generates a long ordinal based upon the values of the integers
1966         * stored in the cell position array. With this mechanism, the
1967         * Cell information can be stored using a long key (rather than
1968         * the array integer of positions) thus saving memory. The trick
1969         * is to use a 'large number' per axis in order to convert from
1970         * position array to long key where the 'large number' is greater
1971         * than the number of members in the axis.
1972         * The largest 'long' is java.lang.Long.MAX_VALUE which is
1973         * 9,223,372,036,854,776,000. The product of the maximum number
1974         * of members per axis must be less than this maximum 'long'
1975         * value (otherwise one gets hashing collisions).</p>
1976         *
1977         * <p>For a single axis, the maximum number of members is equal to
1978         * the max 'long' number, 9,223,372,036,854,776,000.
1979         *
1980         * <p>For two axes, the maximum number of members is the square root
1981         * of the max 'long' number, 9,223,372,036,854,776,000, which is
1982         * slightly bigger than 2,147,483,647 (which is the maximum integer).
1983         *
1984         * <p>For three axes, the maximum number of members per axis is the
1985         * cube root of the max 'long' which is about 2,000,000.
1986         *
1987         * <p>For four axes the forth root is about 50,000.
1988         *
1989         * <p>For five or more axes, the maximum number of members per axis
1990         * based upon the root of the maximum 'long' number,
1991         * start getting too small to guarantee that it will be
1992         * smaller than the number of members on a given axis and so
1993         * we must resort to the Map-base Cell container.
1994         */
1995        interface CellKeyMaker {
1996            long generate(int[] pos);
1997        }
1998
1999        /**
2000         * For axis of size 0.
2001         */
2002        static class Zero implements CellKeyMaker {
2003            public long generate(int[] pos) {
2004                return 0;
2005            }
2006        }
2007
2008        /**
2009         * For axis of size 1.
2010         */
2011        static class One implements CellKeyMaker {
2012            public long generate(int[] pos) {
2013                return pos[0];
2014            }
2015        }
2016
2017        /**
2018         * For axis of size 2.
2019         */
2020        static class Two implements CellKeyMaker {
2021            public long generate(int[] pos) {
2022                long l = pos[0];
2023                l += (MAX_AXIS_SIZE_2 * (long) pos[1]);
2024                return l;
2025            }
2026        }
2027
2028        /**
2029         * For axis of size 3.
2030         */
2031        static class Three implements CellKeyMaker {
2032            public long generate(int[] pos) {
2033                long l = pos[0];
2034                l += (MAX_AXIS_SIZE_3 * (long) pos[1]);
2035                l += (MAX_AXIS_SIZE_3 * MAX_AXIS_SIZE_3 * (long) pos[2]);
2036                return l;
2037            }
2038        }
2039
2040        /**
2041         * For axis of size 4.
2042         */
2043        static class Four implements CellKeyMaker {
2044            public long generate(int[] pos) {
2045                long l = pos[0];
2046                l += (MAX_AXIS_SIZE_4 * (long) pos[1]);
2047                l += (MAX_AXIS_SIZE_4 * MAX_AXIS_SIZE_4 * (long) pos[2]);
2048                l += (MAX_AXIS_SIZE_4 * MAX_AXIS_SIZE_4 * MAX_AXIS_SIZE_4
2049                      * (long) pos[3]);
2050                return l;
2051            }
2052        }
2053
2054        private final ObjectPool<CellInfo> cellInfoPool;
2055        private final CellKeyMaker cellKeyMaker;
2056
2057        CellInfoPool(int axisLength) {
2058            this.cellInfoPool = new ObjectPool<CellInfo>();
2059            this.cellKeyMaker = createCellKeyMaker(axisLength);
2060        }
2061
2062        CellInfoPool(int axisLength, int initialSize) {
2063            this.cellInfoPool = new ObjectPool<CellInfo>(initialSize);
2064            this.cellKeyMaker = createCellKeyMaker(axisLength);
2065        }
2066
2067        private static CellKeyMaker createCellKeyMaker(int axisLength) {
2068            switch (axisLength) {
2069            case 0:
2070                return new Zero();
2071            case 1:
2072                return new One();
2073            case 2:
2074                return new Two();
2075            case 3:
2076                return new Three();
2077            case 4:
2078                return new Four();
2079            default:
2080                throw new RuntimeException(
2081                    "Creating CellInfoPool with axisLength=" + axisLength);
2082            }
2083        }
2084
2085        public int size() {
2086            return this.cellInfoPool.size();
2087        }
2088        public void trimToSize() {
2089            this.cellInfoPool.trimToSize();
2090        }
2091        public void clear() {
2092            this.cellInfoPool.clear();
2093        }
2094        public CellInfo create(int[] pos) {
2095            long key = this.cellKeyMaker.generate(pos);
2096            return this.cellInfoPool.add(new CellInfo(key));
2097        }
2098        public CellInfo lookup(int[] pos) {
2099            long key = this.cellKeyMaker.generate(pos);
2100            return this.cellInfoPool.add(new CellInfo(key));
2101        }
2102    }
2103
2104    static TupleList mergeAxes(
2105        TupleList axis1,
2106        TupleIterable axis2,
2107        boolean ordered)
2108    {
2109        if (axis1.isEmpty() && axis2 instanceof TupleList) {
2110            return (TupleList) axis2;
2111        }
2112        Set<List<Member>> set = new HashSet<List<Member>>();
2113        TupleList list = TupleCollections.createList(axis2.getArity());
2114        for (List<Member> tuple : axis1) {
2115            if (set.add(tuple)) {
2116                list.add(tuple);
2117            }
2118        }
2119        int halfWay = list.size();
2120        for (List<Member> tuple : axis2) {
2121            if (set.add(tuple)) {
2122                list.add(tuple);
2123            }
2124        }
2125
2126        // if there are unique members on both axes and no order function,
2127        // sort the list to ensure default order
2128        if (halfWay > 0 && halfWay < list.size() && !ordered) {
2129            list = FunUtil.hierarchizeTupleList(list, false);
2130        }
2131
2132        return list;
2133    }
2134
2135    /**
2136     * Member which holds the AggregateCalc used when evaluating
2137     * a compound slicer.  This is used to better handle some cases
2138     * where calculated members elsewhere in the query can override
2139     * the context of the slicer members.
2140     * See MONDRIAN-1226.
2141     */
2142    private class CompoundSlicerRolapMember extends DelegatingRolapMember
2143    implements RolapMeasure
2144    {
2145        private final Calc calc;
2146        private final ValueFormatter valueFormatter;
2147
2148        public CompoundSlicerRolapMember(
2149            RolapMember placeholderMember, Calc calc, ValueFormatter formatter)
2150        {
2151            super(placeholderMember);
2152            this.calc = calc;
2153            valueFormatter = formatter;
2154        }
2155
2156        @Override
2157        public boolean isEvaluated() {
2158            return true;
2159        }
2160
2161        @Override
2162        public Exp getExpression() {
2163            return new DummyExp(calc.getType());
2164        }
2165
2166        @Override
2167        public Calc getCompiledExpression(RolapEvaluatorRoot root) {
2168            return calc;
2169        }
2170
2171        @Override
2172        public int getSolveOrder() {
2173            return 0;
2174        }
2175
2176        public ValueFormatter getFormatter() {
2177            return valueFormatter;
2178        }
2179    }
2180}
2181
2182// End RolapResult.java