001/*
002 * Copyright (C) 2009-2011 Mathias Doenitz
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017package org.parboiled.parserunners;
018
019import org.parboiled.MatchHandler;
020import org.parboiled.MatcherContext;
021import org.parboiled.Rule;
022import org.parboiled.buffers.InputBuffer;
023import org.parboiled.buffers.MutableInputBuffer;
024import org.parboiled.common.ImmutableLinkedList;
025import org.parboiled.common.ImmutableList;
026import org.parboiled.common.Preconditions;
027import org.parboiled.errors.InvalidInputError;
028import org.parboiled.matchers.AbstractMatcher;
029import org.parboiled.matchers.ActionMatcher;
030import org.parboiled.matchers.EmptyMatcher;
031import org.parboiled.matchers.FirstOfMatcher;
032import org.parboiled.matchers.Matcher;
033import org.parboiled.matchers.OneOrMoreMatcher;
034import org.parboiled.matchers.SequenceMatcher;
035import org.parboiled.matchers.TestMatcher;
036import org.parboiled.matchervisitors.DefaultMatcherVisitor;
037import org.parboiled.matchervisitors.FollowMatchersVisitor;
038import org.parboiled.matchervisitors.GetStarterCharVisitor;
039import org.parboiled.matchervisitors.IsSingleCharMatcherVisitor;
040import org.parboiled.matchervisitors.IsStarterCharVisitor;
041import org.parboiled.support.Chars;
042import org.parboiled.support.MatcherPath;
043import org.parboiled.support.ParsingResult;
044import org.parboiled.matchers.AbstractMatcher;
045import org.parboiled.matchers.ActionMatcher;
046import org.parboiled.matchers.EmptyMatcher;
047import org.parboiled.matchers.FirstOfMatcher;
048import org.parboiled.matchers.Matcher;
049import org.parboiled.matchers.OneOrMoreMatcher;
050import org.parboiled.matchers.SequenceMatcher;
051import org.parboiled.matchers.TestMatcher;
052import org.parboiled.support.Chars;
053import org.parboiled.support.MatcherPath;
054import org.parboiled.support.ParsingResult;
055
056import java.util.ArrayList;
057import java.util.List;
058
059import static org.parboiled.common.Preconditions.checkArgNotNull;
060import static org.parboiled.common.Preconditions.checkState;
061import static org.parboiled.support.Chars.DEL_ERROR;
062import static org.parboiled.support.Chars.EOI;
063import static org.parboiled.support.Chars.INS_ERROR;
064import static org.parboiled.support.Chars.RESYNC;
065import static org.parboiled.support.Chars.RESYNC_END;
066import static org.parboiled.support.Chars.RESYNC_EOI;
067import static org.parboiled.support.Chars.RESYNC_START;
068
069/**
070 * A {@link ParseRunner} implementation that is able to recover from {@link InvalidInputError}s in the input and therefore
071 * report more than just the first {@link InvalidInputError} if the input does not conform to the rule grammar.
072 * Error recovery is done by attempting to either delete an error character, insert a potentially missing character
073 * or do both at once (which is equivalent to a one char replace) whereby this implementation is able to determine
074 * itself which of these options is the best strategy.
075 * If the parse error cannot be overcome by either deleting, inserting or replacing one character a resynchronization
076 * rule is determined and the parsing process resynchronized, so that parsing can still continue.
077 * In this way the RecoveringParseRunner is able to completely parse all input texts (This ParseRunner never returns
078 * an unmatched {@link ParsingResult}).
079 * If the input is error free this {@link ParseRunner} implementation will only perform one parsing run, with the same
080 * speed as the {@link BasicParseRunner}. However, if there are {@link InvalidInputError}s in the input potentially
081 * many more runs are performed to properly report all errors and test the various recovery strategies.
082 */
083public class RecoveringParseRunner<V> extends AbstractParseRunner<V> {
084    
085    public static class TimeoutException extends RuntimeException {
086        public final Rule rule;
087        public final InputBuffer inputBuffer;
088        public final ParsingResult<?> lastParsingResult;
089
090        public TimeoutException(Rule rule, InputBuffer inputBuffer, ParsingResult<?> lastParsingResult) {
091            this.rule = rule;
092            this.inputBuffer = inputBuffer;
093            this.lastParsingResult = lastParsingResult;
094        }
095    }
096
097    private final long timeoutNanos;
098    private long startTimeStamp;
099    private int errorIndex;
100    private InvalidInputError currentError;
101    private MutableInputBuffer buffer;
102    private ParsingResult<V> lastParsingResult;
103    private Matcher rootMatcherWithoutPTB; // the root matcher with parse tree building disabled
104
105    /**
106     * Create a new RecoveringParseRunner instance with the given rule and input text and returns the result of
107     * its {@link #run(String)} method invocation.
108     *
109     * @param rule  the parser rule to run
110     * @param input the input text to run on
111     * @return the ParsingResult for the parsing run
112     * @deprecated As of 0.11.0 you should use the "regular" constructor and one of the "run" methods rather than
113     *             this static method. This method will be removed in one of the coming releases.
114     */
115    @Deprecated
116    public static <V> ParsingResult<V> run(Rule rule, String input) {
117        checkArgNotNull(rule, "rule");
118        checkArgNotNull(input, "input");
119        return new RecoveringParseRunner<V>(rule).run(input);
120    }
121
122    /**
123     * Creates a new RecoveringParseRunner instance for the given rule.
124     *
125     * @param rule the parser rule
126     */
127    public RecoveringParseRunner(Rule rule) {
128        this(rule, Long.MAX_VALUE);
129    }
130    
131    /**
132     * Creates a new RecoveringParseRunner instance for the given rule.
133     * A parsing run will throw a TimeoutException if it takes longer than the given number if milliseconds. 
134     *
135     * @param rule the parser rule
136     * @param timeoutMillis the timeout value in milliseconds
137     */
138    public RecoveringParseRunner(Rule rule, long timeoutMillis) {
139        super(rule);
140        if (timeoutMillis > Long.MAX_VALUE / 1000000) {
141            this.timeoutNanos = Long.MAX_VALUE;
142        } else {
143            this.timeoutNanos = timeoutMillis * 1000000;
144        }
145    }
146
147    public ParsingResult<V> run(InputBuffer inputBuffer) {
148        checkArgNotNull(inputBuffer, "inputBuffer");
149        startTimeStamp = System.nanoTime();
150        resetValueStack();
151
152        // first, run a basic match
153        ParseRunner<V> basicRunner = new BasicParseRunner<V>(getRootMatcher())
154                .withParseErrors(getParseErrors())
155                .withValueStack(getValueStack());
156        lastParsingResult = basicRunner.run(inputBuffer);
157
158        if (!lastParsingResult.matched) {
159            // for better performance disable parse tree building during the recovery runs
160            rootMatcherWithoutPTB = (Matcher) getRootMatcher().suppressNode();
161
162            // locate first error
163            performLocatingRun(inputBuffer);
164            checkState(errorIndex >= 0); // we failed before so we must fail again
165
166            // in order to be able to apply fixes we need to wrap the input buffer with a mutability wrapper
167            buffer = new MutableInputBuffer(inputBuffer);
168
169            // report first error
170            performReportingRun();
171
172            // fix and report until done
173            while (!fixError(errorIndex)) {
174                performReportingRun();
175            }
176
177            // rerun once more with parse tree building enabled to create a parse tree for the fixed input
178            if (!getRootMatcher().isNodeSuppressed()) {
179                performFinalRun();
180                checkState(lastParsingResult.matched);
181            }
182        }
183        return lastParsingResult;
184    }
185
186    private boolean performLocatingRun(InputBuffer inputBuffer) {
187        resetValueStack();
188        ParseRunner<V> locatingRunner = new ErrorLocatingParseRunner<V>(rootMatcherWithoutPTB, getInnerHandler())
189                .withParseErrors(getParseErrors())
190                .withValueStack(getValueStack());
191        lastParsingResult = locatingRunner.run(inputBuffer);
192        errorIndex = lastParsingResult.matched ? -1 :
193                getParseErrors().remove(getParseErrors().size() - 1).getStartIndex();
194        return lastParsingResult.matched;
195    }
196
197    private void performReportingRun() {
198        resetValueStack();
199        ParseRunner<V> reportingRunner = new ErrorReportingParseRunner<V>(rootMatcherWithoutPTB, errorIndex,
200                getInnerHandler())
201                .withParseErrors(getParseErrors())
202                .withValueStack(getValueStack());
203        ParsingResult<V> result = reportingRunner.run(buffer);
204        Preconditions.checkState(!result.matched); // we failed before so we should really be failing again
205        currentError = (InvalidInputError) getParseErrors().get(getParseErrors().size() - 1);
206    }
207
208    private void performFinalRun() {
209        resetValueStack();
210        Handler handler = new Handler();
211        MatcherContext<V> rootContext = createRootContext(buffer, handler, false);
212        boolean matched = handler.match(rootContext);
213        lastParsingResult = createParsingResult(matched, rootContext);
214    }
215
216    private MatchHandler getInnerHandler() {
217        return errorIndex >= 0 ? new Handler() : null;
218    }
219
220    private boolean fixError(int fixIndex) {
221        if (tryFixBySingleCharDeletion(fixIndex)) return true;
222        int nextErrorAfterDeletion = errorIndex;
223
224        Character bestInsertionCharacter = findBestSingleCharInsertion(fixIndex);
225        if (bestInsertionCharacter == null) return true;
226        int nextErrorAfterBestInsertion = errorIndex;
227
228        Character bestReplacementCharacter = findBestSingleCharReplacement(fixIndex);
229        if (bestReplacementCharacter == null) return true;
230        int nextErrorAfterBestReplacement = errorIndex;
231
232        int nextErrorAfterBestSingleCharFix =
233                Math.max(Math.max(nextErrorAfterDeletion, nextErrorAfterBestInsertion), nextErrorAfterBestReplacement);
234        if (nextErrorAfterBestSingleCharFix > fixIndex) {
235            // we are able to overcome the error with a single char fix, so apply the best one found
236            if (nextErrorAfterBestSingleCharFix == nextErrorAfterDeletion) {
237                buffer.insertChar(fixIndex, Chars.DEL_ERROR);
238                errorIndex = nextErrorAfterDeletion + 1;
239                currentError.shiftIndexDeltaBy(1);
240            } else if (nextErrorAfterBestSingleCharFix == nextErrorAfterBestInsertion) {
241                // we need to insert the characters in reverse order, since we insert twice at the same location
242                buffer.insertChar(fixIndex, bestInsertionCharacter);
243                buffer.insertChar(fixIndex, Chars.INS_ERROR);
244                errorIndex = nextErrorAfterBestInsertion + 2;
245                currentError.shiftIndexDeltaBy(2);
246            } else {
247                // we need to insert the characters in reverse order, since we insert three times at the same location
248                buffer.insertChar(fixIndex + 1, bestReplacementCharacter);
249                buffer.insertChar(fixIndex + 1, Chars.INS_ERROR);
250                buffer.insertChar(fixIndex, Chars.DEL_ERROR);
251                errorIndex = nextErrorAfterBestReplacement + 5;
252                currentError.shiftIndexDeltaBy(1);
253            }
254        } else {
255            // we can't fix the error with a single char fix, so fall back to resynchronization
256            if (buffer.charAt(fixIndex) == Chars.EOI) {
257                buffer.insertChar(fixIndex, Chars.RESYNC_EOI);
258                currentError.shiftIndexDeltaBy(1);
259                return true;
260            }
261            buffer.insertChar(fixIndex, Chars.RESYNC);
262            currentError.shiftIndexDeltaBy(1);
263            performLocatingRun(buffer); // find the next parse error
264        }
265        return errorIndex == -1;
266    }
267
268    private boolean tryFixBySingleCharDeletion(int fixIndex) {
269        buffer.insertChar(fixIndex, Chars.DEL_ERROR);
270        boolean nowErrorFree = performLocatingRun(buffer);
271        if (nowErrorFree) {
272            currentError.shiftIndexDeltaBy(1); // compensate for the inserted DEL_ERROR char
273        } else {
274            buffer.undoCharInsertion(fixIndex);
275            errorIndex = Math.max(errorIndex - 1, 0);
276        }
277        return nowErrorFree;
278    }
279
280    @SuppressWarnings( {"ConstantConditions"})
281    private Character findBestSingleCharInsertion(int fixIndex) {
282        GetStarterCharVisitor getStarterCharVisitor = new GetStarterCharVisitor();
283        int bestNextErrorIndex = -1;
284        Character bestChar = '\u0000'; // non-null default
285        for (MatcherPath failedMatcherPath : currentError.getFailedMatchers()) {
286            Character starterChar = failedMatcherPath.element.matcher.accept(getStarterCharVisitor);
287            checkState(starterChar != null); // we should only have single character matchers
288            if (starterChar == Chars.EOI) {
289                continue; // we should never conjure up an EOI character (that would be cheating :)
290            }
291            buffer.insertChar(fixIndex, starterChar);
292            buffer.insertChar(fixIndex, Chars.INS_ERROR);
293            if (performLocatingRun(buffer)) {
294                currentError.shiftIndexDeltaBy(2); // compensate for the inserted chars
295                return null; // success, exit immediately
296            }
297            buffer.undoCharInsertion(fixIndex);
298            buffer.undoCharInsertion(fixIndex);
299            errorIndex = Math.max(errorIndex - 2, 0);
300
301            if (bestNextErrorIndex < errorIndex) {
302                bestNextErrorIndex = errorIndex;
303                bestChar = starterChar;
304            }
305        }
306        errorIndex = bestNextErrorIndex;
307        return bestChar;
308    }
309
310    private Character findBestSingleCharReplacement(int fixIndex) {
311        buffer.insertChar(fixIndex, Chars.DEL_ERROR);
312        Character bestChar = findBestSingleCharInsertion(fixIndex + 2);
313        if (bestChar == null) { // success, we found a fix that renders the complete input error free
314            currentError
315                    .shiftIndexDeltaBy(-1); // delta from DEL_ERROR char insertion and index shift by insertion method
316        } else {
317            buffer.undoCharInsertion(fixIndex);
318            errorIndex = Math.max(errorIndex - 3, 0);
319        }
320        return bestChar;
321    }
322
323    /**
324     * A {@link MatchHandler} implementation that recognizes the special
325     * {@link Chars#RESYNC} character to overcome {@link InvalidInputError}s at the respective
326     * error indices.
327     */
328    private class Handler implements MatchHandler {
329        private final IsSingleCharMatcherVisitor isSingleCharMatcherVisitor = new IsSingleCharMatcherVisitor();
330        private int fringeIndex;
331        private MatcherPath lastMatchPath;
332
333        public boolean match(MatcherContext<?> context) {
334            Matcher matcher = context.getMatcher();
335            if (matcher.accept(isSingleCharMatcherVisitor)) {
336                if (prepareErrorLocation(context) && matcher.match(context)) {
337                    if (fringeIndex < context.getCurrentIndex()) {
338                        fringeIndex = context.getCurrentIndex();
339                        lastMatchPath = context.getPath();
340                    }
341                    return true;
342                }
343                return false;
344            }
345
346            if (matcher.match(context)) {
347                return true;
348            }
349
350            // if we didn't match we might have to resynchronize
351            if (matcher instanceof SequenceMatcher) {
352                switch(context.getCurrentChar()) {
353                    case Chars.RESYNC:
354                    case Chars.RESYNC_START:
355                    case Chars.RESYNC_EOI:
356                        // however we only resynchronize if we are at a RESYNC location and the matcher is a SequenceMatcher
357                        // that has already matched at least one character and that is a parent of the last match
358                        return qualifiesForResync(context) && resynchronize(context);
359                }
360                
361                // check for timeout only on failures of sequences so as to not add too much overhead
362                if (System.nanoTime() - startTimeStamp > timeoutNanos) {
363                    throw new TimeoutException(getRootMatcher(), buffer, lastParsingResult);
364                }
365            }
366            return false;
367        }
368
369        private boolean qualifiesForResync(MatcherContext context) {
370            if (context.getCurrentIndex() == context.getStartIndex() || !context.getPath().isPrefixOf(lastMatchPath)) {
371                // if we have a sequence that hasn't match anything yet or is not a prefix we might still have to
372                // resync on it if there is no other sequence parent anymore
373                MatcherContext parent = context.getParent();
374                while (parent != null) {
375                    if (parent.getMatcher() instanceof SequenceMatcher) return false;
376                    parent = parent.getParent();
377                }
378            }            
379            return true;
380        }
381
382        private boolean prepareErrorLocation(MatcherContext context) {
383            switch (context.getCurrentChar()) {
384                case Chars.DEL_ERROR:
385                    return willMatchDelError(context);
386                case Chars.INS_ERROR:
387                    return willMatchInsError(context);
388                case Chars.RESYNC:
389                case Chars.RESYNC_START:
390                case Chars.RESYNC_EOI:
391                    return false;
392                default:
393                    return true;
394            }
395        }
396
397        private boolean willMatchDelError(MatcherContext context) {
398            int preSkipIndex = context.getCurrentIndex();
399            context.advanceIndex(2); // skip del marker char and illegal char
400            if (!runTestMatch(context)) {
401                // if we wouldn't succeed with the match do not swallow the ERROR char & Co
402                context.setCurrentIndex(preSkipIndex);
403                return false;
404            }
405            context.setStartIndex(context.getCurrentIndex());
406            if (context.getParent() != null) context.getParent().markError();
407            return true;
408        }
409
410        private boolean willMatchInsError(MatcherContext context) {
411            int preSkipIndex = context.getCurrentIndex();
412            context.advanceIndex(1); // skip ins marker char
413            if (!runTestMatch(context)) {
414                // if we wouldn't succeed with the match do not swallow the ERROR char
415                context.setCurrentIndex(preSkipIndex);
416                return false;
417            }
418            context.setStartIndex(context.getCurrentIndex());
419            context.markError();
420            return true;
421        }
422
423        private boolean runTestMatch(MatcherContext context) {
424            TestMatcher testMatcher = new TestMatcher(context.getMatcher());
425            MatcherContext testContext = testMatcher.getSubContext(context);
426            return prepareErrorLocation(testContext) && testContext.runMatcher();
427        }
428
429        private boolean resynchronize(MatcherContext context) {
430            context.markError();
431
432            // create a node for the failed Sequence, taking ownership of all sub nodes created so far
433            context.createNode();
434
435            // by resyncing we flip an unmatched sequence to a matched one, so in order to keep the value stack
436            // consistent we go into a special "error action mode" and execute the minimal set of actions underneath
437            // the resync sequence
438            rerunAndExecuteErrorActions(context);
439            
440            // skip over all characters that are not legal followers of the failed Sequence
441            switch (context.getCurrentChar()) {
442                case Chars.RESYNC:
443                    // this RESYNC error is the last error, we establish the length of the bad sequence and
444                    // change this RESYNC marker to a RESYNC_START / RESYNC_END block
445                    context.advanceIndex(1); // gobble RESYNC marker
446                    List<Matcher> followMatchers = new FollowMatchersVisitor().getFollowMatchers(context);
447                    int endIndex = gobbleIllegalCharacters(context, followMatchers);
448                    currentError.setEndIndex(endIndex);
449                    buffer.replaceInsertedChar(currentError.getStartIndex() - 1, Chars.RESYNC_START);
450                    buffer.insertChar(endIndex, Chars.RESYNC_END);
451                    context.advanceIndex(1); // gobble RESYNC_END marker
452                    break;
453
454                case Chars.RESYNC_START:
455                    // a RESYNC error we have already recovered from before
456                    context.advanceIndex(1); // gobble RESYNC_START
457                    while (context.getCurrentChar() != Chars.RESYNC_END) {
458                        context.advanceIndex(1); // skip all characters up to the RESYNC_END
459                        checkState(context.getCurrentChar() != Chars.EOI); // we MUST find a RESYNC_END before EOI
460                    }
461                    context.advanceIndex(1); // gobble RESYNC_END marker
462                    break;
463                
464                case Chars.RESYNC_EOI:
465                    // if we are resyncing on EOI we don't swallow anything
466                    // we also do not have to update the currentError since we only hit this code here
467                    // in the final run
468                    break;
469
470                default:
471                    throw new IllegalStateException();
472            }
473
474            return true;
475        }
476
477        @SuppressWarnings( {"ConstantConditions"})
478        private void rerunAndExecuteErrorActions(MatcherContext context) {
479            // the context is for the resync action, which at this point has FAILED, i.e. ALL its sub actions haven't
480            // had a chance to change the value stack, even the ones having run before the actual parse error matcher
481            // so we need to rerun all sub matchers of the resync sequence up to the point of the parse error
482            // and then run the minimal set of action in "error action mode"
483
484            int savedCurrentIndex = context.getCurrentIndex();
485            context.setCurrentIndex(context.getStartIndex()); // restart matching the resync sequence
486
487            boolean preError = true;
488            for (Matcher child : context.getMatcher().getChildren()) {
489                if (preError && !child.getSubContext(context).runMatcher()) {
490                    // run what will be the preceding matcher of all error actions
491                    new EmptyMatcher().getSubContext(context).runMatcher();
492                    context.setIntTag(1); // signal that at least one rule has run before the error actions
493                    preError = false;
494                }
495                if (!preError) {
496                    context.setInErrorRecovery(true);
497                    List<ActionMatcher> errorActions = child.accept(new CollectResyncActionsVisitor());
498                    checkState(errorActions != null);
499                    for (ActionMatcher errorAction : errorActions) {
500                        // execute the error actions without looking at their boolean results !!!
501                        errorAction.getSubContext(context).runMatcher();
502                    }
503                    context.setInErrorRecovery(false);
504                }
505            }
506
507            context.setCurrentIndex(savedCurrentIndex);
508        }
509
510        private int gobbleIllegalCharacters(MatcherContext context, List<Matcher> followMatchers) {
511            while_loop:
512            while (true) {
513                char currentChar = context.getCurrentChar();
514                if (currentChar == Chars.EOI) break;
515                for (Matcher followMatcher : followMatchers) {
516                    if (followMatcher.accept(new IsStarterCharVisitor(currentChar))) {
517                        break while_loop;
518                    }
519                }
520                context.advanceIndex(1);
521            }
522            return context.getCurrentIndex();
523        }
524    }
525
526    /**
527     * This MatcherVisitor collects the minimal set of actions that has to run underneath a resyncronization sequence
528     * in order to maintain a consistent Value Stack state.
529     */
530    private static class CollectResyncActionsVisitor extends DefaultMatcherVisitor<List<ActionMatcher>> {
531        private ImmutableLinkedList<SequenceMatcher> path = ImmutableLinkedList.nil();
532
533        @Override
534        public List<ActionMatcher> visit(ActionMatcher matcher) {
535            return ImmutableList.of(matcher);
536        }
537
538        @Override
539        public List<ActionMatcher> visit(FirstOfMatcher matcher) {
540            for (Matcher child : matcher.getChildren()) {
541                List<ActionMatcher> actions = child.accept(this);
542                if (actions != null) return actions;
543            }
544            return null;
545        }
546
547        @Override
548        public List<ActionMatcher> visit(OneOrMoreMatcher matcher) {
549            return matcher.subMatcher.accept(this);
550        }
551
552        @Override
553        public List<ActionMatcher> visit(SequenceMatcher matcher) {
554            if (path.contains(matcher)) {
555                return null;
556            }
557
558            ImmutableLinkedList<SequenceMatcher> previousPath = path;
559            path = path.prepend(matcher);
560
561            List<ActionMatcher> actions = new ArrayList<ActionMatcher>();
562            for (Matcher sub : matcher.getChildren()) {
563                List<ActionMatcher> subActions = sub.accept(this);
564                if (subActions == null) return null;
565                actions.addAll(subActions);
566            }
567
568            path = previousPath;
569            return actions;
570        }
571
572        @Override
573        public List<ActionMatcher> defaultValue(AbstractMatcher matcher) {
574            return ImmutableList.of();
575        }
576    }
577}