001/* 002 * Copyright (C) 2009-2011 Mathias Doenitz 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016 017package org.parboiled.parserunners; 018 019import org.parboiled.MatchHandler; 020import org.parboiled.MatcherContext; 021import org.parboiled.Rule; 022import org.parboiled.buffers.InputBuffer; 023import org.parboiled.buffers.MutableInputBuffer; 024import org.parboiled.common.ImmutableLinkedList; 025import org.parboiled.common.ImmutableList; 026import org.parboiled.common.Preconditions; 027import org.parboiled.errors.InvalidInputError; 028import org.parboiled.matchers.AbstractMatcher; 029import org.parboiled.matchers.ActionMatcher; 030import org.parboiled.matchers.EmptyMatcher; 031import org.parboiled.matchers.FirstOfMatcher; 032import org.parboiled.matchers.Matcher; 033import org.parboiled.matchers.OneOrMoreMatcher; 034import org.parboiled.matchers.SequenceMatcher; 035import org.parboiled.matchers.TestMatcher; 036import org.parboiled.matchervisitors.DefaultMatcherVisitor; 037import org.parboiled.matchervisitors.FollowMatchersVisitor; 038import org.parboiled.matchervisitors.GetStarterCharVisitor; 039import org.parboiled.matchervisitors.IsSingleCharMatcherVisitor; 040import org.parboiled.matchervisitors.IsStarterCharVisitor; 041import org.parboiled.support.Chars; 042import org.parboiled.support.MatcherPath; 043import org.parboiled.support.ParsingResult; 044import org.parboiled.matchers.AbstractMatcher; 045import org.parboiled.matchers.ActionMatcher; 046import org.parboiled.matchers.EmptyMatcher; 047import org.parboiled.matchers.FirstOfMatcher; 048import org.parboiled.matchers.Matcher; 049import org.parboiled.matchers.OneOrMoreMatcher; 050import org.parboiled.matchers.SequenceMatcher; 051import org.parboiled.matchers.TestMatcher; 052import org.parboiled.support.Chars; 053import org.parboiled.support.MatcherPath; 054import org.parboiled.support.ParsingResult; 055 056import java.util.ArrayList; 057import java.util.List; 058 059import static org.parboiled.common.Preconditions.checkArgNotNull; 060import static org.parboiled.common.Preconditions.checkState; 061import static org.parboiled.support.Chars.DEL_ERROR; 062import static org.parboiled.support.Chars.EOI; 063import static org.parboiled.support.Chars.INS_ERROR; 064import static org.parboiled.support.Chars.RESYNC; 065import static org.parboiled.support.Chars.RESYNC_END; 066import static org.parboiled.support.Chars.RESYNC_EOI; 067import static org.parboiled.support.Chars.RESYNC_START; 068 069/** 070 * A {@link ParseRunner} implementation that is able to recover from {@link InvalidInputError}s in the input and therefore 071 * report more than just the first {@link InvalidInputError} if the input does not conform to the rule grammar. 072 * Error recovery is done by attempting to either delete an error character, insert a potentially missing character 073 * or do both at once (which is equivalent to a one char replace) whereby this implementation is able to determine 074 * itself which of these options is the best strategy. 075 * If the parse error cannot be overcome by either deleting, inserting or replacing one character a resynchronization 076 * rule is determined and the parsing process resynchronized, so that parsing can still continue. 077 * In this way the RecoveringParseRunner is able to completely parse all input texts (This ParseRunner never returns 078 * an unmatched {@link ParsingResult}). 079 * If the input is error free this {@link ParseRunner} implementation will only perform one parsing run, with the same 080 * speed as the {@link BasicParseRunner}. However, if there are {@link InvalidInputError}s in the input potentially 081 * many more runs are performed to properly report all errors and test the various recovery strategies. 082 */ 083public class RecoveringParseRunner<V> extends AbstractParseRunner<V> { 084 085 public static class TimeoutException extends RuntimeException { 086 public final Rule rule; 087 public final InputBuffer inputBuffer; 088 public final ParsingResult<?> lastParsingResult; 089 090 public TimeoutException(Rule rule, InputBuffer inputBuffer, ParsingResult<?> lastParsingResult) { 091 this.rule = rule; 092 this.inputBuffer = inputBuffer; 093 this.lastParsingResult = lastParsingResult; 094 } 095 } 096 097 private final long timeoutNanos; 098 private long startTimeStamp; 099 private int errorIndex; 100 private InvalidInputError currentError; 101 private MutableInputBuffer buffer; 102 private ParsingResult<V> lastParsingResult; 103 private Matcher rootMatcherWithoutPTB; // the root matcher with parse tree building disabled 104 105 /** 106 * Create a new RecoveringParseRunner instance with the given rule and input text and returns the result of 107 * its {@link #run(String)} method invocation. 108 * 109 * @param rule the parser rule to run 110 * @param input the input text to run on 111 * @return the ParsingResult for the parsing run 112 * @deprecated As of 0.11.0 you should use the "regular" constructor and one of the "run" methods rather than 113 * this static method. This method will be removed in one of the coming releases. 114 */ 115 @Deprecated 116 public static <V> ParsingResult<V> run(Rule rule, String input) { 117 checkArgNotNull(rule, "rule"); 118 checkArgNotNull(input, "input"); 119 return new RecoveringParseRunner<V>(rule).run(input); 120 } 121 122 /** 123 * Creates a new RecoveringParseRunner instance for the given rule. 124 * 125 * @param rule the parser rule 126 */ 127 public RecoveringParseRunner(Rule rule) { 128 this(rule, Long.MAX_VALUE); 129 } 130 131 /** 132 * Creates a new RecoveringParseRunner instance for the given rule. 133 * A parsing run will throw a TimeoutException if it takes longer than the given number if milliseconds. 134 * 135 * @param rule the parser rule 136 * @param timeoutMillis the timeout value in milliseconds 137 */ 138 public RecoveringParseRunner(Rule rule, long timeoutMillis) { 139 super(rule); 140 if (timeoutMillis > Long.MAX_VALUE / 1000000) { 141 this.timeoutNanos = Long.MAX_VALUE; 142 } else { 143 this.timeoutNanos = timeoutMillis * 1000000; 144 } 145 } 146 147 public ParsingResult<V> run(InputBuffer inputBuffer) { 148 checkArgNotNull(inputBuffer, "inputBuffer"); 149 startTimeStamp = System.nanoTime(); 150 resetValueStack(); 151 152 // first, run a basic match 153 ParseRunner<V> basicRunner = new BasicParseRunner<V>(getRootMatcher()) 154 .withParseErrors(getParseErrors()) 155 .withValueStack(getValueStack()); 156 lastParsingResult = basicRunner.run(inputBuffer); 157 158 if (!lastParsingResult.matched) { 159 // for better performance disable parse tree building during the recovery runs 160 rootMatcherWithoutPTB = (Matcher) getRootMatcher().suppressNode(); 161 162 // locate first error 163 performLocatingRun(inputBuffer); 164 checkState(errorIndex >= 0); // we failed before so we must fail again 165 166 // in order to be able to apply fixes we need to wrap the input buffer with a mutability wrapper 167 buffer = new MutableInputBuffer(inputBuffer); 168 169 // report first error 170 performReportingRun(); 171 172 // fix and report until done 173 while (!fixError(errorIndex)) { 174 performReportingRun(); 175 } 176 177 // rerun once more with parse tree building enabled to create a parse tree for the fixed input 178 if (!getRootMatcher().isNodeSuppressed()) { 179 performFinalRun(); 180 checkState(lastParsingResult.matched); 181 } 182 } 183 return lastParsingResult; 184 } 185 186 private boolean performLocatingRun(InputBuffer inputBuffer) { 187 resetValueStack(); 188 ParseRunner<V> locatingRunner = new ErrorLocatingParseRunner<V>(rootMatcherWithoutPTB, getInnerHandler()) 189 .withParseErrors(getParseErrors()) 190 .withValueStack(getValueStack()); 191 lastParsingResult = locatingRunner.run(inputBuffer); 192 errorIndex = lastParsingResult.matched ? -1 : 193 getParseErrors().remove(getParseErrors().size() - 1).getStartIndex(); 194 return lastParsingResult.matched; 195 } 196 197 private void performReportingRun() { 198 resetValueStack(); 199 ParseRunner<V> reportingRunner = new ErrorReportingParseRunner<V>(rootMatcherWithoutPTB, errorIndex, 200 getInnerHandler()) 201 .withParseErrors(getParseErrors()) 202 .withValueStack(getValueStack()); 203 ParsingResult<V> result = reportingRunner.run(buffer); 204 Preconditions.checkState(!result.matched); // we failed before so we should really be failing again 205 currentError = (InvalidInputError) getParseErrors().get(getParseErrors().size() - 1); 206 } 207 208 private void performFinalRun() { 209 resetValueStack(); 210 Handler handler = new Handler(); 211 MatcherContext<V> rootContext = createRootContext(buffer, handler, false); 212 boolean matched = handler.match(rootContext); 213 lastParsingResult = createParsingResult(matched, rootContext); 214 } 215 216 private MatchHandler getInnerHandler() { 217 return errorIndex >= 0 ? new Handler() : null; 218 } 219 220 private boolean fixError(int fixIndex) { 221 if (tryFixBySingleCharDeletion(fixIndex)) return true; 222 int nextErrorAfterDeletion = errorIndex; 223 224 Character bestInsertionCharacter = findBestSingleCharInsertion(fixIndex); 225 if (bestInsertionCharacter == null) return true; 226 int nextErrorAfterBestInsertion = errorIndex; 227 228 Character bestReplacementCharacter = findBestSingleCharReplacement(fixIndex); 229 if (bestReplacementCharacter == null) return true; 230 int nextErrorAfterBestReplacement = errorIndex; 231 232 int nextErrorAfterBestSingleCharFix = 233 Math.max(Math.max(nextErrorAfterDeletion, nextErrorAfterBestInsertion), nextErrorAfterBestReplacement); 234 if (nextErrorAfterBestSingleCharFix > fixIndex) { 235 // we are able to overcome the error with a single char fix, so apply the best one found 236 if (nextErrorAfterBestSingleCharFix == nextErrorAfterDeletion) { 237 buffer.insertChar(fixIndex, Chars.DEL_ERROR); 238 errorIndex = nextErrorAfterDeletion + 1; 239 currentError.shiftIndexDeltaBy(1); 240 } else if (nextErrorAfterBestSingleCharFix == nextErrorAfterBestInsertion) { 241 // we need to insert the characters in reverse order, since we insert twice at the same location 242 buffer.insertChar(fixIndex, bestInsertionCharacter); 243 buffer.insertChar(fixIndex, Chars.INS_ERROR); 244 errorIndex = nextErrorAfterBestInsertion + 2; 245 currentError.shiftIndexDeltaBy(2); 246 } else { 247 // we need to insert the characters in reverse order, since we insert three times at the same location 248 buffer.insertChar(fixIndex + 1, bestReplacementCharacter); 249 buffer.insertChar(fixIndex + 1, Chars.INS_ERROR); 250 buffer.insertChar(fixIndex, Chars.DEL_ERROR); 251 errorIndex = nextErrorAfterBestReplacement + 5; 252 currentError.shiftIndexDeltaBy(1); 253 } 254 } else { 255 // we can't fix the error with a single char fix, so fall back to resynchronization 256 if (buffer.charAt(fixIndex) == Chars.EOI) { 257 buffer.insertChar(fixIndex, Chars.RESYNC_EOI); 258 currentError.shiftIndexDeltaBy(1); 259 return true; 260 } 261 buffer.insertChar(fixIndex, Chars.RESYNC); 262 currentError.shiftIndexDeltaBy(1); 263 performLocatingRun(buffer); // find the next parse error 264 } 265 return errorIndex == -1; 266 } 267 268 private boolean tryFixBySingleCharDeletion(int fixIndex) { 269 buffer.insertChar(fixIndex, Chars.DEL_ERROR); 270 boolean nowErrorFree = performLocatingRun(buffer); 271 if (nowErrorFree) { 272 currentError.shiftIndexDeltaBy(1); // compensate for the inserted DEL_ERROR char 273 } else { 274 buffer.undoCharInsertion(fixIndex); 275 errorIndex = Math.max(errorIndex - 1, 0); 276 } 277 return nowErrorFree; 278 } 279 280 @SuppressWarnings( {"ConstantConditions"}) 281 private Character findBestSingleCharInsertion(int fixIndex) { 282 GetStarterCharVisitor getStarterCharVisitor = new GetStarterCharVisitor(); 283 int bestNextErrorIndex = -1; 284 Character bestChar = '\u0000'; // non-null default 285 for (MatcherPath failedMatcherPath : currentError.getFailedMatchers()) { 286 Character starterChar = failedMatcherPath.element.matcher.accept(getStarterCharVisitor); 287 checkState(starterChar != null); // we should only have single character matchers 288 if (starterChar == Chars.EOI) { 289 continue; // we should never conjure up an EOI character (that would be cheating :) 290 } 291 buffer.insertChar(fixIndex, starterChar); 292 buffer.insertChar(fixIndex, Chars.INS_ERROR); 293 if (performLocatingRun(buffer)) { 294 currentError.shiftIndexDeltaBy(2); // compensate for the inserted chars 295 return null; // success, exit immediately 296 } 297 buffer.undoCharInsertion(fixIndex); 298 buffer.undoCharInsertion(fixIndex); 299 errorIndex = Math.max(errorIndex - 2, 0); 300 301 if (bestNextErrorIndex < errorIndex) { 302 bestNextErrorIndex = errorIndex; 303 bestChar = starterChar; 304 } 305 } 306 errorIndex = bestNextErrorIndex; 307 return bestChar; 308 } 309 310 private Character findBestSingleCharReplacement(int fixIndex) { 311 buffer.insertChar(fixIndex, Chars.DEL_ERROR); 312 Character bestChar = findBestSingleCharInsertion(fixIndex + 2); 313 if (bestChar == null) { // success, we found a fix that renders the complete input error free 314 currentError 315 .shiftIndexDeltaBy(-1); // delta from DEL_ERROR char insertion and index shift by insertion method 316 } else { 317 buffer.undoCharInsertion(fixIndex); 318 errorIndex = Math.max(errorIndex - 3, 0); 319 } 320 return bestChar; 321 } 322 323 /** 324 * A {@link MatchHandler} implementation that recognizes the special 325 * {@link Chars#RESYNC} character to overcome {@link InvalidInputError}s at the respective 326 * error indices. 327 */ 328 private class Handler implements MatchHandler { 329 private final IsSingleCharMatcherVisitor isSingleCharMatcherVisitor = new IsSingleCharMatcherVisitor(); 330 private int fringeIndex; 331 private MatcherPath lastMatchPath; 332 333 public boolean match(MatcherContext<?> context) { 334 Matcher matcher = context.getMatcher(); 335 if (matcher.accept(isSingleCharMatcherVisitor)) { 336 if (prepareErrorLocation(context) && matcher.match(context)) { 337 if (fringeIndex < context.getCurrentIndex()) { 338 fringeIndex = context.getCurrentIndex(); 339 lastMatchPath = context.getPath(); 340 } 341 return true; 342 } 343 return false; 344 } 345 346 if (matcher.match(context)) { 347 return true; 348 } 349 350 // if we didn't match we might have to resynchronize 351 if (matcher instanceof SequenceMatcher) { 352 switch(context.getCurrentChar()) { 353 case Chars.RESYNC: 354 case Chars.RESYNC_START: 355 case Chars.RESYNC_EOI: 356 // however we only resynchronize if we are at a RESYNC location and the matcher is a SequenceMatcher 357 // that has already matched at least one character and that is a parent of the last match 358 return qualifiesForResync(context) && resynchronize(context); 359 } 360 361 // check for timeout only on failures of sequences so as to not add too much overhead 362 if (System.nanoTime() - startTimeStamp > timeoutNanos) { 363 throw new TimeoutException(getRootMatcher(), buffer, lastParsingResult); 364 } 365 } 366 return false; 367 } 368 369 private boolean qualifiesForResync(MatcherContext context) { 370 if (context.getCurrentIndex() == context.getStartIndex() || !context.getPath().isPrefixOf(lastMatchPath)) { 371 // if we have a sequence that hasn't match anything yet or is not a prefix we might still have to 372 // resync on it if there is no other sequence parent anymore 373 MatcherContext parent = context.getParent(); 374 while (parent != null) { 375 if (parent.getMatcher() instanceof SequenceMatcher) return false; 376 parent = parent.getParent(); 377 } 378 } 379 return true; 380 } 381 382 private boolean prepareErrorLocation(MatcherContext context) { 383 switch (context.getCurrentChar()) { 384 case Chars.DEL_ERROR: 385 return willMatchDelError(context); 386 case Chars.INS_ERROR: 387 return willMatchInsError(context); 388 case Chars.RESYNC: 389 case Chars.RESYNC_START: 390 case Chars.RESYNC_EOI: 391 return false; 392 default: 393 return true; 394 } 395 } 396 397 private boolean willMatchDelError(MatcherContext context) { 398 int preSkipIndex = context.getCurrentIndex(); 399 context.advanceIndex(2); // skip del marker char and illegal char 400 if (!runTestMatch(context)) { 401 // if we wouldn't succeed with the match do not swallow the ERROR char & Co 402 context.setCurrentIndex(preSkipIndex); 403 return false; 404 } 405 context.setStartIndex(context.getCurrentIndex()); 406 if (context.getParent() != null) context.getParent().markError(); 407 return true; 408 } 409 410 private boolean willMatchInsError(MatcherContext context) { 411 int preSkipIndex = context.getCurrentIndex(); 412 context.advanceIndex(1); // skip ins marker char 413 if (!runTestMatch(context)) { 414 // if we wouldn't succeed with the match do not swallow the ERROR char 415 context.setCurrentIndex(preSkipIndex); 416 return false; 417 } 418 context.setStartIndex(context.getCurrentIndex()); 419 context.markError(); 420 return true; 421 } 422 423 private boolean runTestMatch(MatcherContext context) { 424 TestMatcher testMatcher = new TestMatcher(context.getMatcher()); 425 MatcherContext testContext = testMatcher.getSubContext(context); 426 return prepareErrorLocation(testContext) && testContext.runMatcher(); 427 } 428 429 private boolean resynchronize(MatcherContext context) { 430 context.markError(); 431 432 // create a node for the failed Sequence, taking ownership of all sub nodes created so far 433 context.createNode(); 434 435 // by resyncing we flip an unmatched sequence to a matched one, so in order to keep the value stack 436 // consistent we go into a special "error action mode" and execute the minimal set of actions underneath 437 // the resync sequence 438 rerunAndExecuteErrorActions(context); 439 440 // skip over all characters that are not legal followers of the failed Sequence 441 switch (context.getCurrentChar()) { 442 case Chars.RESYNC: 443 // this RESYNC error is the last error, we establish the length of the bad sequence and 444 // change this RESYNC marker to a RESYNC_START / RESYNC_END block 445 context.advanceIndex(1); // gobble RESYNC marker 446 List<Matcher> followMatchers = new FollowMatchersVisitor().getFollowMatchers(context); 447 int endIndex = gobbleIllegalCharacters(context, followMatchers); 448 currentError.setEndIndex(endIndex); 449 buffer.replaceInsertedChar(currentError.getStartIndex() - 1, Chars.RESYNC_START); 450 buffer.insertChar(endIndex, Chars.RESYNC_END); 451 context.advanceIndex(1); // gobble RESYNC_END marker 452 break; 453 454 case Chars.RESYNC_START: 455 // a RESYNC error we have already recovered from before 456 context.advanceIndex(1); // gobble RESYNC_START 457 while (context.getCurrentChar() != Chars.RESYNC_END) { 458 context.advanceIndex(1); // skip all characters up to the RESYNC_END 459 checkState(context.getCurrentChar() != Chars.EOI); // we MUST find a RESYNC_END before EOI 460 } 461 context.advanceIndex(1); // gobble RESYNC_END marker 462 break; 463 464 case Chars.RESYNC_EOI: 465 // if we are resyncing on EOI we don't swallow anything 466 // we also do not have to update the currentError since we only hit this code here 467 // in the final run 468 break; 469 470 default: 471 throw new IllegalStateException(); 472 } 473 474 return true; 475 } 476 477 @SuppressWarnings( {"ConstantConditions"}) 478 private void rerunAndExecuteErrorActions(MatcherContext context) { 479 // the context is for the resync action, which at this point has FAILED, i.e. ALL its sub actions haven't 480 // had a chance to change the value stack, even the ones having run before the actual parse error matcher 481 // so we need to rerun all sub matchers of the resync sequence up to the point of the parse error 482 // and then run the minimal set of action in "error action mode" 483 484 int savedCurrentIndex = context.getCurrentIndex(); 485 context.setCurrentIndex(context.getStartIndex()); // restart matching the resync sequence 486 487 boolean preError = true; 488 for (Matcher child : context.getMatcher().getChildren()) { 489 if (preError && !child.getSubContext(context).runMatcher()) { 490 // run what will be the preceding matcher of all error actions 491 new EmptyMatcher().getSubContext(context).runMatcher(); 492 context.setIntTag(1); // signal that at least one rule has run before the error actions 493 preError = false; 494 } 495 if (!preError) { 496 context.setInErrorRecovery(true); 497 List<ActionMatcher> errorActions = child.accept(new CollectResyncActionsVisitor()); 498 checkState(errorActions != null); 499 for (ActionMatcher errorAction : errorActions) { 500 // execute the error actions without looking at their boolean results !!! 501 errorAction.getSubContext(context).runMatcher(); 502 } 503 context.setInErrorRecovery(false); 504 } 505 } 506 507 context.setCurrentIndex(savedCurrentIndex); 508 } 509 510 private int gobbleIllegalCharacters(MatcherContext context, List<Matcher> followMatchers) { 511 while_loop: 512 while (true) { 513 char currentChar = context.getCurrentChar(); 514 if (currentChar == Chars.EOI) break; 515 for (Matcher followMatcher : followMatchers) { 516 if (followMatcher.accept(new IsStarterCharVisitor(currentChar))) { 517 break while_loop; 518 } 519 } 520 context.advanceIndex(1); 521 } 522 return context.getCurrentIndex(); 523 } 524 } 525 526 /** 527 * This MatcherVisitor collects the minimal set of actions that has to run underneath a resyncronization sequence 528 * in order to maintain a consistent Value Stack state. 529 */ 530 private static class CollectResyncActionsVisitor extends DefaultMatcherVisitor<List<ActionMatcher>> { 531 private ImmutableLinkedList<SequenceMatcher> path = ImmutableLinkedList.nil(); 532 533 @Override 534 public List<ActionMatcher> visit(ActionMatcher matcher) { 535 return ImmutableList.of(matcher); 536 } 537 538 @Override 539 public List<ActionMatcher> visit(FirstOfMatcher matcher) { 540 for (Matcher child : matcher.getChildren()) { 541 List<ActionMatcher> actions = child.accept(this); 542 if (actions != null) return actions; 543 } 544 return null; 545 } 546 547 @Override 548 public List<ActionMatcher> visit(OneOrMoreMatcher matcher) { 549 return matcher.subMatcher.accept(this); 550 } 551 552 @Override 553 public List<ActionMatcher> visit(SequenceMatcher matcher) { 554 if (path.contains(matcher)) { 555 return null; 556 } 557 558 ImmutableLinkedList<SequenceMatcher> previousPath = path; 559 path = path.prepend(matcher); 560 561 List<ActionMatcher> actions = new ArrayList<ActionMatcher>(); 562 for (Matcher sub : matcher.getChildren()) { 563 List<ActionMatcher> subActions = sub.accept(this); 564 if (subActions == null) return null; 565 actions.addAll(subActions); 566 } 567 568 path = previousPath; 569 return actions; 570 } 571 572 @Override 573 public List<ActionMatcher> defaultValue(AbstractMatcher matcher) { 574 return ImmutableList.of(); 575 } 576 } 577}