// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html // // file: rbbiscan.cpp // // Copyright (C) 2002-2016, International Business Machines Corporation and others. // All Rights Reserved. // // This file contains the Rule Based Break Iterator Rule Builder functions for // scanning the rules and assembling a parse tree. This is the first phase // of compiling the rules. // // The overall of the rules is managed by class RBBIRuleBuilder, which will // create and use an instance of this class as part of the process. // #include "unicode/utypes.h" #if !UCONFIG_NO_BREAK_ITERATION #include "unicode/unistr.h" #include "unicode/uniset.h" #include "unicode/uchar.h" #include "unicode/uchriter.h" #include "unicode/parsepos.h" #include "unicode/parseerr.h" #include "cmemory.h" #include "cstring.h" #include "rbbirpt.h" // Contains state table for the rbbi rules parser. // generated by a Perl script. #include "rbbirb.h" #include "rbbinode.h" #include "rbbiscan.h" #include "rbbitblb.h" #include "uassert.h" //------------------------------------------------------------------------------ // // Unicode Set init strings for each of the character classes needed for parsing a rule file. // (Initialized with hex values for portability to EBCDIC based machines. // Really ugly, but there's no good way to avoid it.) // // The sets are referred to by name in the rbbirpt.txt, which is the // source form of the state transition table for the RBBI rule parser. // //------------------------------------------------------------------------------ static const UChar gRuleSet_rule_char_pattern[] = { // Characters that may appear as literals in patterns without escaping or quoting. // [ ^ [ \ p { Z } \ u 0 0 2 0 0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30, // - \ u 0 0 7 f ] - [ \ p 0x2d, 0x5c, 0x75, 0x30, 0x30, 0x37, 0x66, 0x5d, 0x2d, 0x5b, 0x5c, 0x70, // { L } ] - [ \ p { N } ] ] 0x7b, 0x4c, 0x7d, 0x5d, 0x2d, 0x5b, 0x5c, 0x70, 0x7b, 0x4e, 0x7d, 0x5d, 0x5d, 0}; static const UChar gRuleSet_name_char_pattern[] = { // [ _ \ p { L } \ p { N } ] 0x5b, 0x5f, 0x5c, 0x70, 0x7b, 0x4c, 0x7d, 0x5c, 0x70, 0x7b, 0x4e, 0x7d, 0x5d, 0}; static const UChar gRuleSet_digit_char_pattern[] = { // [ 0 - 9 ] 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0}; static const UChar gRuleSet_name_start_char_pattern[] = { // [ _ \ p { L } ] 0x5b, 0x5f, 0x5c, 0x70, 0x7b, 0x4c, 0x7d, 0x5d, 0 }; static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00}; // "any" U_CDECL_BEGIN static void U_CALLCONV RBBISetTable_deleter(void *p) { icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p; delete px->key; // Note: px->val is owned by the linked list "fSetsListHead" in scanner. // Don't delete the value nodes here. uprv_free(px); } U_CDECL_END U_NAMESPACE_BEGIN //------------------------------------------------------------------------------ // // Constructor. // //------------------------------------------------------------------------------ RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) { fRB = rb; fScanIndex = 0; fNextIndex = 0; fQuoteMode = false; fLineNum = 1; fCharNum = 0; fLastChar = 0; fStateTable = NULL; fStack[0] = 0; fStackPtr = 0; fNodeStack[0] = NULL; fNodeStackPtr = 0; fReverseRule = false; fLookAheadRule = false; fNoChainInRule = false; fSymbolTable = NULL; fSetTable = NULL; fRuleNum = 0; fOptionStart = 0; // Do not check status until after all critical fields are sufficiently initialized // that the destructor can run cleanly. if (U_FAILURE(*rb->fStatus)) { return; } // // Set up the constant Unicode Sets. // Note: These could be made static, lazily initialized, and shared among // all instances of RBBIRuleScanners. BUT this is quite a bit simpler, // and the time to build these few sets should be small compared to a // full break iterator build. fRuleSets[kRuleSet_rule_char-128] = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern), *rb->fStatus); // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:] fRuleSets[kRuleSet_white_space-128]. add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029); fRuleSets[kRuleSet_name_char-128] = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern), *rb->fStatus); fRuleSets[kRuleSet_name_start_char-128] = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus); fRuleSets[kRuleSet_digit_char-128] = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern), *rb->fStatus); if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) { // This case happens if ICU's data is missing. UnicodeSet tries to look up property // names from the init string, can't find them, and claims an illegal argument. // Change the error so that the actual problem will be clearer to users. *rb->fStatus = U_BRK_INIT_ERROR; } if (U_FAILURE(*rb->fStatus)) { return; } fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus); if (fSymbolTable == NULL) { *rb->fStatus = U_MEMORY_ALLOCATION_ERROR; return; } fSetTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, rb->fStatus); if (U_FAILURE(*rb->fStatus)) { return; } uhash_setValueDeleter(fSetTable, RBBISetTable_deleter); } //------------------------------------------------------------------------------ // // Destructor // //------------------------------------------------------------------------------ RBBIRuleScanner::~RBBIRuleScanner() { delete fSymbolTable; if (fSetTable != NULL) { uhash_close(fSetTable); fSetTable = NULL; } // Node Stack. // Normally has one entry, which is the entire parse tree for the rules. // If errors occurred, there may be additional subtrees left on the stack. while (fNodeStackPtr > 0) { delete fNodeStack[fNodeStackPtr]; fNodeStackPtr--; } } //------------------------------------------------------------------------------ // // doParseAction Do some action during rule parsing. // Called by the parse state machine. // Actions build the parse tree and Unicode Sets, // and maintain the parse stack for nested expressions. // // TODO: unify EParseAction and RBBI_RuleParseAction enum types. // They represent exactly the same thing. They're separate // only to work around enum forward declaration restrictions // in some compilers, while at the same time avoiding multiple // definitions problems. I'm sure that there's a better way. // //------------------------------------------------------------------------------ UBool RBBIRuleScanner::doParseActions(int32_t action) { RBBINode *n = NULL; UBool returnVal = true; switch (action) { case doExprStart: pushNewNode(RBBINode::opStart); fRuleNum++; break; case doNoChain: // Scanned a '^' while on the rule start state. fNoChainInRule = true; break; case doExprOrOperator: { fixOpStack(RBBINode::precOpCat); RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *orNode = pushNewNode(RBBINode::opOr); if (U_FAILURE(*fRB->fStatus)) { break; } orNode->fLeftChild = operandNode; operandNode->fParent = orNode; } break; case doExprCatOperator: // concatenation operator. // For the implicit concatenation of adjacent terms in an expression that are // not separated by any other operator. Action is invoked between the // actions for the two terms. { fixOpStack(RBBINode::precOpCat); RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *catNode = pushNewNode(RBBINode::opCat); if (U_FAILURE(*fRB->fStatus)) { break; } catNode->fLeftChild = operandNode; operandNode->fParent = catNode; } break; case doLParen: // Open Paren. // The openParen node is a dummy operation type with a low precedence, // which has the affect of ensuring that any real binary op that // follows within the parens binds more tightly to the operands than // stuff outside of the parens. pushNewNode(RBBINode::opLParen); break; case doExprRParen: fixOpStack(RBBINode::precLParen); break; case doNOP: break; case doStartAssign: // We've just scanned "$variable = " // The top of the node stack has the $variable ref node. // Save the start position of the RHS text in the StartExpression node // that precedes the $variableReference node on the stack. // This will eventually be used when saving the full $variable replacement // text as a string. n = fNodeStack[fNodeStackPtr-1]; n->fFirstPos = fNextIndex; // move past the '=' // Push a new start-of-expression node; needed to keep parse of the // RHS expression happy. pushNewNode(RBBINode::opStart); break; case doEndAssign: { // We have reached the end of an assignment statement. // Current scan char is the ';' that terminates the assignment. // Terminate expression, leaves expression parse tree rooted in TOS node. fixOpStack(RBBINode::precStart); RBBINode *startExprNode = fNodeStack[fNodeStackPtr-2]; RBBINode *varRefNode = fNodeStack[fNodeStackPtr-1]; RBBINode *RHSExprNode = fNodeStack[fNodeStackPtr]; // Save original text of right side of assignment, excluding the terminating ';' // in the root of the node for the right-hand-side expression. RHSExprNode->fFirstPos = startExprNode->fFirstPos; RHSExprNode->fLastPos = fScanIndex; fRB->fRules.extractBetween(RHSExprNode->fFirstPos, RHSExprNode->fLastPos, RHSExprNode->fText); // Expression parse tree becomes l. child of the $variable reference node. varRefNode->fLeftChild = RHSExprNode; RHSExprNode->fParent = varRefNode; // Make a symbol table entry for the $variableRef node. fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus); if (U_FAILURE(*fRB->fStatus)) { // This is a round-about way to get the parse position set // so that duplicate symbols error messages include a line number. UErrorCode t = *fRB->fStatus; *fRB->fStatus = U_ZERO_ERROR; error(t); } // Clean up the stack. delete startExprNode; fNodeStackPtr-=3; break; } case doEndOfRule: { fixOpStack(RBBINode::precStart); // Terminate expression, leaves expression if (U_FAILURE(*fRB->fStatus)) { // parse tree rooted in TOS node. break; } #ifdef RBBI_DEBUG if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");} #endif U_ASSERT(fNodeStackPtr == 1); RBBINode *thisRule = fNodeStack[fNodeStackPtr]; // If this rule includes a look-ahead '/', add a endMark node to the // expression tree. if (fLookAheadRule) { RBBINode *endNode = pushNewNode(RBBINode::endMark); RBBINode *catNode = pushNewNode(RBBINode::opCat); if (U_FAILURE(*fRB->fStatus)) { break; } fNodeStackPtr -= 2; catNode->fLeftChild = thisRule; catNode->fRightChild = endNode; fNodeStack[fNodeStackPtr] = catNode; endNode->fVal = fRuleNum; endNode->fLookAheadEnd = true; thisRule = catNode; // TODO: Disable chaining out of look-ahead (hard break) rules. // The break on rule match is forced, so there is no point in building up // the state table to chain into another rule for a longer match. } // Mark this node as being the root of a rule. thisRule->fRuleRoot = true; // Flag if chaining into this rule is wanted. // if (fRB->fChainRules && // If rule chaining is enabled globally via !!chain !fNoChainInRule) { // and no '^' chain-in inhibit was on this rule thisRule->fChainIn = true; } // All rule expressions are ORed together. // The ';' that terminates an expression really just functions as a '|' with // a low operator prededence. // // Each of the four sets of rules are collected separately. // (forward, reverse, safe_forward, safe_reverse) // OR this rule into the appropriate group of them. // RBBINode **destRules = (fReverseRule? &fRB->fSafeRevTree : fRB->fDefaultTree); if (*destRules != NULL) { // This is not the first rule encountered. // OR previous stuff (from *destRules) // with the current rule expression (on the Node Stack) // with the resulting OR expression going to *destRules // thisRule = fNodeStack[fNodeStackPtr]; RBBINode *prevRules = *destRules; RBBINode *orNode = pushNewNode(RBBINode::opOr); if (U_FAILURE(*fRB->fStatus)) { break; } orNode->fLeftChild = prevRules; prevRules->fParent = orNode; orNode->fRightChild = thisRule; thisRule->fParent = orNode; *destRules = orNode; } else { // This is the first rule encountered (for this direction). // Just move its parse tree from the stack to *destRules. *destRules = fNodeStack[fNodeStackPtr]; } fReverseRule = false; // in preparation for the next rule. fLookAheadRule = false; fNoChainInRule = false; fNodeStackPtr = 0; } break; case doRuleError: error(U_BRK_RULE_SYNTAX); returnVal = false; break; case doVariableNameExpectedErr: error(U_BRK_RULE_SYNTAX); break; // // Unary operands + ? * // These all appear after the operand to which they apply. // When we hit one, the operand (may be a whole sub expression) // will be on the top of the stack. // Unary Operator becomes TOS, with the old TOS as its one child. case doUnaryOpPlus: { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *plusNode = pushNewNode(RBBINode::opPlus); if (U_FAILURE(*fRB->fStatus)) { break; } plusNode->fLeftChild = operandNode; operandNode->fParent = plusNode; } break; case doUnaryOpQuestion: { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *qNode = pushNewNode(RBBINode::opQuestion); if (U_FAILURE(*fRB->fStatus)) { break; } qNode->fLeftChild = operandNode; operandNode->fParent = qNode; } break; case doUnaryOpStar: { RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; RBBINode *starNode = pushNewNode(RBBINode::opStar); if (U_FAILURE(*fRB->fStatus)) { break; } starNode->fLeftChild = operandNode; operandNode->fParent = starNode; } break; case doRuleChar: // A "Rule Character" is any single character that is a literal part // of the regular expression. Like a, b and c in the expression "(abc*) | [:L:]" // These are pretty uncommon in break rules; the terms are more commonly // sets. To keep things uniform, treat these characters like as // sets that just happen to contain only one character. { n = pushNewNode(RBBINode::setRef); if (U_FAILURE(*fRB->fStatus)) { break; } findSetFor(UnicodeString(fC.fChar), n); n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); break; } case doDotAny: // scanned a ".", meaning match any single character. { n = pushNewNode(RBBINode::setRef); if (U_FAILURE(*fRB->fStatus)) { break; } findSetFor(UnicodeString(true, kAny, 3), n); n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); break; } case doSlash: // Scanned a '/', which identifies a look-ahead break position in a rule. n = pushNewNode(RBBINode::lookAhead); if (U_FAILURE(*fRB->fStatus)) { break; } n->fVal = fRuleNum; n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); fLookAheadRule = true; break; case doStartTagValue: // Scanned a '{', the opening delimiter for a tag value within a rule. n = pushNewNode(RBBINode::tag); if (U_FAILURE(*fRB->fStatus)) { break; } n->fVal = 0; n->fFirstPos = fScanIndex; n->fLastPos = fNextIndex; break; case doTagDigit: // Just scanned a decimal digit that's part of a tag value { n = fNodeStack[fNodeStackPtr]; uint32_t v = u_charDigitValue(fC.fChar); U_ASSERT(v < 10); n->fVal = n->fVal*10 + v; break; } case doTagValue: n = fNodeStack[fNodeStackPtr]; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); break; case doTagExpectedError: error(U_BRK_MALFORMED_RULE_TAG); returnVal = false; break; case doOptionStart: // Scanning a !!option. At the start of string. fOptionStart = fScanIndex; break; case doOptionEnd: { UnicodeString opt(fRB->fRules, fOptionStart, fScanIndex-fOptionStart); if (opt == UNICODE_STRING("chain", 5)) { fRB->fChainRules = true; } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) { fRB->fLBCMNoChain = true; } else if (opt == UNICODE_STRING("forward", 7)) { fRB->fDefaultTree = &fRB->fForwardTree; } else if (opt == UNICODE_STRING("reverse", 7)) { fRB->fDefaultTree = &fRB->fReverseTree; } else if (opt == UNICODE_STRING("safe_forward", 12)) { fRB->fDefaultTree = &fRB->fSafeFwdTree; } else if (opt == UNICODE_STRING("safe_reverse", 12)) { fRB->fDefaultTree = &fRB->fSafeRevTree; } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) { fRB->fLookAheadHardBreak = true; } else if (opt == UNICODE_STRING("quoted_literals_only", 20)) { fRuleSets[kRuleSet_rule_char-128].clear(); } else if (opt == UNICODE_STRING("unquoted_literals", 17)) { fRuleSets[kRuleSet_rule_char-128].applyPattern(UnicodeString(gRuleSet_rule_char_pattern), *fRB->fStatus); } else { error(U_BRK_UNRECOGNIZED_OPTION); } } break; case doReverseDir: fReverseRule = true; break; case doStartVariableName: n = pushNewNode(RBBINode::varRef); if (U_FAILURE(*fRB->fStatus)) { break; } n->fFirstPos = fScanIndex; break; case doEndVariableName: n = fNodeStack[fNodeStackPtr]; if (n==NULL || n->fType != RBBINode::varRef) { error(U_BRK_INTERNAL_ERROR); break; } n->fLastPos = fScanIndex; fRB->fRules.extractBetween(n->fFirstPos+1, n->fLastPos, n->fText); // Look the newly scanned name up in the symbol table // If there's an entry, set the l. child of the var ref to the replacement expression. // (We also pass through here when scanning assignments, but no harm is done, other // than a slight wasted effort that seems hard to avoid. Lookup will be null) n->fLeftChild = fSymbolTable->lookupNode(n->fText); break; case doCheckVarDef: n = fNodeStack[fNodeStackPtr]; if (n->fLeftChild == NULL) { error(U_BRK_UNDEFINED_VARIABLE); returnVal = false; } break; case doExprFinished: break; case doRuleErrorAssignExpr: error(U_BRK_ASSIGN_ERROR); returnVal = false; break; case doExit: returnVal = false; break; case doScanUnicodeSet: scanSet(); break; default: error(U_BRK_INTERNAL_ERROR); returnVal = false; break; } return returnVal && U_SUCCESS(*fRB->fStatus); } //------------------------------------------------------------------------------ // // Error Report a rule parse error. // Only report it if no previous error has been recorded. // //------------------------------------------------------------------------------ void RBBIRuleScanner::error(UErrorCode e) { if (U_SUCCESS(*fRB->fStatus)) { *fRB->fStatus = e; if (fRB->fParseError) { fRB->fParseError->line = fLineNum; fRB->fParseError->offset = fCharNum; fRB->fParseError->preContext[0] = 0; fRB->fParseError->postContext[0] = 0; } } } //------------------------------------------------------------------------------ // // fixOpStack The parse stack holds partially assembled chunks of the parse tree. // An entry on the stack may be as small as a single setRef node, // or as large as the parse tree // for an entire expression (this will be the one item left on the stack // when the parsing of an RBBI rule completes. // // This function is called when a binary operator is encountered. // It looks back up the stack for operators that are not yet associated // with a right operand, and if the precedence of the stacked operator >= // the precedence of the current operator, binds the operand left, // to the previously encountered operator. // //------------------------------------------------------------------------------ void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) { RBBINode *n; // printNodeStack("entering fixOpStack()"); for (;;) { n = fNodeStack[fNodeStackPtr-1]; // an operator node if (n->fPrecedence == 0) { RBBIDebugPuts("RBBIRuleScanner::fixOpStack, bad operator node"); error(U_BRK_INTERNAL_ERROR); return; } if (n->fPrecedence < p || n->fPrecedence <= RBBINode::precLParen) { // The most recent operand goes with the current operator, // not with the previously stacked one. break; } // Stack operator is a binary op ( '|' or concatenation) // TOS operand becomes right child of this operator. // Resulting subexpression becomes the TOS operand. n->fRightChild = fNodeStack[fNodeStackPtr]; fNodeStack[fNodeStackPtr]->fParent = n; fNodeStackPtr--; // printNodeStack("looping in fixOpStack() "); } if (p <= RBBINode::precLParen) { // Scan is at a right paren or end of expression. // The scanned item must match the stack, or else there was an error. // Discard the left paren (or start expr) node from the stack, // leaving the completed (sub)expression as TOS. if (n->fPrecedence != p) { // Right paren encountered matched start of expression node, or // end of expression matched with a left paren node. error(U_BRK_MISMATCHED_PAREN); } fNodeStack[fNodeStackPtr-1] = fNodeStack[fNodeStackPtr]; fNodeStackPtr--; // Delete the now-discarded LParen or Start node. delete n; } // printNodeStack("leaving fixOpStack()"); } //------------------------------------------------------------------------------ // // findSetFor given a UnicodeString, // - find the corresponding Unicode Set (uset node) // (create one if necessary) // - Set fLeftChild of the caller's node (should be a setRef node) // to the uset node // Maintain a hash table of uset nodes, so the same one is always used // for the same string. // If a "to adopt" set is provided and we haven't seen this key before, // add the provided set to the hash table. // If the string is one (32 bit) char in length, the set contains // just one element which is the char in question. // If the string is "any", return a set containing all chars. // //------------------------------------------------------------------------------ void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) { RBBISetTableEl *el; // First check whether we've already cached a set for this string. // If so, just use the cached set in the new node. // delete any set provided by the caller, since we own it. el = (RBBISetTableEl *)uhash_get(fSetTable, &s); if (el != NULL) { delete setToAdopt; node->fLeftChild = el->val; U_ASSERT(node->fLeftChild->fType == RBBINode::uset); return; } // Haven't seen this set before. // If the caller didn't provide us with a prebuilt set, // create a new UnicodeSet now. if (setToAdopt == NULL) { if (s.compare(kAny, -1) == 0) { setToAdopt = new UnicodeSet(0x000000, 0x10ffff); } else { UChar32 c; c = s.char32At(0); setToAdopt = new UnicodeSet(c, c); } } // // Make a new uset node to refer to this UnicodeSet // This new uset node becomes the child of the caller's setReference node. // RBBINode *usetNode = new RBBINode(RBBINode::uset); if (usetNode == NULL) { error(U_MEMORY_ALLOCATION_ERROR); return; } usetNode->fInputSet = setToAdopt; usetNode->fParent = node; node->fLeftChild = usetNode; usetNode->fText = s; // // Add the new uset node to the list of all uset nodes. // fRB->fUSetNodes->addElement(usetNode, *fRB->fStatus); // // Add the new set to the set hash table. // el = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl)); UnicodeString *tkey = new UnicodeString(s); if (tkey == NULL || el == NULL || setToAdopt == NULL) { // Delete to avoid memory leak delete tkey; tkey = NULL; uprv_free(el); el = NULL; delete setToAdopt; setToAdopt = NULL; error(U_MEMORY_ALLOCATION_ERROR); return; } el->key = tkey; el->val = usetNode; uhash_put(fSetTable, el->key, el, fRB->fStatus); return; } // // Assorted Unicode character constants. // Numeric because there is no portable way to enter them as literals. // (Think EBCDIC). // static const UChar chCR = 0x0d; // New lines, for terminating comments. static const UChar chLF = 0x0a; static const UChar chNEL = 0x85; // NEL newline variant static const UChar chLS = 0x2028; // Unicode Line Separator static const UChar chApos = 0x27; // single quote, for quoted chars. static const UChar chPound = 0x23; // '#', introduces a comment. static const UChar chBackSlash = 0x5c; // '\' introduces a char escape static const UChar chLParen = 0x28; static const UChar chRParen = 0x29; //------------------------------------------------------------------------------ // // stripRules Return a rules string without extra spaces. // (Comments are removed separately, during rule parsing.) // //------------------------------------------------------------------------------ UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) { UnicodeString strippedRules; int32_t rulesLength = rules.length(); for (int32_t idx=0; idx<rulesLength; idx = rules.moveIndex32(idx, 1)) { UChar32 cp = rules.char32At(idx); bool whiteSpace = u_hasBinaryProperty(cp, UCHAR_PATTERN_WHITE_SPACE); if (whiteSpace) { continue; } strippedRules.append(cp); } return strippedRules; } //------------------------------------------------------------------------------ // // nextCharLL Low Level Next Char from rule input source. // Get a char from the input character iterator, // keep track of input position for error reporting. // //------------------------------------------------------------------------------ UChar32 RBBIRuleScanner::nextCharLL() { UChar32 ch; if (fNextIndex >= fRB->fRules.length()) { return (UChar32)-1; } ch = fRB->fRules.char32At(fNextIndex); if (U_IS_SURROGATE(ch)) { error(U_ILLEGAL_CHAR_FOUND); return U_SENTINEL; } fNextIndex = fRB->fRules.moveIndex32(fNextIndex, 1); if (ch == chCR || ch == chNEL || ch == chLS || (ch == chLF && fLastChar != chCR)) { // Character is starting a new line. Bump up the line number, and // reset the column to 0. fLineNum++; fCharNum=0; if (fQuoteMode) { error(U_BRK_NEW_LINE_IN_QUOTED_STRING); fQuoteMode = false; } } else { // Character is not starting a new line. Except in the case of a // LF following a CR, increment the column position. if (ch != chLF) { fCharNum++; } } fLastChar = ch; return ch; } //------------------------------------------------------------------------------ // // nextChar for rules scanning. At this level, we handle stripping // out comments and processing backslash character escapes. // The rest of the rules grammar is handled at the next level up. // //------------------------------------------------------------------------------ void RBBIRuleScanner::nextChar(RBBIRuleChar &c) { // Unicode Character constants needed for the processing done by nextChar(), // in hex because literals wont work on EBCDIC machines. fScanIndex = fNextIndex; c.fChar = nextCharLL(); c.fEscaped = false; // // check for '' sequence. // These are recognized in all contexts, whether in quoted text or not. // if (c.fChar == chApos) { if (fRB->fRules.char32At(fNextIndex) == chApos) { c.fChar = nextCharLL(); // get nextChar officially so character counts c.fEscaped = true; // stay correct. } else { // Single quote, by itself. // Toggle quoting mode. // Return either '(' or ')', because quotes cause a grouping of the quoted text. fQuoteMode = !fQuoteMode; if (fQuoteMode == true) { c.fChar = chLParen; } else { c.fChar = chRParen; } c.fEscaped = false; // The paren that we return is not escaped. return; } } if (fQuoteMode) { c.fEscaped = true; } else { // We are not in a 'quoted region' of the source. // if (c.fChar == chPound) { // Start of a comment. Consume the rest of it. // The new-line char that terminates the comment is always returned. // It will be treated as white-space, and serves to break up anything // that might otherwise incorrectly clump together with a comment in // the middle (a variable name, for example.) int32_t commentStart = fScanIndex; for (;;) { c.fChar = nextCharLL(); if (c.fChar == (UChar32)-1 || // EOF c.fChar == chCR || c.fChar == chLF || c.fChar == chNEL || c.fChar == chLS) {break;} } for (int32_t i=commentStart; i<fNextIndex-1; ++i) { fRB->fStrippedRules.setCharAt(i, u' '); } } if (c.fChar == (UChar32)-1) { return; } // // check for backslash escaped characters. // Use UnicodeString::unescapeAt() to handle them. // if (c.fChar == chBackSlash) { c.fEscaped = true; int32_t startX = fNextIndex; c.fChar = fRB->fRules.unescapeAt(fNextIndex); if (fNextIndex == startX) { error(U_BRK_HEX_DIGITS_EXPECTED); } fCharNum += fNextIndex-startX; } } // putc(c.fChar, stdout); } //------------------------------------------------------------------------------ // // Parse RBBI rules. The state machine for rules parsing is here. // The state tables are hand-written in the file rbbirpt.txt, // and converted to the form used here by a perl // script rbbicst.pl // //------------------------------------------------------------------------------ void RBBIRuleScanner::parse() { uint16_t state; const RBBIRuleTableEl *tableEl; if (U_FAILURE(*fRB->fStatus)) { return; } state = 1; nextChar(fC); // // Main loop for the rule parsing state machine. // Runs once per state transition. // Each time through optionally performs, depending on the state table, // - an advance to the the next input char // - an action to be performed. // - pushing or popping a state to/from the local state return stack. // for (;;) { // Bail out if anything has gone wrong. // RBBI rule file parsing stops on the first error encountered. if (U_FAILURE(*fRB->fStatus)) { break; } // Quit if state == 0. This is the normal way to exit the state machine. // if (state == 0) { break; } // Find the state table element that matches the input char from the rule, or the // class of the input character. Start with the first table row for this // state, then linearly scan forward until we find a row that matches the // character. The last row for each state always matches all characters, so // the search will stop there, if not before. // tableEl = &gRuleParseStateTable[state]; #ifdef RBBI_DEBUG if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d) state=%s ", fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]); } #endif for (;;) { #ifdef RBBI_DEBUG if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf("."); fflush(stdout);} #endif if (tableEl->fCharClass < 127 && fC.fEscaped == false && tableEl->fCharClass == fC.fChar) { // Table row specified an individual character, not a set, and // the input character is not escaped, and // the input character matched it. break; } if (tableEl->fCharClass == 255) { // Table row specified default, match anything character class. break; } if (tableEl->fCharClass == 254 && fC.fEscaped) { // Table row specified "escaped" and the char was escaped. break; } if (tableEl->fCharClass == 253 && fC.fEscaped && (fC.fChar == 0x50 || fC.fChar == 0x70 )) { // Table row specified "escaped P" and the char is either 'p' or 'P'. break; } if (tableEl->fCharClass == 252 && fC.fChar == (UChar32)-1) { // Table row specified eof and we hit eof on the input. break; } if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class && fC.fEscaped == false && // char is not escaped && fC.fChar != (UChar32)-1) { // char is not EOF U_ASSERT((tableEl->fCharClass-128) < UPRV_LENGTHOF(fRuleSets)); if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) { // Table row specified a character class, or set of characters, // and the current char matches it. break; } } // No match on this row, advance to the next row for this state, tableEl++; } if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPuts("");} // // We've found the row of the state table that matches the current input // character from the rules string. // Perform any action specified by this row in the state table. if (doParseActions((int32_t)tableEl->fAction) == false) { // Break out of the state machine loop if the // the action signalled some kind of error, or // the action was to exit, occurs on normal end-of-rules-input. break; } if (tableEl->fPushState != 0) { fStackPtr++; if (fStackPtr >= kStackSize) { error(U_BRK_INTERNAL_ERROR); RBBIDebugPuts("RBBIRuleScanner::parse() - state stack overflow."); fStackPtr--; } fStack[fStackPtr] = tableEl->fPushState; } if (tableEl->fNextChar) { nextChar(fC); } // Get the next state from the table entry, or from the // state stack if the next state was specified as "pop". if (tableEl->fNextState != 255) { state = tableEl->fNextState; } else { state = fStack[fStackPtr]; fStackPtr--; if (fStackPtr < 0) { error(U_BRK_INTERNAL_ERROR); RBBIDebugPuts("RBBIRuleScanner::parse() - state stack underflow."); fStackPtr++; } } } if (U_FAILURE(*fRB->fStatus)) { return; } // If there are no forward rules set an error. // if (fRB->fForwardTree == NULL) { error(U_BRK_RULE_SYNTAX); return; } // // Parsing of the input RBBI rules is complete. // We now have a parse tree for the rule expressions // and a list of all UnicodeSets that are referenced. // #ifdef RBBI_DEBUG if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();} if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) { RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n"); RBBINode::printTree(fRB->fForwardTree, true); RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n"); RBBINode::printTree(fRB->fReverseTree, true); RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n"); RBBINode::printTree(fRB->fSafeFwdTree, true); RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n"); RBBINode::printTree(fRB->fSafeRevTree, true); } #endif } //------------------------------------------------------------------------------ // // printNodeStack for debugging... // //------------------------------------------------------------------------------ #ifdef RBBI_DEBUG void RBBIRuleScanner::printNodeStack(const char *title) { int i; RBBIDebugPrintf("%s. Dumping node stack...\n", title); for (i=fNodeStackPtr; i>0; i--) {RBBINode::printTree(fNodeStack[i], true);} } #endif //------------------------------------------------------------------------------ // // pushNewNode create a new RBBINode of the specified type and push it // onto the stack of nodes. // //------------------------------------------------------------------------------ RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) { if (U_FAILURE(*fRB->fStatus)) { return NULL; } if (fNodeStackPtr >= kStackSize - 1) { error(U_BRK_RULE_SYNTAX); RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow."); return NULL; } fNodeStackPtr++; fNodeStack[fNodeStackPtr] = new RBBINode(t); if (fNodeStack[fNodeStackPtr] == NULL) { *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR; } return fNodeStack[fNodeStackPtr]; } //------------------------------------------------------------------------------ // // scanSet Construct a UnicodeSet from the text at the current scan // position. Advance the scan position to the first character // after the set. // // A new RBBI setref node referring to the set is pushed onto the node // stack. // // The scan position is normally under the control of the state machine // that controls rule parsing. UnicodeSets, however, are parsed by // the UnicodeSet constructor, not by the RBBI rule parser. // //------------------------------------------------------------------------------ void RBBIRuleScanner::scanSet() { UnicodeSet *uset; ParsePosition pos; int startPos; int i; if (U_FAILURE(*fRB->fStatus)) { return; } pos.setIndex(fScanIndex); startPos = fScanIndex; UErrorCode localStatus = U_ZERO_ERROR; uset = new UnicodeSet(); if (uset == NULL) { localStatus = U_MEMORY_ALLOCATION_ERROR; } else { uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus); } if (U_FAILURE(localStatus)) { // TODO: Get more accurate position of the error from UnicodeSet's return info. // UnicodeSet appears to not be reporting correctly at this time. #ifdef RBBI_DEBUG RBBIDebugPrintf("UnicodeSet parse position.ErrorIndex = %d\n", pos.getIndex()); #endif error(localStatus); delete uset; return; } // Verify that the set contains at least one code point. // U_ASSERT(uset!=NULL); if (uset->isEmpty()) { // This set is empty. // Make it an error, because it almost certainly is not what the user wanted. // Also, avoids having to think about corner cases in the tree manipulation code // that occurs later on. error(U_BRK_RULE_EMPTY_SET); delete uset; return; } // Advance the RBBI parse position over the UnicodeSet pattern. // Don't just set fScanIndex because the line/char positions maintained // for error reporting would be thrown off. i = pos.getIndex(); for (;;) { if (fNextIndex >= i) { break; } nextCharLL(); } if (U_SUCCESS(*fRB->fStatus)) { RBBINode *n; n = pushNewNode(RBBINode::setRef); if (U_FAILURE(*fRB->fStatus)) { return; } n->fFirstPos = startPos; n->fLastPos = fNextIndex; fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); // findSetFor() serves several purposes here: // - Adopts storage for the UnicodeSet, will be responsible for deleting. // - Maintains collection of all sets in use, needed later for establishing // character categories for run time engine. // - Eliminates mulitiple instances of the same set. // - Creates a new uset node if necessary (if this isn't a duplicate.) findSetFor(n->fText, n, uset); } } int32_t RBBIRuleScanner::numRules() { return fRuleNum; } U_NAMESPACE_END #endif /* #if !UCONFIG_NO_BREAK_ITERATION */