interface ITokenizeLineResult {
    ruleStack: StateStack;
    stoppedEarly: boolean;
    tokens: IToken[];
}

Properties

ruleStack: StateStack

The prevState to be passed on to the next line tokenization.

stoppedEarly: boolean

Did tokenization stop early due to reaching the time limit.

tokens: IToken[]