interface ITokenizeLineResult2 {
    ruleStack: StateStack;
    stoppedEarly: boolean;
    tokens: Uint32Array;
}

Properties

ruleStack: StateStack

The prevState to be passed on to the next line tokenization.

stoppedEarly: boolean

Did tokenization stop early due to reaching the time limit.

tokens: Uint32Array

The tokens in binary format. Each token occupies two array indices. For token i:

  • at offset 2*i => startIndex
  • at offset 2*i + 1 => metadata