Test qx.dev.Tokenizer.
Text to Tokenize:
/** * Java Doc string. */ function tokenizeDemo() { /* this is an example of the tokenizer in action */ var s = "So how's the weather?" + "\n"; for (i = 0; i < a.length; i++) { s = s + a[i] + "\n"; } if (s.length > 10) { print('you have a lot say!'); } // this is a single line comment test // /* this is a commented // comment test */ /****/ }