Archives/jamiebuilds_the-super-tiny-compiler
1
0
Fork 0
mirror of https://github.com/jamiebuilds/the-super-tiny-compiler.git synced 2024-10-27 20:34:08 +00:00
Code Issues Projects Releases Wiki Activity

Make tokenizer faster.

Browse Source
This commit is contained in:
Nathaniel van Diepen 2016-03-31 16:47:57 -06:00
parent 55692eb77d
commit c6673887e7
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
super-tiny-compiler-unannotated.js
View File

@ -1,6 +1,9 @@
function tokenizer(input) { function tokenizer(input) {
var current = 0; var current = 0;
var tokens = []; var tokens = [];
var WHITESPACE = /\s/;
var NUMBERS = /[0-9]/;
var LETTERS = /[a-zA-Z]/;
while (current < input.length) { while (current < input.length) {
var char = input[current]; var char = input[current];
@ -23,13 +26,11 @@ function tokenizer(input) {
continue; continue;
} }
var WHITESPACE = /\s/;
if (WHITESPACE.test(char)) { if (WHITESPACE.test(char)) {
current++; current++;
continue; continue;
} }
var NUMBERS = /[0-9]/;
if (NUMBERS.test(char)) { if (NUMBERS.test(char)) {
var value = ''; var value = '';
@ -46,7 +47,6 @@ function tokenizer(input) {
continue; continue;
} }
var LETTERS = /[a-zA-Z]/;
if (LETTERS.test(char)) { if (LETTERS.test(char)) {
var value = ''; var value = '';

8
super-tiny-compiler.js
View File

@ -348,6 +348,11 @@ function tokenizer(input) {
// And a `tokens` array for pushing our tokens to. // And a `tokens` array for pushing our tokens to.
var tokens = []; var tokens = [];
// Regex tests for later
var WHITESPACE = /\s/;
var NUMBERS = /[0-9]/;
var LETTERS = /[a-zA-Z]/;
// We start by creating a `while` loop where we are setting up our `current` // We start by creating a `while` loop where we are setting up our `current`
// variable to be incremented as much as we want `inside` the loop. // variable to be incremented as much as we want `inside` the loop.
// //
@ -398,7 +403,6 @@ function tokenizer(input) {
// //
// So here we're just going to test for existence and if it does exist we're // So here we're just going to test for existence and if it does exist we're
// going to just `continue` on. // going to just `continue` on.
var WHITESPACE = /\s/;
if (WHITESPACE.test(char)) { if (WHITESPACE.test(char)) {
current++; current++;
continue; continue;
@ -413,7 +417,6 @@ function tokenizer(input) {
// Only two separate tokens // Only two separate tokens
// //
// So we start this off when we encounter the first number in a sequence. // So we start this off when we encounter the first number in a sequence.
var NUMBERS = /[0-9]/;
if (NUMBERS.test(char)) { if (NUMBERS.test(char)) {
// We're going to create a `value` string that we are going to push // We're going to create a `value` string that we are going to push
@ -446,7 +449,6 @@ function tokenizer(input) {
// ^^^ // ^^^
// Name token // Name token
// //
var LETTERS = /[a-zA-Z]/;
if (LETTERS.test(char)) { if (LETTERS.test(char)) {
var value = ''; var value = '';