🚉💍 Updated with Glitch

1-tokenizer.js

@@ -15,12 +15,42 @@
  *   (add 2 (subtract 4 2))   =>   [{ type: 'paren', value: '(' }, ...]
  */
 
+/**
+ * First, let’s create a class to remember the position of each token.
+ */
+class Position {
+  constructor(index, line = 1, column = 1) {
+    this.line = line;
+    this.column = column;
+    this.index = index;
+  }
+  nextCh() {
+    this.column++;
+    this.index++;
+    return this;
+  }
+  nextLine() {
+    this.column = 1;
+    this.line++;
+    return this;
+  }
+  clone() {
+    return new Position(
+      this.index,
+      this.line,
+      this.column
+    );
+  }
+  toString() {
+    return this.line + ':' + this.column;
+  }
+}
+
 // We start by accepting an input string of code, and we're gonna set up two
 // things...
 function tokenizer(input) {
-
   // A `current` variable for tracking our position in the code like a cursor.
-  let current = 0;
+  let current = new Position(0);
 
   // And a `tokens` array for pushing our tokens to.
   let tokens = [];
@@ -30,10 +60,9 @@ function tokenizer(input) {
   //
   // We do this because we may want to increment `current` many times within a
   // single loop because our tokens can be any length.
-  while (current < input.length) {
+  while (current.index < input.length) {
-
     // We're also going to store the `current` character in the `input`.
-    let char = input[current];
+    let char = input[current.index];
 
     // The first thing we want to check for is an open parenthesis. This will
     // later be used for `CallExpression` but for now we only care about the
@@ -43,14 +72,16 @@ function tokenizer(input) {
     if (char === '(') {
 
       // If we do, we push a new token with the type `paren` and set the value
-      // to an open parenthesis.
+      // to an open parenthesis. We also store the `start` and `end` of this
+      // token for future reference.
       tokens.push({
         type: 'paren',
         value: '(',
+        start: current.clone(),
+        end: current.clone(),
       });
-
+      // Then we increment `current`.
-      // Then we increment `current`
+      current.nextCh();
-      current++;
 
       // And we `continue` onto the next cycle of the loop.
       continue;
@@ -63,8 +94,10 @@ function tokenizer(input) {
       tokens.push({
         type: 'paren',
         value: ')',
+        start: current.clone(),
+        end: current.clone().nextCh(),
       });
-      current++;
+      current.nextCh();
       continue;
     }
 
@@ -77,7 +110,12 @@ function tokenizer(input) {
     // going to just `continue` on.
     let WHITESPACE = /\s/;
     if (WHITESPACE.test(char)) {
-      current++;
+      current.nextCh();
+      // If the character is a newline, we'll tell the cursor that we've
+      // moved to the next line.
+      if (char === '\n') {
+        current.nextLine();
+      }
       continue;
     }
 
@@ -96,17 +134,23 @@ function tokenizer(input) {
       // We're going to create a `value` string that we are going to push
       // characters to.
       let value = '';
+      // We'll also save the start of the number for later.
+      const start = current.clone();
 
       // Then we're going to loop through each character in the sequence until
       // we encounter a character that is not a number, pushing each character
       // that is a number to our `value` and incrementing `current` as we go.
       while (NUMBERS.test(char)) {
         value += char;
-        char = input[++current];
+        current.nextCh();
+        if (current.index >= input.length) {
+          break;
+        }
+        char = input[current.index];
       }
 
       // After that we push our `number` token to the `tokens` array.
-      tokens.push({ type: 'number', value });
+      tokens.push({ type: 'number', value, start, end: current.clone() });
 
       // And we continue on.
       continue;
@@ -122,22 +166,41 @@ function tokenizer(input) {
     if (char === '"') {
       // Keep a `value` variable for building up our string token.
       let value = '';
+      // We'll also save the start of the string for later.
+      const start = current.clone();
+      // If the quote is the last character in the program,
+      // throw a syntax error:
+      if (current.index + 1 >= input.length) {
+        throw new SyntaxError(`Unterminated string at ${start}-${current}`);
+      }
+      // Otherwise, skip past the quote...
+      current.nextCh();
 
-      // We'll skip the opening double quote in our token.
+
-      char = input[++current];
+      // ...and grab the first character of the string.
+      char = input[current.index];
 
       // Then we'll iterate through each character until we reach another
       // double quote.
       while (char !== '"') {
         value += char;
-        char = input[++current];
+        // If the string is not terminated before the end of the program,
+        // throw a syntax error
+        if (current.index + 1 >= input.length) {
+          throw new SyntaxError(`Unterminated string at ${start}-${current}`);
+        }
+        // Otherwise, increment the cursor
+        current.nextCh();
+        // And grab the next character.
+        char = input[current.index];
       }
 
       // Skip the closing double quote.
-      char = input[++current];
+      current.nextCh();
+      char = input[current.index];
 
       // And add our `string` token to the `tokens` array.
-      tokens.push({ type: 'string', value });
+      tokens.push({ type: 'string', value, start, end: current.clone() });
 
       continue;
     }
@@ -152,24 +215,28 @@ function tokenizer(input) {
     //
     let LETTERS = /[a-z]/i;
     if (LETTERS.test(char)) {
+      // First, we'll create a string to hold the value
       let value = '';
+      // And save the current position for later.
+      const start = current.clone();
 
       // Again we're just going to loop through all the letters pushing them to
       // a value.
-      while (LETTERS.test(char)) {
+      while (LETTERS.test(char) && current.index < input.length) {
         value += char;
-        char = input[++current];
+        current.nextCh();
+        char = input[current.index];
       }
 
       // And pushing that value as a token with the type `name` and continuing.
-      tokens.push({ type: 'name', value });
+      tokens.push({ type: 'name', value, start, end: current.clone() });
 
       continue;
     }
 
     // Finally if we have not matched a character by now, we're going to throw
-    // an error and completely exit.
+    // a syntax error and completely exit.
-    throw new TypeError('I dont know what this character is: ' + char);
+    throw new SyntaxError('I dont know what this character is: ' + char);
   }
 
   // Then at the end of our `tokenizer` we simply return the tokens array.

2-parser.js

@@ -14,8 +14,9 @@
 
 // Okay, so we define a `parser` function that accepts our array of `tokens`.
 function parser(tokens) {
-
   // Again we keep a `current` variable that we will use as a cursor.
+  // Because we're not iterating over the source code, we are just
+  // using a number.
   let current = 0;
 
   // But this time we're going to use recursion instead of a `while` loop. So we
@@ -24,6 +25,7 @@ function parser(tokens) {
 
     // Inside the walk function we start by grabbing the `current` token.
     let token = tokens[current];
+    if (!token) return;
 
     // We're going to split each type of token off into a different code path,
     // starting off with `number` tokens.
@@ -39,6 +41,8 @@ function parser(tokens) {
       return {
         type: 'NumberLiteral',
         value: token.value,
+        start: token.start,
+        end: token.end,
       };
     }
 
@@ -50,6 +54,8 @@ function parser(tokens) {
       return {
         type: 'StringLiteral',
         value: token.value,
+        start: token.start,
+        end: token.end,
       };
     }
 
@@ -60,9 +66,14 @@ function parser(tokens) {
       token.value === '('
     ) {
 
+      const initial = token;
       // We'll increment `current` to skip the parenthesis since we don't care
       // about it in our AST.
       token = tokens[++current];
+      // If we've reached the end of the program, throw a syntax error.
+      if (!token) {
+        throw new SyntaxError(`Unclosed function call at ${initial.start}`);
+      }
 
       // We create a base node with the type `CallExpression`, and we're going
       // to set the name as the current token's value since the next token after
@@ -71,10 +82,19 @@ function parser(tokens) {
         type: 'CallExpression',
         name: token.value,
         params: [],
+        start: token.start,
       };
 
+      // If the user writes `()` as code, throw a syntax error.
+      if (token.type === 'paren' && token.value === ')') {
+        throw new SyntaxError(`Unexpected empty function call at ${tokens[current - 1].start}–${token.end}`)
+      }
       // We increment `current` *again* to skip the name token.
       token = tokens[++current];
+      // If we've reached the end of the program (for example: `(foo`), throw a syntax error.
+      if (!token) {
+        throw new SyntaxError(`Unclosed function call at ${initial.start}-${tokens[current - 1].end}`);
+      }
 
       // And now we want to loop through each token that will be the `params` of
       // our `CallExpression` until we encounter a closing parenthesis.
@@ -118,7 +138,13 @@ function parser(tokens) {
         // push it into our `node.params`.
         node.params.push(walk());
         token = tokens[current];
+        // If we've reached the end of the program, throw a syntax error.
+        if (!token) {
+          throw new SyntaxError('Unclosed function call at ' + initial.start + '-' + tokens[current - 1].end);
+        }
       }
+      // Next, we save the end position of the call
+      node.end = token.end;
 
       // Finally we will increment `current` one last time to skip the closing
       // parenthesis.
@@ -128,9 +154,14 @@ function parser(tokens) {
       return node;
     }
     
+    // If the user writes a literal name token (`foo`), throw a syntax error.
+    if (token.type === 'name') {
+      throw new SyntaxError(`Unexpected name '${token.value}' at ${token.start}–${token.end}`)
+    }
+
     // Again, if we haven't recognized the token type by now we're going to
     // throw an error.
-    throw new TypeError(token.type);
+    throw new SyntaxError(`Unrecognized token: ${JSON.stringify(token)}`);
   }
 
   // Now, we're going to create our AST which will have a root which is a

3-traverser.js

@@ -78,7 +78,7 @@ function traverser(ast, visitor) {
       // And again, if we haven't recognized the node type then we'll throw an
       // error.
       default:
-        throw new TypeError(node.type);
+        throw new SyntaxError(`Unrecognized node ${node.type}: ${JSON.stringify(node)}}`);
     }
 
     // If there is an `exit` method for this node type we'll call it with the

7-test.html

@@ -0,0 +1,8 @@
+<h1>Code in:</h1>
+<!-- This <textarea> will hold the user-inputted code. -->
+<textarea id="input" cols="100" rows="15">
+</textarea>
+
+<h1>Code out:</h1>
+<!-- This will hold the generated code or an error message. -->
+<pre id="output" style="overflow: scroll"></pre>

content.ejs

@@ -0,0 +1,13 @@
+<main <% if (isCode) { %>class="is-code"<% } %>>
+  <% if (isCode) { %>
+    <pre id="code"><%- fileContents %></pre>
+  <% } else { %>
+    <div class="container">
+      <%- fileContents %>
+    </div>
+  <% } %>
+
+  <% if (fileName === '6-compiler.js') { %>
+    <img src="https://cdn.glitch.com/da026c15-c2dc-4ff8-bbed-d9d003c04338%2Ftumblr_mvemcyarmn1rslphyo1_400.gif?1492115698121" alt="Carlton Dance">
+  <% } %>
+</main>

package.json

@@ -7,7 +7,8 @@
     "express": "^4.15.2",
     "markdown-it": "^8.3.1",
     "ejs": "^2.5.6",
-    "prismjs": "^9000.0.1"
+    "prismjs": "^9000.0.1",
+    "body-parser": "^1.17.1"
   },
   "scripts": {
     "start": "node server.js"

public/client.js

@@ -0,0 +1,95 @@
+/* globals NProgress */
+
+// Load pages using AJAX for a speed boost
+var cache = Object.create(null); // Don’t include default Object properties
+document.addEventListener('click', function (e) {
+  // When something gets clicked,
+  var target = closest(e.target, 'a');
+  // Check if it’s a link to a page on this domain.
+  if (!target || target.host !== location.host) return;
+  
+  // If it is, stop the browser from loading the next page,
+  e.preventDefault();
+  // get the path of the page,
+  var path = getPath(target);
+  // and load it ourselves.
+  loadPage(path).then(function (json) {
+    // Then, once the page is loaded,
+    // insert the rendered HTML in the correct place
+    document.querySelector('main').outerHTML = json.html;
+    // and change the file name in the header.
+    document.querySelector('.js-file-name').textContent = json.context.fileName;
+
+    // Finally, update the title bar and address bar,
+    history.pushState(null, json.title, target.href);
+    // and tell listeners about the page load.
+    emitLoad(path, target);
+  })
+});
+
+function getPath(l) {
+  return l.pathname + l.search + l.hash;
+}
+
+function emitLoad(path, target) {
+  var event = new Event('page:load');
+  event.target = target;
+  event.data = {
+    path: path,
+  };
+  document.dispatchEvent(event);
+}
+
+function loadPage(path) {
+  // If the path is in the cache, load it immediately.
+  if (path in cache) {
+    NProgress.done(true); // show bar anyway
+    return Promise.resolve(cache[path]);
+  }
+
+  // Otherwise, fetch it from the server,
+  NProgress.start();
+  var promise = fetch('/api/fetch?path=' + encodeURIComponent(path)).then(function (res) {
+    // then convert it to JSON.
+    return res.json();
+  })
+  // Once we have the JSON,
+  promise.then(function (json) {
+    // cache it!
+    cache[path] = json;
+    NProgress.done();
+    return json
+  }).catch(function (error) {
+    // If something went wrong, log it in the console.
+    console.warn(error);
+    NProgress.done();
+  });
+  return promise;
+
+}
+
+// When the page loads,
+document.addEventListener('page:load', function () {
+  // set the correct link as active.
+  document.querySelector('a.active').classList.remove('active');
+  document.querySelector('a[href="' + window.location.pathname + '"]').classList.add('active');
+})
+
+
+// Emit an initial load event when the page is ready
+window.addEventListener('load', function () {
+  emitLoad(getPath(location), document)
+})
+
+
+if (!Element.prototype.matches && Element.prototype.matchesSelector) {
+  Element.prototype.matches = Element.prototype.matchesSelector;
+}
+
+function closest(element, selector) {
+  if (!element) return null;
+  if (element.matches(selector)) {
+    return element;
+  }
+  return closest(element.parentElement, selector);
+}

public/nprogress.css

@@ -0,0 +1,75 @@
+/* Make clicks pass-through */
+#nprogress {
+  pointer-events: none;
+}
+
+#nprogress .bar {
+  background: #fff;
+
+  position: fixed;
+  z-index: 1031;
+  top: 0;
+  left: 0;
+
+  width: 100%;
+  height: 2px;
+}
+
+/* Fancy blur effect */
+#nprogress .peg {
+  display: block;
+  position: absolute;
+  right: 0px;
+  width: 100px;
+  height: 100%;
+  box-shadow: 0 0 10px #fff, 0 0 5px #fff;
+  opacity: 1.0;
+
+  -webkit-transform: rotate(3deg) translate(0px, -4px);
+      -ms-transform: rotate(3deg) translate(0px, -4px);
+          transform: rotate(3deg) translate(0px, -4px);
+}
+
+/* Remove these to get rid of the spinner
+#nprogress .spinner {
+  display: block;
+  position: fixed;
+  z-index: 1031;
+  top: 15px;
+  right: 15px;
+}
+
+#nprogress .spinner-icon {
+  width: 18px;
+  height: 18px;
+  box-sizing: border-box;
+
+  border: solid 2px transparent;
+  border-top-color: #fff;
+  border-left-color: #fff;
+  border-radius: 50%;
+
+  -webkit-animation: nprogress-spinner 400ms linear infinite;
+          animation: nprogress-spinner 400ms linear infinite;
+}
+*/
+
+.nprogress-custom-parent {
+  overflow: hidden;
+  position: relative;
+}
+
+.nprogress-custom-parent #nprogress .spinner,
+.nprogress-custom-parent #nprogress .bar {
+  position: absolute;
+}
+
+@-webkit-keyframes nprogress-spinner {
+  0%   { -webkit-transform: rotate(0deg); }
+  100% { -webkit-transform: rotate(360deg); }
+}
+@keyframes nprogress-spinner {
+  0%   { transform: rotate(0deg); }
+  100% { transform: rotate(360deg); }
+}
+

public/test.js

@@ -0,0 +1,32 @@
+// It would be simpler to use this if we could just `require()` the `compiler`,
+// but this code runs on the client side and it would be really complicated
+// to enable using `require()` here, so we just call the server instead.
+
+document.addEventListener('page:load', function (e) {
+  if (e.data.path !== '/test') return;
+  // When the user types into the input field...
+  document.getElementById('input').addEventListener('input', function (event) {
+    // POST the code to the API
+    fetch('/api/convert', {
+      method: 'POST',
+      body: event.target.value,
+    }).then(function (res) {
+      // then get the JSON back
+      return res.json();
+    }).then(function (json) {
+      // then either:
+      var output = document.getElementById('output');
+      if (json.ok) {
+        // output the transformed code if
+        // the compilation succeeded, or
+        output.style.backgroundColor = 'black';
+        output.textContent = json.code;
+      } else {
+        // output the error if there was
+        // a problem.
+        output.style.backgroundColor = 'darkred';
+        output.textContent = json.stack;
+      }
+    })
+  })
+});

server.js

@@ -1,12 +1,21 @@
 var markdown = require('markdown-it')();
 var Prism = require('prismjs');
+var bodyParser = require('body-parser');
 var express = require('express');
 var path = require('path');
 var ejs = require('ejs');
 var fs = require('fs');
 
+// First, let's create the server,
 var app = express();
+// add a `req.body` property to requests containing
+// the body as text,
+app.use(bodyParser.text());
+// and serve any files in the `public` directory from the /assets path.
+app.use('/assets', express.static(path.join(__dirname, 'public')));
 
+
+// Next, let's create the URLs to read the code used:
 var ROUTES_MAP = {
   '/'               : 'README.md',
   '/intro'          : '0-introduction.md',
@@ -15,7 +24,11 @@ var ROUTES_MAP = {
   '/traverser'      : '3-traverser.js',
   '/transformer'    : '4-transformer.js',
   '/code-generator' : '5-code-generator.js',
-  '/compiler'       : '6-compiler.js'
+  '/compiler'       : '6-compiler.js',
+  '/test'           : '7-test.html',
+  '/server'         : 'server.js',
+  '/client'         : 'public/client.js',
+  '/test-js'        : 'public/test.js',
 };
 
 var routes = Object.keys(ROUTES_MAP).map(function(routePath) {
@@ -25,46 +38,112 @@ var routes = Object.keys(ROUTES_MAP).map(function(routePath) {
   };
 });
 
+// Next, let's create helpers to read files,
 function readFile(fileName) {
   return fs.readFileSync(path.join(__dirname, fileName)).toString();
 }
 
+// render Markdown,
 function renderMarkdown(fileContents) {
   return markdown.render(fileContents);
 }
 
+// and highlight JavaScript code.
 function renderJavaScript(fileName, fileContents) {
   return Prism.highlight(fileContents, Prism.languages.javascript);
 }
 
-var template = ejs.compile(readFile('./template.html.ejs'));
+// We'll be using EJS (http://ejs.co) to render the files.
+var template = ejs.compile(readFile('./template.html.ejs'), {
+  filename: path.join(__dirname, 'template.html.ejs')
+});
 
 function render(routeName) {
+  return template(getContext(routeName));
+}
+
+function getContext(routeName) {
+  // We'll read the file at the specified path,
   var fileName = routeName;
   var fileContents = readFile(fileName);
   
+  // render it appropriately,
   var extName = path.extname(fileName);
   if (extName === '.md') fileContents = renderMarkdown(fileContents);
   if (extName === '.js') fileContents = renderJavaScript(fileName, fileContents);
   
-  let isCode = extName !== '.md';
+  let isCode = extName === '.js';
   
-  return template({
+  return {
-    routes: routes,
+    routes,
-    fileName: fileName,
+    fileName,
-    fileContents: fileContents,
+    fileContents,
-    isCode: isCode,
+    isCode
-  });
+  }
 }
 
+// Next, let's tell Express about each file we're rendering.
 routes.forEach(function(route) {
-  var html = render(route.routeName);
-  
   app.get(route.routePath, function(req, res) {
+    var html = render(route.routeName);
     res.send(html);
   });
 });
 
+// To convert the code,
+app.post('/api/convert', function(req, res) {
+  try {
+    // First, try to convert the code and send it back.
+    var code = require('./6-compiler')(req.body);
+    res.send({
+      ok: true,
+      code: code,
+    });
+  } catch (err) {
+    try {
+      // If that fails, send the error back.
+      res.send({
+        ok: false,
+        err: err.toString(),
+        stack: err.stack,
+      });
+    } catch (e) {
+      // If sending the error fails, send a generic error.
+      res.send({
+        ok: false,
+        err: 'unknown error',
+        stack: 'unknown error\n<no stack>',
+      });
+      // (if *that* fails, you're on your own :))
+    }
+  }
+});
+
+
+var contentTemplate = ejs.compile(readFile('./content.ejs'), {
+  filename: path.join(__dirname, 'content.ejs')
+});
+
+var titleTemplate = ejs.compile(readFile('./title.ejs'), {
+  filename: path.join(__dirname, 'title.ejs')
+});
+
+app.get('/api/fetch', function(req, res) {
+  var path = ROUTES_MAP[req.query.path]
+  if (!path) {
+    res.status('404').send({
+      error: 'route not found'
+    });
+  }
+  var context = getContext(path);
+  res.send({
+    title: titleTemplate(context),
+    html: contentTemplate(context),
+    context
+  });
+});
+
+// Finally, start the server.
 var listener = app.listen(process.env.PORT, function () {
   console.log('Your app is listening on port ' + listener.address().port);
 });

template.html.ejs

@@ -1,12 +1,18 @@
 <!doctype html>
-<html <% if (isCode) { %>class="is-code"<% } %>>
+<html>
   <head>
-    <title>The Super Tiny Compiler - <%= fileName %></title>
+    <title><%- include('title') %></title>
     <meta name="description" content="">
     <link id="favicon" rel="icon" href="https://glitch.com/edit/favicon-app.ico" type="image/x-icon">
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/nprogress/0.2.0/nprogress.min.js"></script>
+    <!-- We're including this to allow us to use `fetch` in more browsers. -->
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/fetch/2.0.3/fetch.min.js"></script>
+    <script src="/assets/client.js"></script>
+    <script src="/assets/test.js"></script>
+    <link rel="stylesheet" href="/assets/nprogress.css" />
     <style>
       * {
         box-sizing: border-box;
@@ -24,10 +30,12 @@
         overflow: hidden;
       }
       
-      html.is-code,
+      #app {
-      .is-code body {
         background: black;
       }
+      main:not(.is-code) {
+        background: white;
+      }
       
       header {
         position: absolute;
@@ -85,7 +93,11 @@
         left: 300px;
         right: 0;
         overflow: auto;
-        padding-bottom: 25%;
+        /* The topbar, the padding, and the last line */
+        padding-bottom: calc(100vh - 2em - 1em - 2em);
+      }
+      main pre#code {
+        padding-bottom: 0;
       }
       
       .container {
@@ -278,7 +290,7 @@
     <div id="app">
       <header>
         <a href="https://github.com/thejameskyle/the-super-tiny-compiler">
-          /Users/thejameskyle/code/the-super-tiny-compiler/<%= fileName %>
+          /Users/thejameskyle/code/the-super-tiny-compiler/<span class="js-file-name"><%= fileName %></span>
         </a>
         
         <a class="right" href="https://github.com/thejameskyle/the-super-tiny-compiler">
@@ -302,18 +314,7 @@
         <% }); %>
       </nav>
 
-      <main>
+      <%- include('content') %>
-        <% if (isCode) { %>
-          <pre id="code"><%- fileContents %></pre>
-        <% } else { %>
-          <div class="container">
-            <%- fileContents %>
-          </div>
-        <% } %>
-        
-        <% if (fileName === '6-compiler.js') { %>
-          <img src="https://cdn.glitch.com/da026c15-c2dc-4ff8-bbed-d9d003c04338%2Ftumblr_mvemcyarmn1rslphyo1_400.gif?1492115698121" alt="Carlton Dance">
-        <% } %>
       </main>
     </div>
   </body>

title.ejs

@@ -0,0 +1 @@
+The Super Tiny Compiler - <%= fileName %>

watch.json

@@ -0,0 +1,18 @@
+{
+  "install": {
+    "include": [
+      "^package\\.json$",
+      "^\\.env$"
+    ]
+  },
+  "restart": {
+    "exclude": [
+      "^public/",
+      "^dist/"
+    ],
+    "include": [
+      "\\.ejs$"
+    ]
+  },
+  "throttle": 100
+}