Archives/jamiebuilds_the-super-tiny-compiler
1
0
Fork 0
mirror of https://github.com/jamiebuilds/the-super-tiny-compiler.git synced 2024-10-27 20:34:08 +00:00
Code Issues Projects Releases Wiki Activity

🚉💍 Updated with Glitch

Browse Source
This commit is contained in:
System 2017-04-25 12:57:20 +00:00 committed by Jed Fox
parent 4ec6074b55
commit e910529c15
13 changed files with 477 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
1-tokenizer.js
View File

@ -15,12 +15,42 @@
* (add 2 (subtract 4 2)) => [{ type: 'paren', value: '(' }, ...]
*/
/**
* First, let’s create a class to remember the position of each token.
*/
class Position {
constructor(index, line = 1, column = 1) {
this.line = line;
this.column = column;
this.index = index;
}
nextCh() {
this.column++;
this.index++;
return this;
}
nextLine() {
this.column = 1;
this.line++;
return this;
}
clone() {
return new Position(
this.index,
this.line,
this.column
);
}
toString() {
return this.line + ':' + this.column;
}
}
// We start by accepting an input string of code, and we're gonna set up two
// things...
function tokenizer(input) {
// A `current` variable for tracking our position in the code like a cursor.
let current = 0;
let current = new Position(0);
// And a `tokens` array for pushing our tokens to.
let tokens = [];
@ -30,10 +60,9 @@ function tokenizer(input) {
//
// We do this because we may want to increment `current` many times within a
// single loop because our tokens can be any length.
while (current < input.length) {
while (current.index < input.length) {
// We're also going to store the `current` character in the `input`.
let char = input[current];
let char = input[current.index];
// The first thing we want to check for is an open parenthesis. This will
// later be used for `CallExpression` but for now we only care about the
@ -43,14 +72,16 @@ function tokenizer(input) {
if (char === '(') {
// If we do, we push a new token with the type `paren` and set the value
// to an open parenthesis.
// to an open parenthesis. We also store the `start` and `end` of this
// token for future reference.
tokens.push({
type: 'paren',
value: '(',
start: current.clone(),
end: current.clone(),
});
// Then we increment `current`
current++;
// Then we increment `current`.
current.nextCh();
// And we `continue` onto the next cycle of the loop.
continue;
@ -63,8 +94,10 @@ function tokenizer(input) {
tokens.push({
type: 'paren',
value: ')',
start: current.clone(),
end: current.clone().nextCh(),
});
current++;
current.nextCh();
continue;
}
@ -77,7 +110,12 @@ function tokenizer(input) {
// going to just `continue` on.
let WHITESPACE = /\s/;
if (WHITESPACE.test(char)) {
current++;
current.nextCh();
// If the character is a newline, we'll tell the cursor that we've
// moved to the next line.
if (char === '\n') {
current.nextLine();
}
continue;
}
@ -96,17 +134,23 @@ function tokenizer(input) {
// We're going to create a `value` string that we are going to push
// characters to.
let value = '';
// We'll also save the start of the number for later.
const start = current.clone();
// Then we're going to loop through each character in the sequence until
// we encounter a character that is not a number, pushing each character
// that is a number to our `value` and incrementing `current` as we go.
while (NUMBERS.test(char)) {
value += char;
char = input[++current];
current.nextCh();
if (current.index >= input.length) {
break;
}
char = input[current.index];
}
// After that we push our `number` token to the `tokens` array.
tokens.push({ type: 'number', value });
tokens.push({ type: 'number', value, start, end: current.clone() });
// And we continue on.
continue;
@ -122,22 +166,41 @@ function tokenizer(input) {
if (char === '"') {
// Keep a `value` variable for building up our string token.
let value = '';
// We'll also save the start of the string for later.
const start = current.clone();
// If the quote is the last character in the program,
// throw a syntax error:
if (current.index + 1 >= input.length) {
throw new SyntaxError(`Unterminated string at ${start}-${current}`);
}
// Otherwise, skip past the quote...
current.nextCh();
// We'll skip the opening double quote in our token.
char = input[++current];
// ...and grab the first character of the string.
char = input[current.index];
// Then we'll iterate through each character until we reach another
// double quote.
while (char !== '"') {
value += char;
char = input[++current];
// If the string is not terminated before the end of the program,
// throw a syntax error
if (current.index + 1 >= input.length) {
throw new SyntaxError(`Unterminated string at ${start}-${current}`);
}
// Otherwise, increment the cursor
current.nextCh();
// And grab the next character.
char = input[current.index];
}
// Skip the closing double quote.
char = input[++current];
current.nextCh();
char = input[current.index];
// And add our `string` token to the `tokens` array.
tokens.push({ type: 'string', value });
tokens.push({ type: 'string', value, start, end: current.clone() });
continue;
}
@ -152,24 +215,28 @@ function tokenizer(input) {
//
let LETTERS = /[a-z]/i;
if (LETTERS.test(char)) {
// First, we'll create a string to hold the value
let value = '';
// And save the current position for later.
const start = current.clone();
// Again we're just going to loop through all the letters pushing them to
// a value.
while (LETTERS.test(char)) {
while (LETTERS.test(char) && current.index < input.length) {
value += char;
char = input[++current];
current.nextCh();
char = input[current.index];
}
// And pushing that value as a token with the type `name` and continuing.
tokens.push({ type: 'name', value });
tokens.push({ type: 'name', value, start, end: current.clone() });
continue;
}
// Finally if we have not matched a character by now, we're going to throw
// an error and completely exit.
throw new TypeError('I dont know what this character is: ' + char);
// a syntax error and completely exit.
throw new SyntaxError('I dont know what this character is: ' + char);
}
// Then at the end of our `tokenizer` we simply return the tokens array.

35
2-parser.js
View File

@ -14,8 +14,9 @@
// Okay, so we define a `parser` function that accepts our array of `tokens`.
function parser(tokens) {
// Again we keep a `current` variable that we will use as a cursor.
// Because we're not iterating over the source code, we are just
// using a number.
let current = 0;
// But this time we're going to use recursion instead of a `while` loop. So we
@ -24,6 +25,7 @@ function parser(tokens) {
// Inside the walk function we start by grabbing the `current` token.
let token = tokens[current];
if (!token) return;
// We're going to split each type of token off into a different code path,
// starting off with `number` tokens.
@ -39,6 +41,8 @@ function parser(tokens) {
return {
type: 'NumberLiteral',
value: token.value,
start: token.start,
end: token.end,
};
}
@ -50,6 +54,8 @@ function parser(tokens) {
return {
type: 'StringLiteral',
value: token.value,
start: token.start,
end: token.end,
};
}
@ -60,9 +66,14 @@ function parser(tokens) {
token.value === '('
) {
const initial = token;
// We'll increment `current` to skip the parenthesis since we don't care
// about it in our AST.
token = tokens[++current];
// If we've reached the end of the program, throw a syntax error.
if (!token) {
throw new SyntaxError(`Unclosed function call at ${initial.start}`);
}
// We create a base node with the type `CallExpression`, and we're going
// to set the name as the current token's value since the next token after
@ -71,10 +82,19 @@ function parser(tokens) {
type: 'CallExpression',
name: token.value,
params: [],
start: token.start,
};
// If the user writes `()` as code, throw a syntax error.
if (token.type === 'paren' && token.value === ')') {
throw new SyntaxError(`Unexpected empty function call at ${tokens[current - 1].start}–${token.end}`)
}
// We increment `current` *again* to skip the name token.
token = tokens[++current];
// If we've reached the end of the program (for example: `(foo`), throw a syntax error.
if (!token) {
throw new SyntaxError(`Unclosed function call at ${initial.start}-${tokens[current - 1].end}`);
}
// And now we want to loop through each token that will be the `params` of
// our `CallExpression` until we encounter a closing parenthesis.
@ -118,7 +138,13 @@ function parser(tokens) {
// push it into our `node.params`.
node.params.push(walk());
token = tokens[current];
// If we've reached the end of the program, throw a syntax error.
if (!token) {
throw new SyntaxError('Unclosed function call at ' + initial.start + '-' + tokens[current - 1].end);
}
}
// Next, we save the end position of the call
node.end = token.end;
// Finally we will increment `current` one last time to skip the closing
// parenthesis.
@ -127,10 +153,15 @@ function parser(tokens) {
// And return the node.
return node;
}
// If the user writes a literal name token (`foo`), throw a syntax error.
if (token.type === 'name') {
throw new SyntaxError(`Unexpected name '${token.value}' at ${token.start}–${token.end}`)
}
// Again, if we haven't recognized the token type by now we're going to
// throw an error.
throw new TypeError(token.type);
throw new SyntaxError(`Unrecognized token: ${JSON.stringify(token)}`);
}
// Now, we're going to create our AST which will have a root which is a

2
3-traverser.js
View File

@ -78,7 +78,7 @@ function traverser(ast, visitor) {
// And again, if we haven't recognized the node type then we'll throw an
// error.
default:
throw new TypeError(node.type);
throw new SyntaxError(`Unrecognized node ${node.type}: ${JSON.stringify(node)}}`);
}
// If there is an `exit` method for this node type we'll call it with the

8
7-test.html Normal file
View File

@ -0,0 +1,8 @@
<h1>Code in:</h1>
<!-- This <textarea> will hold the user-inputted code. -->
<textarea id="input" cols="100" rows="15">
</textarea>
<h1>Code out:</h1>
<!-- This will hold the generated code or an error message. -->
<pre id="output" style="overflow: scroll"></pre>

13
content.ejs Normal file
View File

@ -0,0 +1,13 @@
<main <% if (isCode) { %>class="is-code"<% } %>>
<% if (isCode) { %>
<pre id="code"><%- fileContents %></pre>
<% } else { %>
<div class="container">
<%- fileContents %>
</div>
<% } %>
<% if (fileName === '6-compiler.js') { %>
<img src="https://cdn.glitch.com/da026c15-c2dc-4ff8-bbed-d9d003c04338%2Ftumblr_mvemcyarmn1rslphyo1_400.gif?1492115698121" alt="Carlton Dance">
<% } %>
</main>

3
package.json
View File

@ -7,7 +7,8 @@
"express": "^4.15.2",
"markdown-it": "^8.3.1",
"ejs": "^2.5.6",
"prismjs": "^9000.0.1"
"prismjs": "^9000.0.1",
"body-parser": "^1.17.1"
},
"scripts": {
"start": "node server.js"

95
public/client.js Normal file
View File

@ -0,0 +1,95 @@
/* globals NProgress */
// Load pages using AJAX for a speed boost
var cache = Object.create(null); // Don’t include default Object properties
document.addEventListener('click', function (e) {
// When something gets clicked,
var target = closest(e.target, 'a');
// Check if it’s a link to a page on this domain.
if (!target || target.host !== location.host) return;
// If it is, stop the browser from loading the next page,
e.preventDefault();
// get the path of the page,
var path = getPath(target);
// and load it ourselves.
loadPage(path).then(function (json) {
// Then, once the page is loaded,
// insert the rendered HTML in the correct place
document.querySelector('main').outerHTML = json.html;
// and change the file name in the header.
document.querySelector('.js-file-name').textContent = json.context.fileName;
// Finally, update the title bar and address bar,
history.pushState(null, json.title, target.href);
// and tell listeners about the page load.
emitLoad(path, target);
})
});
function getPath(l) {
return l.pathname + l.search + l.hash;
}
function emitLoad(path, target) {
var event = new Event('page:load');
event.target = target;
event.data = {
path: path,
};
document.dispatchEvent(event);
}
function loadPage(path) {
// If the path is in the cache, load it immediately.
if (path in cache) {
NProgress.done(true); // show bar anyway
return Promise.resolve(cache[path]);
}
// Otherwise, fetch it from the server,
NProgress.start();
var promise = fetch('/api/fetch?path=' + encodeURIComponent(path)).then(function (res) {
// then convert it to JSON.
return res.json();
})
// Once we have the JSON,
promise.then(function (json) {
// cache it!
cache[path] = json;
NProgress.done();
return json
}).catch(function (error) {
// If something went wrong, log it in the console.
console.warn(error);
NProgress.done();
});
return promise;
}
// When the page loads,
document.addEventListener('page:load', function () {
// set the correct link as active.
document.querySelector('a.active').classList.remove('active');
document.querySelector('a[href="' + window.location.pathname + '"]').classList.add('active');
})
// Emit an initial load event when the page is ready
window.addEventListener('load', function () {
emitLoad(getPath(location), document)
})
if (!Element.prototype.matches && Element.prototype.matchesSelector) {
Element.prototype.matches = Element.prototype.matchesSelector;
}
function closest(element, selector) {
if (!element) return null;
if (element.matches(selector)) {
return element;
}
return closest(element.parentElement, selector);
}

75
public/nprogress.css Normal file
View File

@ -0,0 +1,75 @@
/* Make clicks pass-through */
#nprogress {
pointer-events: none;
}
#nprogress .bar {
background: #fff;
position: fixed;
z-index: 1031;
top: 0;
left: 0;
width: 100%;
height: 2px;
}
/* Fancy blur effect */
#nprogress .peg {
display: block;
position: absolute;
right: 0px;
width: 100px;
height: 100%;
box-shadow: 0 0 10px #fff, 0 0 5px #fff;
opacity: 1.0;
-webkit-transform: rotate(3deg) translate(0px, -4px);
-ms-transform: rotate(3deg) translate(0px, -4px);
transform: rotate(3deg) translate(0px, -4px);
}
/* Remove these to get rid of the spinner
#nprogress .spinner {
display: block;
position: fixed;
z-index: 1031;
top: 15px;
right: 15px;
}
#nprogress .spinner-icon {
width: 18px;
height: 18px;
box-sizing: border-box;
border: solid 2px transparent;
border-top-color: #fff;
border-left-color: #fff;
border-radius: 50%;
-webkit-animation: nprogress-spinner 400ms linear infinite;
animation: nprogress-spinner 400ms linear infinite;
}
*/
.nprogress-custom-parent {
overflow: hidden;
position: relative;
}
.nprogress-custom-parent #nprogress .spinner,
.nprogress-custom-parent #nprogress .bar {
position: absolute;
}
@-webkit-keyframes nprogress-spinner {
0% { -webkit-transform: rotate(0deg); }
100% { -webkit-transform: rotate(360deg); }
}
@keyframes nprogress-spinner {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}

32
public/test.js Normal file
View File

@ -0,0 +1,32 @@
// It would be simpler to use this if we could just `require()` the `compiler`,
// but this code runs on the client side and it would be really complicated
// to enable using `require()` here, so we just call the server instead.
document.addEventListener('page:load', function (e) {
if (e.data.path !== '/test') return;
// When the user types into the input field...
document.getElementById('input').addEventListener('input', function (event) {
// POST the code to the API
fetch('/api/convert', {
method: 'POST',
body: event.target.value,
}).then(function (res) {
// then get the JSON back
return res.json();
}).then(function (json) {
// then either:
var output = document.getElementById('output');
if (json.ok) {
// output the transformed code if
// the compilation succeeded, or
output.style.backgroundColor = 'black';
output.textContent = json.code;
} else {
// output the error if there was
// a problem.
output.style.backgroundColor = 'darkred';
output.textContent = json.stack;
}
})
})
});

101
server.js
View File

@ -1,12 +1,21 @@
var markdown = require('markdown-it')();
var Prism = require('prismjs');
var bodyParser = require('body-parser');
var express = require('express');
var path = require('path');
var ejs = require('ejs');
var fs = require('fs');
// First, let's create the server,
var app = express();
// add a `req.body` property to requests containing
// the body as text,
app.use(bodyParser.text());
// and serve any files in the `public` directory from the /assets path.
app.use('/assets', express.static(path.join(__dirname, 'public')));
// Next, let's create the URLs to read the code used:
var ROUTES_MAP = {
'/' : 'README.md',
'/intro' : '0-introduction.md',
@ -15,7 +24,11 @@ var ROUTES_MAP = {
'/traverser' : '3-traverser.js',
'/transformer' : '4-transformer.js',
'/code-generator' : '5-code-generator.js',
'/compiler' : '6-compiler.js'
'/compiler' : '6-compiler.js',
'/test' : '7-test.html',
'/server' : 'server.js',
'/client' : 'public/client.js',
'/test-js' : 'public/test.js',
};
var routes = Object.keys(ROUTES_MAP).map(function(routePath) {
@ -25,46 +38,112 @@ var routes = Object.keys(ROUTES_MAP).map(function(routePath) {
};
});
// Next, let's create helpers to read files,
function readFile(fileName) {
return fs.readFileSync(path.join(__dirname, fileName)).toString();
}
// render Markdown,
function renderMarkdown(fileContents) {
return markdown.render(fileContents);
}
// and highlight JavaScript code.
function renderJavaScript(fileName, fileContents) {
return Prism.highlight(fileContents, Prism.languages.javascript);
}
var template = ejs.compile(readFile('./template.html.ejs'));
// We'll be using EJS (http://ejs.co) to render the files.
var template = ejs.compile(readFile('./template.html.ejs'), {
filename: path.join(__dirname, 'template.html.ejs')
});
function render(routeName) {
return template(getContext(routeName));
}
function getContext(routeName) {
// We'll read the file at the specified path,
var fileName = routeName;
var fileContents = readFile(fileName);
// render it appropriately,
var extName = path.extname(fileName);
if (extName === '.md') fileContents = renderMarkdown(fileContents);
if (extName === '.js') fileContents = renderJavaScript(fileName, fileContents);
let isCode = extName !== '.md';
let isCode = extName === '.js';
return template({
routes: routes,
fileName: fileName,
fileContents: fileContents,
isCode: isCode,
});
return {
routes,
fileName,
fileContents,
isCode
}
}
// Next, let's tell Express about each file we're rendering.
routes.forEach(function(route) {
var html = render(route.routeName);
app.get(route.routePath, function(req, res) {
var html = render(route.routeName);
res.send(html);
});
});
// To convert the code,
app.post('/api/convert', function(req, res) {
try {
// First, try to convert the code and send it back.
var code = require('./6-compiler')(req.body);
res.send({
ok: true,
code: code,
});
} catch (err) {
try {
// If that fails, send the error back.
res.send({
ok: false,
err: err.toString(),
stack: err.stack,
});
} catch (e) {
// If sending the error fails, send a generic error.
res.send({
ok: false,
err: 'unknown error',
stack: 'unknown error\n<no stack>',
});
// (if *that* fails, you're on your own :))
}
}
});
var contentTemplate = ejs.compile(readFile('./content.ejs'), {
filename: path.join(__dirname, 'content.ejs')
});
var titleTemplate = ejs.compile(readFile('./title.ejs'), {
filename: path.join(__dirname, 'title.ejs')
});
app.get('/api/fetch', function(req, res) {
var path = ROUTES_MAP[req.query.path]
if (!path) {
res.status('404').send({
error: 'route not found'
});
}
var context = getContext(path);
res.send({
title: titleTemplate(context),
html: contentTemplate(context),
context
});
});
// Finally, start the server.
var listener = app.listen(process.env.PORT, function () {
console.log('Your app is listening on port ' + listener.address().port);
});

37
template.html.ejs
View File

@ -1,12 +1,18 @@
<!doctype html>
<html <% if (isCode) { %>class="is-code"<% } %>>
<html>
<head>
<title>The Super Tiny Compiler - <%= fileName %></title>
<title><%- include('title') %></title>
<meta name="description" content="">
<link id="favicon" rel="icon" href="https://glitch.com/edit/favicon-app.ico" type="image/x-icon">
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdnjs.cloudflare.com/ajax/libs/nprogress/0.2.0/nprogress.min.js"></script>
<!-- We're including this to allow us to use `fetch` in more browsers. -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/fetch/2.0.3/fetch.min.js"></script>
<script src="/assets/client.js"></script>
<script src="/assets/test.js"></script>
<link rel="stylesheet" href="/assets/nprogress.css" />
<style>
* {
box-sizing: border-box;
@ -24,10 +30,12 @@
overflow: hidden;
}
html.is-code,
.is-code body {
#app {
background: black;
}
main:not(.is-code) {
background: white;
}
header {
position: absolute;
@ -85,7 +93,11 @@
left: 300px;
right: 0;
overflow: auto;
padding-bottom: 25%;
/* The topbar, the padding, and the last line */
padding-bottom: calc(100vh - 2em - 1em - 2em);
}
main pre#code {
padding-bottom: 0;
}
.container {
@ -278,7 +290,7 @@
<div id="app">
<header>
<a href="https://github.com/thejameskyle/the-super-tiny-compiler">
/Users/thejameskyle/code/the-super-tiny-compiler/<%= fileName %>
/Users/thejameskyle/code/the-super-tiny-compiler/<span class="js-file-name"><%= fileName %></span>
</a>
<a class="right" href="https://github.com/thejameskyle/the-super-tiny-compiler">
@ -302,18 +314,7 @@
<% }); %>
</nav>
<main>
<% if (isCode) { %>
<pre id="code"><%- fileContents %></pre>
<% } else { %>
<div class="container">
<%- fileContents %>
</div>
<% } %>
<% if (fileName === '6-compiler.js') { %>
<img src="https://cdn.glitch.com/da026c15-c2dc-4ff8-bbed-d9d003c04338%2Ftumblr_mvemcyarmn1rslphyo1_400.gif?1492115698121" alt="Carlton Dance">
<% } %>
<%- include('content') %>
</main>
</div>
</body>

1
title.ejs Normal file
View File

@ -0,0 +1 @@
The Super Tiny Compiler - <%= fileName %>

18
watch.json Normal file
View File

@ -0,0 +1,18 @@
{
"install": {
"include": [
"^package\\.json$",
"^\\.env$"
]
},
"restart": {
"exclude": [
"^public/",
"^dist/"
],
"include": [
"\\.ejs$"
]
},
"throttle": 100
}