wip 2

src/core/build.js

@@ -0,0 +1,47 @@
+process.env.NODE_ENV = 'production';
+
+import * as fs from 'fs';
+import * as path from 'path';
+import mkdirp from 'mkdirp';
+import rimraf from 'rimraf';
+import { client, server } from './utils/compilers.js';
+import create_app from './utils/create_app.js';
+import generate_asset_cache from './generate_asset_cache.js';
+import { dest } from '../config.js';
+
+export default () => {
+	mkdirp.sync(dest);
+	rimraf.sync(path.join(dest, '**/*'));
+
+	// create main.js and server-routes.js
+	create_app();
+
+	return new Promise((fulfil, reject) => {
+		function handleErrors(err, stats) {
+			if (err) {
+				reject(err);
+				process.exit(1);
+			}
+
+			if (stats.hasErrors()) {
+				console.error(stats.toString({ colors: true }));
+				reject(new Error(`Encountered errors while building app`));
+			}
+		}
+
+		client.run((err, clientStats) => {
+			handleErrors(err, clientStats);
+			const clientInfo = clientStats.toJson();
+			fs.writeFileSync(path.join(dest, 'stats.client.json'), JSON.stringify(clientInfo, null, '  '));
+
+			server.run((err, serverStats) => {
+				handleErrors(err, serverStats);
+				const serverInfo = serverStats.toJson();
+				fs.writeFileSync(path.join(dest, 'stats.server.json'), JSON.stringify(serverInfo, null, '  '));
+
+				generate_asset_cache(clientInfo, serverInfo);
+				fulfil();
+			});
+		});
+	});
+};

src/core/export.js

@@ -0,0 +1,100 @@
+import * as path from 'path';
+import * as sander from 'sander';
+import express from 'express';
+import cheerio from 'cheerio';
+import fetch from 'node-fetch';
+import URL from 'url-parse';
+import generate_asset_cache from './generate_asset_cache.js';
+import { dest } from '../config.js';
+import middleware from '../middleware/index.js';
+
+const { PORT = 3000, OUTPUT_DIR = 'dist' } = process.env;
+
+const origin = `http://localhost:${PORT}`;
+
+const app = express();
+
+function read_json(file) {
+	return JSON.parse(sander.readFileSync(file, { encoding: 'utf-8' }));
+}
+
+export default function() {
+	// Prep output directory
+	sander.rimrafSync(OUTPUT_DIR);
+
+	const { service_worker } = generate_asset_cache(
+		read_json(path.join(dest, 'stats.client.json')),
+		read_json(path.join(dest, 'stats.server.json'))
+	);
+
+	sander.copydirSync('assets').to(OUTPUT_DIR);
+	sander.copydirSync(dest, 'client').to(OUTPUT_DIR, 'client');
+	sander.writeFileSync(OUTPUT_DIR, 'service-worker.js', service_worker);
+
+	// Intercept server route fetches
+	function save(res) {
+		res = res.clone();
+
+		return res.text().then(body => {
+			const { pathname } = new URL(res.url);
+			let dest = OUTPUT_DIR + pathname;
+
+			const type = res.headers.get('Content-Type');
+			if (type.startsWith('text/html')) dest += '/index.html';
+
+			sander.writeFileSync(dest, body);
+
+			return body;
+		});
+	}
+
+	global.fetch = (url, opts) => {
+		if (url[0] === '/') {
+			url = `http://localhost:${PORT}${url}`;
+
+			return fetch(url, opts)
+				.then(r => {
+					save(r);
+					return r;
+				});
+		}
+
+		return fetch(url, opts);
+	};
+
+	app.use(middleware());
+	const server = app.listen(PORT);
+
+	const seen = new Set();
+
+	function handle(url) {
+		if (url.origin !== origin) return;
+
+		if (seen.has(url.pathname)) return;
+		seen.add(url.pathname);
+
+		return fetch(url.href)
+			.then(r => {
+				save(r);
+				return r.text();
+			})
+			.then(body => {
+				const $ = cheerio.load(body);
+				const hrefs = [];
+
+				$('a[href]').each((i, $a) => {
+					hrefs.push($a.attribs.href);
+				});
+
+				return hrefs.reduce((promise, href) => {
+					return promise.then(() => handle(new URL(href, url.href)));
+				}, Promise.resolve());
+			})
+			.catch(err => {
+				console.error(`Error rendering ${url.pathname}: ${err.message}`);
+			});
+	}
+
+	return handle(new URL(origin)) // TODO all static routes
+		.then(() => server.close());
+}

src/core/generate_asset_cache.js

@@ -0,0 +1,81 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import glob from 'glob';
+import * as templates from './templates.js';
+import * as route_manager from './route_manager.js';
+import { dest, dev } from '../config.js';
+
+function ensure_array(thing) {
+	return Array.isArray(thing) ? thing : [thing]; // omg webpack what the HELL are you doing
+}
+
+export default function generate_asset_cache(clientInfo, serverInfo) {
+	const main_file = `/client/${ensure_array(clientInfo.assetsByChunkName.main)[0]}`;
+
+	const chunk_files = clientInfo.assets.map(chunk => `/client/${chunk.name}`);
+
+	const service_worker = generate_service_worker(chunk_files);
+	const index = generate_index(main_file);
+
+	if (dev) {
+		fs.writeFileSync(path.join(dest, 'service-worker.js'), service_worker);
+		fs.writeFileSync(path.join(dest, 'index.html'), index);
+	}
+
+	return {
+		client: {
+			main_file,
+			chunk_files,
+
+			main: read(`${dest}${main_file}`),
+			chunks: chunk_files.reduce((lookup, file) => {
+				lookup[file] = read(`${dest}${file}`);
+				return lookup;
+			}, {}),
+
+			routes: route_manager.routes.reduce((lookup, route) => {
+				lookup[route.id] = `/client/${ensure_array(clientInfo.assetsByChunkName[route.id])[0]}`;
+				return lookup;
+			}, {}),
+
+			index,
+			service_worker
+		},
+
+		server: {
+			entry: path.resolve(dest, 'server', serverInfo.assetsByChunkName.main)
+		},
+
+		service_worker
+	};
+}
+
+function generate_service_worker(chunk_files) {
+	const assets = glob.sync('**', { cwd: 'assets', nodir: true });
+
+	const route_code = `[${
+		route_manager.routes
+			.filter(route => route.type === 'page')
+			.map(route => `{ pattern: ${route.pattern} }`)
+			.join(', ')
+	}]`;
+
+	return read('templates/service-worker.js')
+		.replace(/__timestamp__/g, Date.now())
+		.replace(/__assets__/g, JSON.stringify(assets))
+		.replace(/__shell__/g, JSON.stringify(chunk_files.concat('/index.html')))
+		.replace(/__routes__/g, route_code);
+}
+
+function generate_index(main_file) {
+	return templates.render(200, {
+		styles: '',
+		head: '',
+		html: '<noscript>Please enable JavaScript!</noscript>',
+		main: main_file
+	});
+}
+
+function read(file) {
+	return fs.readFileSync(file, 'utf-8');
+}

src/core/index.js

@@ -0,0 +1,10 @@
+import * as route_manager from './route_manager.js';
+import * as templates from './templates.js';
+import * as compilers from './utils/compilers.js';
+
+export { default as build } from './build.js';
+export { default as export } from './export.js';
+export { default as generate_asset_cache } from './generate_asset_cache.js';
+export { default as create_app } from './utils/create_app.js';
+
+export { compilers, route_manager, templates };

src/core/route_manager.js

@@ -0,0 +1,32 @@
+import glob from 'glob';
+import create_routes from './utils/create_routes.js';
+import { src, dev } from '../config.js';
+
+const callbacks = [];
+
+export function onchange(fn) {
+	callbacks.push(fn);
+}
+
+export let routes;
+
+function update() {
+	routes = create_routes(
+		glob.sync('**/*.+(html|js|mjs)', { cwd: src })
+	);
+
+	callbacks.forEach(fn => fn());
+}
+
+update();
+
+if (dev) {
+	const watcher = require('chokidar').watch(`${src}/**/*.+(html|js|mjs)`, {
+		ignoreInitial: true,
+		persistent: false
+	});
+
+	watcher.on('add', update);
+	watcher.on('change', update);
+	watcher.on('unlink', update);
+}

src/core/templates.js

@@ -0,0 +1,127 @@
+import * as fs from 'fs';
+import glob from 'glob';
+import chalk from 'chalk';
+import framer from 'code-frame';
+import { locate } from 'locate-character';
+import { dev } from '../config.js';
+
+let templates;
+
+function error(e) {
+	if (e.title) console.error(chalk.bold.red(e.title));
+	if (e.body) console.error(chalk.red(e.body));
+	if (e.url) console.error(chalk.cyan(e.url));
+	if (e.frame) console.error(chalk.grey(e.frame));
+
+	process.exit(1);
+}
+
+function create_templates() {
+	templates = glob.sync('*.html', { cwd: 'templates' })
+		.map(file => {
+			const template = fs.readFileSync(`templates/${file}`, 'utf-8');
+			const status = file.replace('.html', '').toLowerCase();
+
+			if (!/^[0-9x]{3}$/.test(status)) {
+				error({
+					title: `templates/${file}`,
+					body: `Bad template — should be a valid status code like 404.html, or a wildcard like 2xx.html`
+				});
+			}
+
+			const index = template.indexOf('%sapper.main%');
+			if (index !== -1) {
+				// TODO remove this in a future version
+				const { line, column } = locate(template, index, { offsetLine: 1 });
+				const frame = framer(template, line, column);
+
+				error({
+					title: `templates/${file}`,
+					body: `<script src='%sapper.main%'> is unsupported — use %sapper.scripts% (without the <script> tag) instead`,
+					url: 'https://github.com/sveltejs/sapper/issues/86',
+					frame
+				});
+			}
+
+			const specificity = (
+				(status[0] === 'x' ? 0 : 4) +
+				(status[1] === 'x' ? 0 : 2) +
+				(status[2] === 'x' ? 0 : 1)
+			);
+
+			const pattern = new RegExp(`^${status.split('').map(d => d === 'x' ? '\\d' : d).join('')}$`);
+
+			return {
+				test: status => pattern.test(status),
+				specificity,
+				render: data => {
+					return template.replace(/%sapper\.(\w+)%/g, (match, key) => {
+						return key in data ? data[key] : '';
+					});
+				},
+				stream: (res, data) => {
+					let i = 0;
+
+					function stream_inner() {
+						if (i >= template.length) {
+							return;
+						}
+
+						const start = template.indexOf('%sapper', i);
+
+						if (start === -1) {
+							res.end(template.slice(i));
+							return;
+						}
+
+						res.write(template.slice(i, start));
+
+						const end = template.indexOf('%', start + 1);
+						if (end === -1) {
+							throw new Error(`Bad template`); // TODO validate ahead of time
+						}
+
+						const tag = template.slice(start + 1, end);
+						const match = /sapper\.(\w+)/.exec(tag);
+						if (!match || !(match[1] in data)) throw new Error(`Bad template`); // TODO ditto
+
+						return Promise.resolve(data[match[1]]).then(datamatch => {
+							res.write(datamatch);
+							i = end + 1;
+							return stream_inner();
+						});
+					}
+
+					return Promise.resolve().then(stream_inner);
+				}
+			};
+		})
+		.sort((a, b) => b.specificity - a.specificity);
+}
+
+create_templates();
+
+if (dev) {
+	const watcher = require('chokidar').watch('templates/**.html', {
+		ignoreInitial: true,
+		persistent: false
+	});
+
+	watcher.on('add', create_templates);
+	watcher.on('change', create_templates);
+	watcher.on('unlink', create_templates);
+}
+
+export function render(status, data) {
+	const template = templates.find(template => template.test(status));
+	if (template) return template.render(data);
+
+	return `Missing template for status code ${status}`;
+}
+
+export function stream(res, status, data) {
+	const template = templates.find(template => template.test(status));
+	if (template) return template.stream(res, data);
+
+	return `Missing template for status code ${status}`;
+}

src/core/utils/compilers.js

@@ -0,0 +1,12 @@
+import * as path from 'path';
+import relative from 'require-relative';
+
+const webpack = relative('webpack', process.cwd());
+
+export const client = webpack(
+	require(path.resolve('webpack.client.config.js'))
+);
+
+export const server = webpack(
+	require(path.resolve('webpack.server.config.js'))
+);

src/core/utils/create_app.js

@@ -0,0 +1,82 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as route_manager from '../route_manager.js';
+import { src, entry, dev } from '../../config.js';
+
+function posixify(file) {
+	return file.replace(/[/\\]/g, '/');
+}
+
+function create_app() {
+	const { routes } = route_manager;
+
+	function create_client_main() {
+		const template = fs.readFileSync('templates/main.js', 'utf-8');
+
+		const code = `[${
+			routes
+				.filter(route => route.type === 'page')
+				.map(route => {
+					const params = route.dynamic.length === 0 ?
+						'{}' :
+						`{ ${route.dynamic.map((part, i) => `${part}: match[${i + 1}]`).join(', ') } }`;
+
+					const file = posixify(`${src}/${route.file}`);
+					return `{ pattern: ${route.pattern}, params: match => (${params}), load: () => import(/* webpackChunkName: "${route.id}" */ '${file}') }`
+				})
+				.join(', ')
+		}]`;
+
+		let main = template
+			.replace(/__app__/g, posixify(path.resolve(__dirname, '../../runtime/app.js')))
+			.replace(/__routes__/g, code)
+			.replace(/__dev__/g, String(dev));
+
+		if (dev) {
+			const hmr_client = posixify(require.resolve(`webpack-hot-middleware/client`));
+			main += `\n\nimport('${hmr_client}?path=/__webpack_hmr&timeout=20000'); if (module.hot) module.hot.accept();`
+		}
+
+		fs.writeFileSync(entry.client, main);
+
+		// need to fudge the mtime, because webpack is soft in the head
+		const { atime, mtime } = fs.statSync(entry.client);
+		fs.utimesSync(entry.client, new Date(atime.getTime() - 999999), new Date(mtime.getTime() - 999999));
+	}
+
+	function create_server_routes() {
+		const imports = routes
+			.map(route => {
+				const file = posixify(`${src}/${route.file}`);
+				return route.type === 'page' ?
+					`import ${route.id} from '${file}';` :
+					`import * as ${route.id} from '${file}';`;
+			})
+			.join('\n');
+
+		const exports = `export { ${routes.map(route => route.id)} };`;
+
+		fs.writeFileSync(entry.server, `${imports}\n\n${exports}`);
+
+		const { atime, mtime } = fs.statSync(entry.server);
+		fs.utimesSync(entry.server, new Date(atime.getTime() - 999999), new Date(mtime.getTime() - 999999));
+	}
+
+	create_client_main();
+	create_server_routes();
+}
+
+if (dev) {
+	route_manager.onchange(create_app);
+
+	const watcher = require('chokidar').watch(`templates/main.js`, {
+		ignoreInitial: true,
+		persistent: false
+	});
+
+	watcher.on('add', create_app);
+	watcher.on('change', create_app);
+	watcher.on('unlink', create_app);
+}
+
+export default create_app;

src/core/utils/create_routes.js

@@ -0,0 +1,90 @@
+import * as path from 'path';
+
+export default function create_matchers(files) {
+	const routes = files
+		.map(file => {
+			if (/(^|\/|\\)_/.test(file)) return;
+
+			const parts = file.replace(/\.(html|js|mjs)$/, '').split('/'); // glob output is always posix-style
+			if (parts[parts.length - 1] === 'index') parts.pop();
+
+			const id = (
+				parts.join('_').replace(/[[\]]/g, '$').replace(/^\d/, '_$&').replace(/[^a-zA-Z0-9_$]/g, '_')
+			 ) || '_';
+
+			const dynamic = parts
+				.filter(part => part[0] === '[')
+				.map(part => part.slice(1, -1));
+
+			let pattern_string = '';
+			let i = parts.length;
+			let nested = true;
+			while (i--) {
+				const part = parts[i];
+				const dynamic = part[0] === '[';
+
+				if (dynamic) {
+					pattern_string = nested ? `(?:\\/([^/]+)${pattern_string})?` : `\\/([^/]+)${pattern_string}`;
+				} else {
+					nested = false;
+					pattern_string = `\\/${part}${pattern_string}`;
+				}
+			}
+
+			const pattern = new RegExp(`^${pattern_string}\\/?$`);
+
+			const test = url => pattern.test(url);
+
+			const exec = url => {
+				const match = pattern.exec(url);
+				if (!match) return;
+
+				const params = {};
+				dynamic.forEach((param, i) => {
+					params[param] = match[i + 1];
+				});
+
+				return params;
+			};
+
+			return {
+				id,
+				type: path.extname(file) === '.html' ? 'page' : 'route',
+				file,
+				pattern,
+				test,
+				exec,
+				parts,
+				dynamic
+			};
+		})
+		.filter(Boolean)
+		.sort((a, b) => {
+			let same = true;
+
+			for (let i = 0; true; i += 1) {
+				const a_part = a.parts[i];
+				const b_part = b.parts[i];
+
+				if (!a_part && !b_part) {
+					if (same) throw new Error(`The ${a.file} and ${b.file} routes clash`);
+					return 0;
+				}
+
+				if (!a_part) return -1;
+				if (!b_part) return 1;
+
+				const a_is_dynamic = a_part[0] === '[';
+				const b_is_dynamic = b_part[0] === '[';
+
+				if (a_is_dynamic === b_is_dynamic) {
+					if (!a_is_dynamic && a_part !== b_part) same = false;
+					continue;
+				}
+
+				return a_is_dynamic ? 1 : -1;
+			}
+		});
+
+	return routes;
+}