merge master -> crawl-queue

src/api/build.ts

@@ -3,7 +3,7 @@ import * as path from 'path';
 import mkdirp from 'mkdirp';
 import rimraf from 'rimraf';
 import { EventEmitter } from 'events';
-import { minify_html } from './utils/minify_html';
+import minify_html from './utils/minify_html';
 import { create_compilers, create_main_manifests, create_routes, create_serviceworker_manifest } from '../core';
 import * as events from './interfaces';
 

src/api/dev.ts

@@ -119,21 +119,30 @@ class Watcher extends EventEmitter {
 		this.dev_server = new DevServer(dev_port);
 
 		this.filewatchers.push(
-			watch_files(locations.routes(), ['add', 'unlink'], () => {
-				const routes = create_routes();
-				create_main_manifests({ routes, dev_port });
-
-				try {
+			watch_dir(
+				locations.routes(),
+				({ path: file, stats }) => {
+					if (stats.isDirectory()) {
+						return path.basename(file)[0] !== '_';
+					}
+					return true;
+				},
+				() => {
 					const routes = create_routes();
 					create_main_manifests({ routes, dev_port });
-				} catch (err) {
-					this.emit('error', <events.ErrorEvent>{
-						message: err.message
-					});
-				}
-			}),
 
-			watch_files(`${locations.app()}/template.html`, ['change'], () => {
+					try {
+						const routes = create_routes();
+						create_main_manifests({ routes, dev_port });
+					} catch (err) {
+						this.emit('error', <events.ErrorEvent>{
+							message: err.message
+						});
+					}
+				}
+			),
+
+			fs.watch(`${locations.app()}/template.html`, () => {
 				this.dev_server.send({
 					action: 'reload'
 				});
@@ -453,20 +462,32 @@ class DevServer {
 
 function noop() {}
 
-function watch_files(pattern: string, events: string[], callback: () => void) {
-	const chokidar = require('chokidar');
+function watch_dir(
+	dir: string,
+	filter: ({ path, stats }: { path: string, stats: fs.Stats }) => boolean,
+	callback: () => void
+) {
+	let watch;
+	let closed = false;
 
-	const watcher = chokidar.watch(pattern, {
-		persistent: true,
-		ignoreInitial: true,
-		disableGlobbing: true
-	});
+	import('cheap-watch').then(CheapWatch => {
+		if (closed) return;
 
-	events.forEach(event => {
-		watcher.on(event, callback);
+		watch = new CheapWatch({ dir, filter, debounce: 50 });
+
+		watch.on('+', ({ isNew }) => {
+			if (isNew) callback();
+		});
+
+		watch.on('-', callback);
+
+		watch.init();
 	});
 
 	return {
-		close: () => watcher.close()
+		close: () => {
+			if (watch) watch.close();
+			closed = true;
+		}
 	};
-}
+}

src/api/export.ts

@@ -1,13 +1,13 @@
 import * as child_process from 'child_process';
 import * as path from 'path';
 import * as sander from 'sander';
-import cheerio from 'cheerio';
 import URL from 'url-parse';
 import fetch from 'node-fetch';
 import * as yootils from 'yootils';
 import * as ports from 'port-authority';
 import { EventEmitter } from 'events';
-import { minify_html } from './utils/minify_html';
+import clean_html from './utils/clean_html';
+import minify_html from './utils/minify_html';
 import Deferred from './utils/Deferred';
 import * as events from './interfaces';
 
@@ -124,20 +124,31 @@ async function execute(emitter: EventEmitter, {
 		if (range === 2) {
 			if (r.headers.get('Content-Type') === 'text/html') {
 				const body = await r.text();
-				const $ = cheerio.load(body);
 				const urls: URL[] = [];
 
-				const base = new URL($('base').attr('href') || '/', url.href);
+				const cleaned = clean_html(body);
 
 				const q = yootils.queue(8);
 				let promise;
 
-				$('a[href]').each((i: number, $a) => {
-					const url = new URL($a.attribs.href, base.href);
-					if (url.origin === origin) {
-						promise = q.add(() => handle(url));
+				const base_match = /<base ([\s\S]+?)>/m.exec(cleaned);
+				const base_href = base_match && get_href(base_match[1]);
+				const base = new URL(base_href || '/', url.href);
+
+				let match;
+				let pattern = /<a ([\s\S]+?)>/gm;
+
+				while (match = pattern.exec(cleaned)) {
+					const attrs = match[1];
+					const href = get_href(attrs);
+
+					if (href) {
+						const url = new URL(href, base.href);
+						if (url.origin === origin) {
+							promise = q.add(() => handle(url));
+						}
 					}
-				});
+				}
 
 				await promise;
 			}
@@ -153,3 +164,8 @@ async function execute(emitter: EventEmitter, {
 		})
 		.then(() => proc.kill());
 }
+
+function get_href(attrs: string) {
+	const match = /href\s*=\s*(?:"(.+?)"|'(.+?)'|([^\s>]+))/.exec(attrs);
+	return match[1] || match[2] || match[3];
+}

src/api/find_page.ts

@@ -1,4 +1,3 @@
-import * as glob from 'glob';
 import { locations } from '../config';
 import { create_routes } from '../core';
 

src/api/utils/clean_html.ts

@@ -0,0 +1,7 @@
+export default function clean_html(html: string) {
+	return html
+		.replace(/<!\[CDATA\[[\s\S]*?\]\]>/gm, '')
+		.replace(/(<script[\s\S]*?>)[\s\S]*?<\/script>/gm, '$1</' + 'script>')
+		.replace(/(<style[\s\S]*?>)[\s\S]*?<\/style>/gm, '$1</' + 'style>')
+		.replace(/<!--[\s\S]*?-->/gm, '');
+}

src/api/utils/minify_html.ts

@@ -1,6 +1,6 @@
 import { minify } from 'html-minifier';
 
-export function minify_html(html: string) {
+export default function minify_html(html: string) {
 	return minify(html, {
 		collapseBooleanAttributes: true,
 		collapseWhitespace: true,

src/cli.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import sade from 'sade';
-import * as colors from 'ansi-colors';
+import colors from 'kleur';
 import prettyMs from 'pretty-ms';
 import * as pkg from '../package.json';
 

src/cli/build.ts

@@ -1,5 +1,5 @@
 import { build as _build } from '../api/build';
-import * as colors from 'ansi-colors';
+import colors from 'kleur';
 import { locations } from '../config';
 
 export function build() {

src/cli/dev.ts

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import * as colors from 'ansi-colors';
+import colors from 'kleur';
 import * as child_process from 'child_process';
 import prettyMs from 'pretty-ms';
 import { dev as _dev } from '../api/dev';

src/cli/export.ts

@@ -1,5 +1,5 @@
 import { exporter as _exporter } from '../api/export';
-import * as colors from 'ansi-colors';
+import colors from 'kleur';
 import prettyBytes from 'pretty-bytes';
 import { locations } from '../config';
 

src/cli/start.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as child_process from 'child_process';
-import * as colors from 'ansi-colors';
+import colors from 'kleur';
 import * as ports from 'port-authority';
 
 export async function start(dir: string, opts: { port: number, open: boolean }) {

src/cli/upgrade.ts

@@ -1,5 +1,5 @@
 import * as fs from 'fs';
-import * as colors from 'ansi-colors';
+import colors from 'kleur';
 
 export default async function upgrade() {
 	const upgraded = [

src/core/create_manifests.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import * as glob from 'glob';
+import glob from 'tiny-glob/sync.js';
 import { posixify, write_if_changed } from './utils';
 import { dev, locations } from '../config';
 import { Page, PageComponent, ServerRoute } from '../interfaces';
@@ -9,24 +9,27 @@ export function create_main_manifests({ routes, dev_port }: {
 	routes: { components: PageComponent[], pages: Page[], server_routes: ServerRoute[] };
 	dev_port?: number;
 }) {
-	const path_to_routes = path.relative(`${locations.app()}/manifest`, locations.routes());
+	const manifest_dir = path.join(locations.app(), 'manifest');
+	if (!fs.existsSync(manifest_dir)) fs.mkdirSync(manifest_dir);
+
+	const path_to_routes = path.relative(manifest_dir, locations.routes());
 
 	const client_manifest = generate_client(routes, path_to_routes, dev_port);
 	const server_manifest = generate_server(routes, path_to_routes);
 
 	write_if_changed(
-		`${locations.app()}/manifest/default-layout.html`,
+		`${manifest_dir}/default-layout.html`,
 		`<svelte:component this={child.component} {...child.props}/>`
 	);
-	write_if_changed(`${locations.app()}/manifest/client.js`, client_manifest);
-	write_if_changed(`${locations.app()}/manifest/server.js`, server_manifest);
+	write_if_changed(`${manifest_dir}/client.js`, client_manifest);
+	write_if_changed(`${manifest_dir}/server.js`, server_manifest);
 }
 
 export function create_serviceworker_manifest({ routes, client_files }: {
 	routes: { components: PageComponent[], pages: Page[], server_routes: ServerRoute[] };
 	client_files: string[];
 }) {
-	const assets = glob.sync('**', { cwd: 'assets', nodir: true });
+	const assets = glob('**', { cwd: 'assets', filesOnly: true });
 
 	let code = `
 		// This file is generated by Sapper — do not edit it!
@@ -42,11 +45,6 @@ export function create_serviceworker_manifest({ routes, client_files }: {
 	write_if_changed(`${locations.app()}/manifest/service-worker.js`, code);
 }
 
-function right_pad(str: string, len: number) {
-	while (str.length < len) str += ' ';
-	return str;
-}
-
 function generate_client(
 	routes: { root: PageComponent, components: PageComponent[], pages: Page[], server_routes: ServerRoute[] },
 	path_to_routes: string,

src/core/create_routes.ts

@@ -4,11 +4,6 @@ import { locations } from '../config';
 import { Page, PageComponent, ServerRoute } from '../interfaces';
 import { posixify } from './utils';
 
-const default_layout_file = posixify(path.resolve(
-	__dirname,
-	'../components/default-layout.html'
-));
-
 export default function create_routes(cwd = locations.routes()) {
 	const components: PageComponent[] = [];
 	const pages: Page[] = [];

src/core/utils.ts

@@ -1,11 +1,11 @@
-import * as sander from 'sander';
+import * as fs from 'fs';
 
 const previous_contents = new Map();
 
 export function write_if_changed(file: string, code: string) {
 	if (code !== previous_contents.get(file)) {
 		previous_contents.set(file, code);
-		sander.writeFileSync(file, code);
+		fs.writeFileSync(file, code);
 		fudge_mtime(file);
 	}
 }
@@ -16,8 +16,8 @@ export function posixify(file: string) {
 
 export function fudge_mtime(file: string) {
 	// need to fudge the mtime so that webpack doesn't go doolally
-	const { atime, mtime } = sander.statSync(file);
-	sander.utimesSync(
+	const { atime, mtime } = fs.statSync(file);
+	fs.utimesSync(
 		file,
 		new Date(atime.getTime() - 999999),
 		new Date(mtime.getTime() - 999999)