CSS extraction and code-splitting

closes #388

src/core/create_compilers/RollupCompiler.ts

@@ -0,0 +1,160 @@
+import * as path from 'path';
+import relative from 'require-relative';
+import { CompileResult } from './interfaces';
+import RollupResult from './RollupResult';
+
+let rollup: any;
+
+export default class RollupCompiler {
+	_: Promise<any>;
+	_oninvalid: (filename: string) => void;
+	_start: number;
+	input: string;
+	warnings: any[];
+	errors: any[];
+	chunks: any[];
+	css_files: Array<{ id: string, code: string }>;
+
+	constructor(config: string) {
+		this._ = this.get_config(path.resolve(config));
+		this.input = null;
+		this.warnings = [];
+		this.errors = [];
+		this.chunks = [];
+		this.css_files = [];
+	}
+
+	async get_config(input: string) {
+		if (!rollup) rollup = relative('rollup', process.cwd());
+
+		const bundle = await rollup.rollup({
+			input,
+			external: (id: string) => {
+				return (id[0] !== '.' && !path.isAbsolute(id)) || id.slice(-5, id.length) === '.json';
+			}
+		});
+
+		const { code } = await bundle.generate({ format: 'cjs' });
+
+		// temporarily override require
+		const defaultLoader = require.extensions['.js'];
+		require.extensions['.js'] = (module: any, filename: string) => {
+			if (filename === input) {
+				module._compile(code, filename);
+			} else {
+				defaultLoader(module, filename);
+			}
+		};
+
+		const mod: any = require(input);
+		delete require.cache[input];
+
+		(mod.plugins || (mod.plugins = [])).push({
+			name: 'sapper-internal',
+			options: (opts: any) => {
+				this.input = opts.input;
+			},
+			renderChunk: (code: string, chunk: any) => {
+				this.chunks.push(chunk);
+			},
+			transform: (code: string, id: string) => {
+				if (/\.css$/.test(id)) {
+					this.css_files.push({ id, code });
+					return ``;
+				}
+			}
+		});
+
+		const onwarn = mod.onwarn || ((warning: any, handler: (warning: any) => void) => {
+			handler(warning);
+		});
+
+		mod.onwarn = (warning: any) => {
+			onwarn(warning, (warning: any) => {
+				this.warnings.push(warning);
+			});
+		};
+
+		return mod;
+	}
+
+	oninvalid(cb: (filename: string) => void) {
+		this._oninvalid = cb;
+	}
+
+	async compile(): Promise<CompileResult> {
+		const config = await this._;
+
+		const start = Date.now();
+
+		try {
+			const bundle = await rollup.rollup(config);
+			await bundle.write(config.output);
+
+			return new RollupResult(Date.now() - start, this);
+		} catch (err) {
+			if (err.filename) {
+				// TODO this is a bit messy. Also, can
+				// Rollup emit other kinds of error?
+				err.message = [
+					`Failed to build — error in ${err.filename}: ${err.message}`,
+					err.frame
+				].filter(Boolean).join('\n');
+			}
+
+			throw err;
+		}
+	}
+
+	async watch(cb: (err?: Error, stats?: any) => void) {
+		const config = await this._;
+
+		const watcher = rollup.watch(config);
+
+		watcher.on('change', (id: string) => {
+			this.chunks = [];
+			this.warnings = [];
+			this.errors = [];
+			this._oninvalid(id);
+		});
+
+		watcher.on('event', (event: any) => {
+			switch (event.code) {
+				case 'FATAL':
+					// TODO kill the process?
+					if (event.error.filename) {
+						// TODO this is a bit messy. Also, can
+						// Rollup emit other kinds of error?
+						event.error.message = [
+							`Failed to build — error in ${event.error.filename}: ${event.error.message}`,
+							event.error.frame
+						].filter(Boolean).join('\n');
+					}
+
+					cb(event.error);
+					break;
+
+				case 'ERROR':
+					this.errors.push(event.error);
+					cb(null, new RollupResult(Date.now() - this._start, this));
+					break;
+
+				case 'START':
+				case 'END':
+					// TODO is there anything to do with this info?
+					break;
+
+				case 'BUNDLE_START':
+					this._start = Date.now();
+					break;
+
+				case 'BUNDLE_END':
+					cb(null, new RollupResult(Date.now() - this._start, this));
+					break;
+
+				default:
+					console.log(`Unexpected event ${event.code}`);
+			}
+		});
+	}
+}

src/core/create_compilers/RollupResult.ts

@@ -0,0 +1,111 @@
+import * as path from 'path';
+import colors from 'kleur';
+import pb from 'pretty-bytes';
+import RollupCompiler from './RollupCompiler';
+import extract_css from './extract_css';
+import { left_pad } from '../../utils';
+import { CompileResult, BuildInfo, CompileError, Chunk, CssFile } from './interfaces';
+import { ManifestData, Dirs, PageComponent } from '../../interfaces';
+
+export default class RollupResult implements CompileResult {
+	duration: number;
+	errors: CompileError[];
+	warnings: CompileError[];
+	chunks: Chunk[];
+	assets: Record<string, string>;
+	css_files: CssFile[];
+	css: {
+		main: string,
+		chunks: Record<string, string[]>
+	};
+	summary: string;
+
+	constructor(duration: number, compiler: RollupCompiler) {
+		this.duration = duration;
+
+		this.errors = compiler.errors.map(munge_warning_or_error);
+		this.warnings = compiler.warnings.map(munge_warning_or_error); // TODO emit this as they happen
+
+		this.chunks = compiler.chunks.map(chunk => ({
+			file: chunk.fileName,
+			imports: chunk.imports.filter(Boolean),
+			modules: Object.keys(chunk.modules)
+		}));
+
+		this.css_files = compiler.css_files;
+
+		// TODO populate this properly. We don't have named chunks, as in
+		// webpack, but we can have a route -> [chunk] map or something
+		this.assets = {};
+
+		compiler.chunks.forEach(chunk => {
+			if (compiler.input in chunk.modules) {
+				this.assets.main = chunk.fileName;
+			}
+		});
+
+		this.summary = compiler.chunks.map(chunk => {
+			const size_color = chunk.code.length > 150000 ? colors.bold.red : chunk.code.length > 50000 ? colors.bold.yellow : colors.bold.white;
+			const size_label = left_pad(pb(chunk.code.length), 10);
+
+			const lines = [size_color(`${size_label} ${chunk.fileName}`)];
+
+			const deps = Object.keys(chunk.modules)
+				.map(file => {
+					return {
+						file: path.relative(process.cwd(), file),
+						size: chunk.modules[file].renderedLength
+					};
+				})
+				.filter(dep => dep.size > 0)
+				.sort((a, b) => b.size - a.size);
+
+			const total_unminified = deps.reduce((t, d) => t + d.size, 0);
+
+			deps.forEach((dep, i) => {
+				const c = i === deps.length - 1 ? '└' : '│';
+				let line = `           ${c} ${dep.file}`;
+
+				if (deps.length > 1) {
+					const p = (100 * dep.size / total_unminified).toFixed(1);
+					line += ` (${p}%)`;
+				}
+
+				lines.push(colors.gray(line));
+			});
+
+			return lines.join('\n');
+		}).join('\n');
+	}
+
+	to_json(manifest_data: ManifestData, dirs: Dirs): BuildInfo {
+		// TODO extract_css has side-effects that don't belong
+		// in a method called to_json
+		return {
+			bundler: 'rollup',
+			shimport: require('shimport/package.json').version,
+			assets: this.assets,
+			css: extract_css(this, manifest_data.components, dirs)
+		};
+	}
+
+	print() {
+		const blocks: string[] = this.warnings.map(warning => {
+			return warning.file
+				? `> ${colors.bold(warning.file)}\n${warning.message}`
+				: `> ${warning.message}`;
+		});
+
+		blocks.push(this.summary);
+
+		return blocks.join('\n\n');
+	}
+}
+
+function munge_warning_or_error(warning_or_error: any) {
+	return {
+		file: warning_or_error.filename,
+		message: [warning_or_error.message, warning_or_error.frame].filter(Boolean).join('\n')
+	};
+}
+

src/core/create_compilers/WebpackCompiler.ts

@@ -0,0 +1,48 @@
+import * as path from 'path';
+import relative from 'require-relative';
+import { CompileResult } from './interfaces';
+import WebpackResult from './WebpackResult';
+
+let webpack: any;
+
+export class WebpackCompiler {
+	_: any;
+
+	constructor(config: string) {
+		if (!webpack) webpack = relative('webpack', process.cwd());
+		this._ = webpack(require(path.resolve(config)));
+	}
+
+	oninvalid(cb: (filename: string) => void) {
+		this._.hooks.invalid.tap('sapper', cb);
+	}
+
+	compile(): Promise<CompileResult> {
+		return new Promise((fulfil, reject) => {
+			this._.run((err: Error, stats: any) => {
+				if (err) {
+					reject(err);
+					process.exit(1);
+				}
+
+				const result = new WebpackResult(stats);
+
+				if (result.errors.length) {
+					// TODO print errors
+					// console.error(stats.toString({ colors: true }));
+					reject(new Error(`Encountered errors while building app`));
+				}
+
+				else {
+					fulfil(result);
+				}
+			});
+		});
+	}
+
+	watch(cb: (err?: Error, stats?: any) => void) {
+		this._.watch({}, (err?: Error, stats?: any) => {
+			cb(err, stats && new WebpackResult(stats));
+		});
+	}
+}

src/core/create_compilers/WebpackResult.ts

@@ -0,0 +1,73 @@
+import format_messages from 'webpack-format-messages';
+import { CompileResult, BuildInfo, CompileError, Chunk, CssFile } from './interfaces';
+import { ManifestData, Dirs } from '../../interfaces';
+
+const locPattern = /\((\d+):(\d+)\)$/;
+
+function munge_warning_or_error(message: string) {
+	// TODO this is all a bit rube goldberg...
+	const lines = message.split('\n');
+
+	const file = lines.shift()
+		.replace('', '') // careful — there is a special character at the beginning of this string
+		.replace('', '')
+		.replace('./', '');
+
+	let line = null;
+	let column = null;
+
+	const match = locPattern.exec(lines[0]);
+	if (match) {
+		lines[0] = lines[0].replace(locPattern, '');
+		line = +match[1];
+		column = +match[2];
+	}
+
+	return {
+		file,
+		message: lines.join('\n')
+	};
+}
+
+export default class WebpackResult implements CompileResult {
+	duration: number;
+	errors: CompileError[];
+	warnings: CompileError[];
+	chunks: Chunk[];
+	assets: Record<string, string>;
+	css_files: CssFile[];
+	stats: any;
+
+	constructor(stats: any) {
+		this.stats = stats;
+
+		const info = stats.toJson();
+
+		const messages = format_messages(stats);
+
+		this.errors = messages.errors.map(munge_warning_or_error);
+		this.warnings = messages.warnings.map(munge_warning_or_error);
+
+		this.duration = info.time;
+
+		this.chunks = info.assets.map((chunk: { name: string }) => ({ file: chunk.name }));
+		this.assets = info.assetsByChunkName;
+	}
+
+	to_json(manifest_data: ManifestData, dirs: Dirs): BuildInfo {
+		return {
+			bundler: 'webpack',
+			shimport: null, // webpack has its own loader
+			assets: this.assets,
+			css: {
+				// TODO
+				main: null,
+				chunks: {}
+			}
+		};
+	}
+
+	print() {
+		return this.stats.toString({ colors: true });
+	}
+}

src/core/create_compilers/extract_css.ts

@@ -0,0 +1,230 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import hash from 'string-hash';
+import * as codec from 'sourcemap-codec';
+import { PageComponent, Dirs } from '../../interfaces';
+import { CompileResult } from './interfaces';
+
+const inline_sourcemap_header = 'data:application/json;charset=utf-8;base64,';
+
+function extract_sourcemap(raw: string, id: string) {
+	let raw_map: string;
+	let map = null;
+
+	const code = raw.replace(/\/\*#\s+sourceMappingURL=(.+)\s+\*\//g, (m, url) => {
+		if (raw_map) {
+			// TODO should not happen!
+			throw new Error(`Found multiple sourcemaps in single CSS file (${id})`);
+		}
+
+		raw_map = url;
+		return '';
+	}).trim();
+
+	if (raw_map) {
+		if (raw_map.startsWith(inline_sourcemap_header)) {
+			const json = Buffer.from(raw_map.slice(inline_sourcemap_header.length), 'base64').toString();
+			map = JSON.parse(json);
+		} else {
+			// TODO do we want to handle non-inline sourcemaps? could be a rabbit hole
+		}
+	}
+
+	return {
+		code,
+		map
+	};
+}
+
+type SourceMap = {
+	version: 3;
+	file: string;
+	sources: string[];
+	sourcesContent: string[];
+	names: string[];
+	mappings: string;
+};
+
+export default function extract_css(client_result: CompileResult, components: PageComponent[], dirs: Dirs) {
+	const result: {
+		main: string | null;
+		chunks: Record<string, string[]>
+	} = {
+		main: null,
+		chunks: {}
+	};
+
+	if (!client_result.css_files) return; // Rollup-only for now
+
+	const unaccounted_for = new Set();
+
+	const css_map = new Map();
+	client_result.css_files.forEach(css => {
+		unaccounted_for.add(css.id);
+		css_map.set(css.id, css.code);
+	});
+
+	const chunk_map = new Map();
+	client_result.chunks.forEach(chunk => {
+		chunk_map.set(chunk.file, chunk);
+	});
+
+	const chunks_with_css = new Set();
+
+	// figure out which chunks belong to which components...
+	const component_owners = new Map();
+	client_result.chunks.forEach(chunk => {
+		chunk.modules.forEach(module => {
+			const component = path.relative(dirs.routes, module);
+			component_owners.set(component, chunk);
+		});
+	});
+
+	const chunks_depended_upon_by_component = new Map();
+
+	// ...so we can figure out which chunks don't belong
+	components.forEach(component => {
+		const chunk = component_owners.get(component.file);
+		if (!chunk) {
+			// this should never happen!
+			throw new Error(`Could not find chunk that owns ${component.file}`);
+		}
+
+		const chunks = new Set([chunk]);
+		chunks.forEach(chunk => {
+			chunk.imports.forEach((file: string) => {
+				const chunk = chunk_map.get(file);
+				if (chunk) chunks.add(chunk);
+			});
+		});
+
+		chunks.forEach(chunk => {
+			chunk.modules.forEach((module: string) => {
+				unaccounted_for.delete(module);
+			});
+		});
+
+		chunks_depended_upon_by_component.set(
+			component,
+			chunks
+		);
+	});
+
+	function get_css_from_modules(modules: string[]) {
+		const parts: string[] = [];
+		const mappings: number[][][] = [];
+
+		const combined_map: SourceMap = {
+			version: 3,
+			file: null,
+			sources: [],
+			sourcesContent: [],
+			names: [],
+			mappings: null
+		};
+
+		modules.forEach(module => {
+			if (!/\.css$/.test(module)) return;
+
+			const css = css_map.get(module);
+
+			const { code, map } = extract_sourcemap(css, module);
+
+			parts.push(code);
+
+			if (map) {
+				const lines = codec.decode(map.mappings);
+
+				if (combined_map.sources.length > 0 || combined_map.names.length > 0) {
+					lines.forEach(line => {
+						line.forEach(segment => {
+							// adjust source index
+							segment[1] += combined_map.sources.length;
+
+							// adjust name index
+							if (segment[4]) segment[4] += combined_map.names.length;
+						});
+					});
+				}
+
+				combined_map.sources.push(...map.sources);
+				combined_map.sourcesContent.push(...map.sourcesContent);
+				combined_map.names.push(...map.names);
+
+				mappings.push(...lines);
+			}
+		});
+
+		if (parts.length > 0) {
+			combined_map.mappings = codec.encode(mappings);
+
+			combined_map.sources = combined_map.sources.map(source => path.relative(`${dirs.dest}/client`, source));
+
+			return {
+				code: parts.join('\n'),
+				map: combined_map
+			};
+		}
+
+		return null;
+	}
+
+	const main = client_result.assets.main;
+	const entry = fs.readFileSync(`${dirs.dest}/client/${main}`, 'utf-8');
+
+	const replacements = new Map();
+
+	chunks_depended_upon_by_component.forEach((chunks, component) => {
+		const chunks_with_css = Array.from(chunks).filter(chunk => {
+			const css = get_css_from_modules(chunk.modules);
+
+			if (css) {
+				const { code, map } = css;
+
+				const output_file_name = chunk.file.replace(/\.js$/, '.css');
+
+				map.file = output_file_name;
+				map.sources = map.sources.map(source => path.relative(`${dirs.dest}/client`, source));
+
+				fs.writeFileSync(`${dirs.dest}/client/${output_file_name}`, `${code}\n/* sourceMappingURL=client/${output_file_name}.map */`);
+				fs.writeFileSync(`${dirs.dest}/client/${output_file_name}.map`, JSON.stringify(map, null, '  '));
+
+				return true;
+			}
+		});
+
+		const files = chunks_with_css.map(chunk => chunk.file.replace(/\.js$/, '.css'));
+
+		replacements.set(
+			component.file,
+			files
+		);
+
+		result.chunks[component.file] = files;
+	});
+
+	const replaced = entry.replace(/["']__SAPPER_CSS_PLACEHOLDER:(.+?)__["']/g, (m, route) => {
+		return JSON.stringify(replacements.get(route));
+	});
+
+	fs.writeFileSync(`${dirs.dest}/client/${main}`, replaced);
+
+	const leftover = get_css_from_modules(Array.from(unaccounted_for));
+	if (leftover) {
+		const { code, map } = leftover;
+
+		const main_hash = hash(code);
+
+		const output_file_name = `main.${main_hash}.css`;
+
+		map.file = output_file_name;
+		map.sources = map.sources.map(source => path.relative(`${dirs.dest}/client`, source));
+
+		fs.writeFileSync(`${dirs.dest}/client/${output_file_name}`, `${code}\n/* sourceMappingURL=client/${output_file_name}.map */`);
+		fs.writeFileSync(`${dirs.dest}/client/${output_file_name}.map`, JSON.stringify(map, null, '  '));
+
+		result.main = output_file_name;
+	}
+
+	return result;
+}

src/core/create_compilers/index.ts

@@ -0,0 +1,37 @@
+import * as fs from 'fs';
+import { Dirs } from '../../interfaces';
+import RollupCompiler from './RollupCompiler';
+import { WebpackCompiler } from './WebpackCompiler';
+
+export type Compiler = RollupCompiler | WebpackCompiler;
+
+export type Compilers = {
+	client: Compiler;
+	server: Compiler;
+	serviceworker?: Compiler;
+}
+
+export default function create_compilers(bundler: string, dirs: Dirs): Compilers {
+	if (bundler === 'rollup') {
+		const sw = `${dirs.rollup}/service-worker.config.js`;
+
+		return {
+			client: new RollupCompiler(`${dirs.rollup}/client.config.js`),
+			server: new RollupCompiler(`${dirs.rollup}/server.config.js`),
+			serviceworker: fs.existsSync(sw) && new RollupCompiler(sw)
+		};
+	}
+
+	if (bundler === 'webpack') {
+		const sw = `${dirs.webpack}/service-worker.config.js`;
+
+		return {
+			client: new WebpackCompiler(`${dirs.webpack}/client.config.js`),
+			server: new WebpackCompiler(`${dirs.webpack}/server.config.js`),
+			serviceworker: fs.existsSync(sw) && new WebpackCompiler(sw)
+		};
+	}
+
+	// this shouldn't be possible...
+	throw new Error(`Invalid bundler option '${bundler}'`);
+}

src/core/create_compilers/interfaces.ts

@@ -0,0 +1,39 @@
+import { ManifestData, Dirs } from '../../interfaces';
+
+export type Chunk = {
+	file: string;
+	imports: string[];
+	modules: string[];
+}
+
+export type CssFile = {
+	id: string;
+	code: string;
+};
+
+export class CompileError {
+	file: string;
+	message: string;
+}
+
+export interface CompileResult {
+	duration: number;
+	errors: CompileError[];
+	warnings: CompileError[];
+	chunks: Chunk[];
+	assets: Record<string, string>;
+	css_files: CssFile[];
+
+	to_json: (manifest_data: ManifestData, dirs: Dirs) => BuildInfo
+}
+
+export type BuildInfo = {
+	bundler: string;
+	shimport: string;
+	assets: Record<string, string>;
+	legacy_assets?: Record<string, string>;
+	css: {
+		main: string | null,
+		chunks: Record<string, string[]>
+	}
+}