Merge pull request #88 from sveltejs/gh-9

Exporting

cli/index.js

@@ -1,8 +1,29 @@
 #!/usr/bin/env node
 
+const build = require('../lib/build.js');
+
 const cmd = process.argv[2];
+const start = Date.now();
 
 if (cmd === 'build') {
-	process.env.NODE_ENV = 'production';
+	build()
-	require('../lib/build.js')();
+		.then(() => {
-}
+			const elapsed = Date.now() - start;
+			console.error(`built in ${elapsed}ms`); // TODO beautify this, e.g. 'built in 4.7 seconds'
+		})
+		.catch(err => {
+			console.error(err ? err.details || err.stack || err.message || err : 'Unknown error');
+		});
+} else if (cmd === 'export') {
+	const start = Date.now();
+
+	build()
+		.then(() => require('../lib/utils/export.js')())
+		.then(() => {
+			const elapsed = Date.now() - start;
+			console.error(`extracted in ${elapsed}ms`); // TODO beautify this, e.g. 'built in 4.7 seconds'
+		})
+		.catch(err => {
+			console.error(err ? err.details || err.stack || err.message || err : 'Unknown error');
+		});
+}

lib/build.js

@@ -1,3 +1,5 @@
+process.env.NODE_ENV = 'production';
+
 const fs = require('fs');
 const path = require('path');
 const mkdirp = require('mkdirp');
@@ -14,29 +16,32 @@ module.exports = () => {
 	// create main.js and server-routes.js
 	create_app();
 
-	function handleErrors(err, stats) {
+	return new Promise((fulfil, reject) => {
-		if (err) {
+		function handleErrors(err, stats) {
-			console.error(err ? err.details || err.stack || err.message || err : 'Unknown error');
+			if (err) {
-			process.exit(1);
+				reject(err);
+				process.exit(1);
+			}
+
+			if (stats.hasErrors()) {
+				console.error(stats.toString({ colors: true }));
+				reject(new Error(`Encountered errors while building app`));
+			}
 		}
 
-		if (stats.hasErrors()) {
+		client.run((err, clientStats) => {
-			console.log(stats.toString({ colors: true }));
+			handleErrors(err, clientStats);
-			process.exit(1);
+			const clientInfo = clientStats.toJson();
-		}
+			fs.writeFileSync(path.join(dest, 'stats.client.json'), JSON.stringify(clientInfo, null, '  '));
-	}
 
-	client.run((err, clientStats) => {
+			server.run((err, serverStats) => {
-		handleErrors(err, clientStats);
+				handleErrors(err, serverStats);
-		const clientInfo = clientStats.toJson();
+				const serverInfo = serverStats.toJson();
-		fs.writeFileSync(path.join(dest, 'stats.client.json'), JSON.stringify(clientInfo, null, '  '));
+				fs.writeFileSync(path.join(dest, 'stats.server.json'), JSON.stringify(serverInfo, null, '  '));
 
-		server.run((err, serverStats) => {
+				generate_asset_cache(clientInfo, serverInfo);
-			handleErrors(err, serverStats);
+				fulfil();
-			const serverInfo = serverStats.toJson();
+			});
-			fs.writeFileSync(path.join(dest, 'stats.server.json'), JSON.stringify(serverInfo, null, '  '));
-
-			generate_asset_cache(clientInfo, serverInfo);
 		});
 	});
-};
+};

lib/utils/create_routes.js

@@ -31,7 +31,7 @@ module.exports = function create_matchers(files) {
 				}
 			}
 
-			const pattern = new RegExp(`^${pattern_string || '\\/'}$`);
+			const pattern = new RegExp(`^${pattern_string}\\/?$`);
 
 			const test = url => pattern.test(url);
 

lib/utils/export.js

@@ -0,0 +1,87 @@
+const sander = require('sander');
+const app = require('express')();
+const cheerio = require('cheerio');
+const fetch = require('node-fetch');
+const URL = require('url-parse');
+const sapper = require('../index.js');
+
+const { PORT = 3000, OUTPUT_DIR = 'dist' } = process.env;
+const { dest } = require('../config.js');
+
+const origin = `http://localhost:${PORT}`;
+
+module.exports = function() {
+	// Prep output directory
+	sander.rimrafSync(OUTPUT_DIR);
+
+	sander.copydirSync('assets').to(OUTPUT_DIR);
+	sander.copydirSync(`${dest}/client`).to(`${OUTPUT_DIR}/client`);
+	sander.copyFileSync(`${dest}/service-worker.js`).to(`${OUTPUT_DIR}/service-worker.js`);
+
+	// Intercept server route fetches
+	function save(res) {
+		res = res.clone();
+
+		return res.text().then(body => {
+			const { pathname } = new URL(res.url);
+			let dest = OUTPUT_DIR + pathname;
+
+			const type = res.headers.get('Content-Type');
+			if (type.startsWith('text/html;')) dest += '/index.html';
+
+			sander.writeFileSync(dest, body);
+
+			return body;
+		});
+	}
+
+	global.fetch = (url, opts) => {
+		if (url[0] === '/') {
+			url = `http://localhost:${PORT}${url}`;
+
+			return fetch(url, opts)
+				.then(r => {
+					save(r);
+					return r;
+				});
+		}
+
+		return fetch(url, opts);
+	};
+
+	app.use(sapper());
+	const server = app.listen(PORT);
+
+	const seen = new Set();
+
+	function handle(url) {
+		if (url.origin !== origin) return;
+
+		if (seen.has(url.pathname)) return;
+		seen.add(url.pathname);
+
+		return fetch(url.href)
+			.then(r => {
+				save(r);
+				return r.text();
+			})
+			.then(body => {
+				const $ = cheerio.load(body);
+				const hrefs = [];
+
+				$('a[href]').each((i, $a) => {
+					hrefs.push($a.attribs.href);
+				});
+
+				return hrefs.reduce((promise, href) => {
+					return promise.then(() => handle(new URL(href, url.href)));
+				}, Promise.resolve());
+			})
+			.catch(err => {
+				console.error(`Error rendering ${url.pathname}: ${err.message}`);
+			});
+	}
+
+	return handle(new URL(origin)) // TODO all static routes
+		.then(() => server.close());
+};

lib/utils/generate_asset_cache.js

@@ -3,7 +3,7 @@ const path = require('path');
 const glob = require('glob');
 const templates = require('../templates.js');
 const route_manager = require('../route_manager.js');
-const { dest, dev } = require('../config.js');
+const { dest } = require('../config.js');
 
 function ensure_array(thing) {
 	return Array.isArray(thing) ? thing : [thing]; // omg webpack what the HELL are you doing
@@ -17,10 +17,8 @@ module.exports = function generate_asset_cache(clientInfo, serverInfo) {
 	const service_worker = generate_service_worker(chunk_files);
 	const index = generate_index(main_file);
 
-	if (dev) {
+	fs.writeFileSync(path.join(dest, 'service-worker.js'), service_worker);
-		fs.writeFileSync(path.join(dest, 'service-worker.js'), service_worker);
+	fs.writeFileSync(path.join(dest, 'index.html'), index);
-		fs.writeFileSync(path.join(dest, 'index.html'), index);
-	}
 
 	return {
 		client: {

package-lock.json

@@ -7,8 +7,7 @@
     "@types/node": {
       "version": "7.0.52",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-7.0.52.tgz",
-      "integrity": "sha512-jjpyQsKGsOF/wUElNjfPULk+d8PKvJOIXk3IUeBYYmNCy5dMWfrI+JiixYNw8ppKOlcRwWTXFl0B+i5oGrf95Q==",
+      "integrity": "sha512-jjpyQsKGsOF/wUElNjfPULk+d8PKvJOIXk3IUeBYYmNCy5dMWfrI+JiixYNw8ppKOlcRwWTXFl0B+i5oGrf95Q=="
-      "dev": true
     },
     "accepts": {
       "version": "1.3.4",
@@ -377,6 +376,11 @@
         }
       }
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
     "boom": {
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz",
@@ -608,6 +612,19 @@
       "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=",
       "dev": true
     },
+    "cheerio": {
+      "version": "1.0.0-rc.2",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz",
+      "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=",
+      "requires": {
+        "css-select": "1.2.0",
+        "dom-serializer": "0.1.0",
+        "entities": "1.1.1",
+        "htmlparser2": "3.9.2",
+        "lodash": "4.17.4",
+        "parse5": "3.0.3"
+      }
+    },
     "chokidar": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-1.7.0.tgz",
@@ -983,6 +1000,17 @@
         "source-list-map": "2.0.0"
       }
     },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "1.0.0",
+        "css-what": "2.1.0",
+        "domutils": "1.5.1",
+        "nth-check": "1.0.1"
+      }
+    },
     "css-selector-tokenizer": {
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/css-selector-tokenizer/-/css-selector-tokenizer-0.7.0.tgz",
@@ -994,6 +1022,11 @@
         "regexpu-core": "1.0.0"
       }
     },
+    "css-what": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz",
+      "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0="
+    },
     "cssesc": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-0.1.0.tgz",
@@ -1224,11 +1257,49 @@
         "esutils": "2.0.2"
       }
     },
+    "dom-serializer": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz",
+      "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=",
+      "requires": {
+        "domelementtype": "1.1.3",
+        "entities": "1.1.1"
+      },
+      "dependencies": {
+        "domelementtype": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
+          "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs="
+        }
+      }
+    },
     "domain-browser": {
       "version": "1.1.7",
       "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.1.7.tgz",
       "integrity": "sha1-hnqksJP6oF8d4IwG9NeyH9+GmLw="
     },
+    "domelementtype": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz",
+      "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI="
+    },
+    "domhandler": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.1.tgz",
+      "integrity": "sha1-iS5HAAqZvlW783dP/qBWHYh5wlk=",
+      "requires": {
+        "domelementtype": "1.3.0"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "requires": {
+        "dom-serializer": "0.1.0",
+        "domelementtype": "1.3.0"
+      }
+    },
     "duplexer": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
@@ -1382,6 +1453,16 @@
         }
       }
     },
+    "ensure-posix-path": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/ensure-posix-path/-/ensure-posix-path-1.0.2.tgz",
+      "integrity": "sha1-pls+QtC3HPxYXrd0+ZQ8jZuRsMI="
+    },
+    "entities": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
+      "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA="
+    },
     "errno": {
       "version": "0.1.6",
       "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.6.tgz",
@@ -3127,6 +3208,19 @@
       "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-1.2.1.tgz",
       "integrity": "sha1-DfKTUfByEWNRXfueVUPl9u7VFi8="
     },
+    "htmlparser2": {
+      "version": "3.9.2",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz",
+      "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=",
+      "requires": {
+        "domelementtype": "1.3.0",
+        "domhandler": "2.4.1",
+        "domutils": "1.5.1",
+        "entities": "1.1.1",
+        "inherits": "2.0.3",
+        "readable-stream": "2.3.3"
+      }
+    },
     "http-errors": {
       "version": "1.6.2",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz",
@@ -3835,6 +3929,14 @@
       "integrity": "sha1-5WqpTEyAVaFkBKBnS3jyFffI4ZQ=",
       "dev": true
     },
+    "matcher-collection": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/matcher-collection/-/matcher-collection-1.0.5.tgz",
+      "integrity": "sha512-nUCmzKipcJEwYsBVAFh5P+d7JBuhJaW1xs85Hara9xuMLqtCVUrW6DSC0JVIkluxEH2W45nPBM/wjHtBXa/tYA==",
+      "requires": {
+        "minimatch": "3.0.4"
+      }
+    },
     "math-expression-evaluator": {
       "version": "1.2.17",
       "resolved": "https://registry.npmjs.org/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz",
@@ -4356,6 +4458,14 @@
         "path-key": "2.0.1"
       }
     },
+    "nth-check": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz",
+      "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=",
+      "requires": {
+        "boolbase": "1.0.0"
+      }
+    },
     "nugget": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/nugget/-/nugget-2.0.1.tgz",
@@ -4556,6 +4666,14 @@
         "error-ex": "1.3.1"
       }
     },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+      "requires": {
+        "@types/node": "7.0.52"
+      }
+    },
     "parseurl": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
@@ -5750,6 +5868,16 @@
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
       "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
     },
+    "sander": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/sander/-/sander-0.6.0.tgz",
+      "integrity": "sha1-rxYkzX+2362Y6+9WUxn5IAeNqSU=",
+      "requires": {
+        "graceful-fs": "4.1.11",
+        "mkdirp": "0.5.1",
+        "rimraf": "2.6.2"
+      }
+    },
     "sax": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
@@ -6605,6 +6733,15 @@
         }
       }
     },
+    "walk-sync": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/walk-sync/-/walk-sync-0.3.2.tgz",
+      "integrity": "sha512-FMB5VqpLqOCcqrzA9okZFc0wq0Qbmdm396qJxvQZhDpyu0W95G9JCmp74tx7iyYnyOcBtUuKJsgIKAqjozvmmQ==",
+      "requires": {
+        "ensure-posix-path": "1.0.2",
+        "matcher-collection": "1.0.5"
+      }
+    },
     "watchpack": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.4.0.tgz",

package.json

@@ -18,6 +18,7 @@
   },
   "dependencies": {
     "chalk": "^2.3.0",
+    "cheerio": "^1.0.0-rc.2",
     "chokidar": "^1.7.0",
     "code-frame": "^5.0.0",
     "escape-html": "^1.0.3",
@@ -26,7 +27,10 @@
     "relative": "^3.0.2",
     "require-relative": "^0.8.7",
     "rimraf": "^2.6.2",
+    "sander": "^0.6.0",
     "serialize-javascript": "^1.4.0",
+    "url-parse": "^1.2.0",
+    "walk-sync": "^0.3.2",
     "webpack": "^3.10.0",
     "webpack-hot-middleware": "^2.21.0"
   },

test/app/.gitignore

@@ -3,4 +3,5 @@ node_modules
 .sapper
 yarn.lock
 cypress/screenshots
-templates/.*
+templates/.*
+dist

test/app/routes/blog/index.html

@@ -32,7 +32,7 @@
 		},
 
 		preload({ params, query }) {
-			return fetch(`/api/blog`).then(r => r.json()).then(posts => {
+			return fetch(`/api/blog/contents`).then(r => r.json()).then(posts => {
 				return { posts };
 			});
 		}

test/common/test.js

@@ -5,6 +5,7 @@ const serve = require('serve-static');
 const Nightmare = require('nightmare');
 const getPort = require('get-port');
 const fetch = require('node-fetch');
+const walkSync = require('walk-sync');
 
 run('production');
 run('development');
@@ -78,7 +79,7 @@ function run(env) {
 
 			if (env === 'production') {
 				const cli = path.resolve(__dirname, '../../cli/index.js');
-				exec_promise = exec(`${cli} build`);
+				exec_promise = exec(`${cli} build`).then(() => exec(`${cli} export`));
 			}
 
 			return exec_promise.then(() => {
@@ -324,20 +325,92 @@ function run(env) {
 				});
 			});
 		});
+
+		if (env === 'production') {
+			describe('export', () => {
+				it('export all pages', () => {
+					const dest = path.resolve(__dirname, '../app/dist');
+
+					// Pages that should show up in the extraction directory.
+					const expectedPages = [
+						'index.html',
+						'about/index.html',
+						'slow-preload/index.html',
+
+						'blog/index.html',
+						'blog/a-very-long-post/index.html',
+						'blog/how-can-i-get-involved/index.html',
+						'blog/how-is-sapper-different-from-next/index.html',
+						'blog/how-to-use-sapper/index.html',
+						'blog/what-is-sapper/index.html',
+						'blog/why-the-name/index.html',
+
+						'api/blog/contents',
+						'api/blog/a-very-long-post',
+						'api/blog/how-can-i-get-involved',
+						'api/blog/how-is-sapper-different-from-next',
+						'api/blog/how-to-use-sapper',
+						'api/blog/what-is-sapper',
+						'api/blog/why-the-name',
+
+						'favicon.png',
+						'global.css',
+						'great-success.png',
+						'manifest.json',
+						'service-worker.js',
+						'svelte-logo-192.png',
+						'svelte-logo-512.png',
+					];
+					// Client scripts that should show up in the extraction directory.
+					const expectedClientRegexes = [
+						/client\/_\..*?\.js/,
+						/client\/about\..*?\.js/,
+						/client\/blog_\$slug\$\..*?\.js/,
+						/client\/blog\..*?\.js/,
+						/client\/main\..*?\.js/,
+						/client\/show_url\..*?\.js/,
+						/client\/slow_preload\..*?\.js/,
+					];
+					const allPages = walkSync(dest);
+
+					expectedPages.forEach((expectedPage) => {
+						assert.ok(allPages.includes(expectedPage),
+						    `Could not find page matching ${expectedPage}`);
+					});
+					expectedClientRegexes.forEach((expectedRegex) => {
+						// Ensure each client page regular expression matches at least one
+						// generated page.
+						let matched = false;
+						for (const page of allPages) {
+							if (expectedRegex.test(page)) {
+								matched = true;
+								break;
+							}
+						}
+						assert.ok(matched,
+							  `Could not find client page matching ${expectedRegex}`);
+					});
+				});
+			});
+		}
 	});
 }
 
 function exec(cmd) {
 	return new Promise((fulfil, reject) => {
-		require('child_process').exec(cmd, (err, stdout, stderr) => {
+		const parts = cmd.split(' ');
-			if (err) {
+		const proc = require('child_process').spawn(parts.shift(), parts);
-				process.stdout.write(stdout);
-				process.stderr.write(stderr);
 
-				return reject(err);
+		proc.stdout.on('data', data => {
-			}
+			process.stdout.write(data);
-
-			fulfil();
 		});
+
+		proc.stderr.on('data', data => {
+			process.stderr.write(data);
+		});
+
+		proc.on('error', reject);
+
+		proc.on('close', () => fulfil());
 	});
 }