diff --git a/src/api/export.ts b/src/api/export.ts index 0fc877b..4f7e3b2 100644 --- a/src/api/export.ts +++ b/src/api/export.ts @@ -54,7 +54,9 @@ async function execute(emitter: EventEmitter, opts: Opts) { const port = await ports.find(3000); const origin = `http://localhost:${port}`; - const root = new URL(opts.basepath || '', origin); + let basepath = opts.basepath || '/'; + if (!basepath.endsWith('/')) basepath += '/'; + let root = new URL(basepath, origin); emitter.emit('info', { message: `Crawling ${root.href}` @@ -72,29 +74,15 @@ async function execute(emitter: EventEmitter, opts: Opts) { const seen = new Set(); const saved = new Set(); - const deferreds = new Map(); - function get_deferred(pathname: string) { - pathname = pathname.replace(root.pathname, ''); - - if (!deferreds.has(pathname)) { - deferreds.set(pathname, new Deferred()); - } - - return deferreds.get(pathname); - } - - proc.on('message', message => { - if (!message.__sapper__ || message.event !== 'file') return; - - const pathname = new URL(message.url, origin).pathname; + function save(url: string, status: number, type: string, body: string) { + const pathname = new URL(url, origin).pathname; let file = pathname.slice(1); - let { body } = message; if (saved.has(file)) return; saved.add(file); - const is_html = message.type === 'text/html'; + const is_html = type === 'text/html'; if (is_html) { file = file === '' ? 'index.html' : `${file}/index.html`; @@ -104,12 +92,15 @@ async function execute(emitter: EventEmitter, opts: Opts) { emitter.emit('file', { file, size: body.length, - status: message.status + status }); sander.writeFileSync(export_dir, file, body); + } - get_deferred(pathname).fulfil(); + proc.on('message', message => { + if (!message.__sapper__ || message.event !== 'file') return; + save(message.url, message.status, message.type, message.body); }); async function handle(url: URL) { @@ -118,25 +109,27 @@ async function execute(emitter: EventEmitter, opts: Opts) { if (seen.has(pathname)) return; seen.add(pathname); - const deferred = get_deferred(pathname); - const timeout_deferred = new Deferred(); const timeout = setTimeout(() => { timeout_deferred.reject(new Error(`Timed out waiting for ${url.href}`)); }, opts.timeout); const r = await Promise.race([ - fetch(url.href), + fetch(url.href, { + redirect: 'manual' + }), timeout_deferred.promise ]); clearTimeout(timeout); // prevent it hanging at the end + let type = r.headers.get('Content-Type'); + let body = await r.text(); + const range = ~~(r.status / 100); if (range === 2) { - if (r.headers.get('Content-Type') === 'text/html') { - const body = await r.text(); + if (type === 'text/html') { const urls: URL[] = []; const cleaned = clean_html(body); @@ -162,7 +155,16 @@ async function execute(emitter: EventEmitter, opts: Opts) { } } - await deferred.promise; + if (range === 3) { + const location = r.headers.get('Location'); + + type = 'text/html'; + body = ``; + + await handle(new URL(location, root)); + } + + save(pathname, r.status, type, body); } return ports.wait(port) diff --git a/src/middleware.ts b/src/middleware.ts index 9f66494..7a95873 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -414,18 +414,6 @@ function get_page_handler( res.setHeader('Location', location); res.end(); - if (process.send) { - process.send({ - __sapper__: true, - event: 'file', - url: req.url, - method: req.method, - status: redirect.statusCode, - type: 'text/html', - body: `` - }); - } - return; } @@ -512,18 +500,6 @@ function get_page_handler( res.statusCode = status; res.end(body); - - if (process.send) { - process.send({ - __sapper__: true, - event: 'file', - url: req.url, - method: req.method, - status, - type: 'text/html', - body - }); - } }).catch(err => { if (error) { // we encountered an error while rendering the error page — oops diff --git a/test/app/app/server.js b/test/app/app/server.js index 430853c..0595bdf 100644 --- a/test/app/app/server.js +++ b/test/app/app/server.js @@ -108,6 +108,13 @@ const middlewares = [ }), ]; +app.get(`${BASEPATH}/non-sapper-redirect-from`, (req, res) => { + res.writeHead(301, { + Location: `${BASEPATH}/non-sapper-redirect-to` + }); + res.end(); +}); + if (BASEPATH) { app.use(BASEPATH, ...middlewares); } else { diff --git a/test/app/routes/index.html b/test/app/routes/index.html index b695afe..e5eecde 100644 --- a/test/app/routes/index.html +++ b/test/app/routes/index.html @@ -7,6 +7,7 @@ home about slow preload +redirect redirect redirect (root) broken link diff --git a/test/app/routes/non-sapper-redirect-to.html b/test/app/routes/non-sapper-redirect-to.html new file mode 100644 index 0000000..eeb0dfc --- /dev/null +++ b/test/app/routes/non-sapper-redirect-to.html @@ -0,0 +1 @@ +

redirected

\ No newline at end of file diff --git a/test/common/test.js b/test/common/test.js index 2db23ff..2ad9730 100644 --- a/test/common/test.js +++ b/test/common/test.js @@ -2,9 +2,7 @@ const fs = require('fs'); const path = require('path'); const assert = require('assert'); const Nightmare = require('nightmare'); -const serve = require('serve-static'); const walkSync = require('walk-sync'); -const fetch = require('node-fetch'); const rimraf = require('rimraf'); const ports = require('port-authority'); @@ -83,6 +81,11 @@ function testExport({ basepath = '' }) { 'about/index.html', 'slow-preload/index.html', + 'redirect-from/index.html', + 'redirect-to/index.html', + 'non-sapper-redirect-from/index.html', + 'non-sapper-redirect-to/index.html', + 'blog/index.html', 'blog/a-very-long-post/index.html', 'blog/how-can-i-get-involved/index.html',