Skip to content

Commit

Permalink
Content negotiation
Browse files Browse the repository at this point in the history 
 * Add the facilities for content negotiation, as specified by
   https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1

 * The provided options are sorted by quality.  When matching, we use
   semver caret semantics of the provided version to allow backward
   compatible changes, without the need for clients to constantly update
   their headers.

 * We use an unreleased version of the npm `negotiator` package from
   github.  We can switch to the official npm release once
   jshttp/negotiator#40 is merged.

Change-Id: Ie3a7042d78c310ef20eee79cb0a38c7cd40ec3cc
  • Loading branch information
@arlolra @cscott
arlolra authored and cscott committed May 13, 2016
1 parent 15e76d3 commit d562ea2
Show file tree
Hide file tree
Showing 6 changed files with 216 additions and 1 deletion.
37 changes: 37 additions & 0 deletions lib/api/apiUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,10 @@ apiUtils.dataParsoidContentType = function(env) {
return 'application/json; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/data-parsoid/' + env.conf.parsoid.DATA_PARSOID_VERSION + '"';
};

apiUtils.pagebundleContentType = function(env) {
return 'application/json; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/pagebundle/' + env.conf.parsoid.HTML_VERSION + '"';
};

/**
* Validates that data-parsoid was provided in the expected format.
*
Expand Down Expand Up @@ -312,3 +316,36 @@ apiUtils.fatalRequest = function(env, text, httpStatus) {
err.suppressLoggingStack = true;
env.log('fatal/request', err);
};

var profileRE = /^https:\/\/www.mediawiki.org\/wiki\/Specs\/(HTML|pagebundle)\/(\d+\.\d+\.\d+)$/;

/**
* Returns false if Parsoid is unable to supply an acceptable version.
*
* @method
* @param {Response} res
* @return {Boolean}
*/
apiUtils.validateContentVersion = function(res) {
var env = res.locals.env;
var opts = res.locals.opts;

// `acceptableTypes` is already sorted by quality.
return !res.locals.acceptableTypes.length ||
res.locals.acceptableTypes.some(function(t) {
if ((opts.format === 'html' && t.type === 'text/html') ||
(opts.format === 'pagebundle' && t.type === 'application/json')) {
var tp = t.parameters;
if (tp && tp.profile) {
var match = profileRE.exec(tp.profile);
return match && (opts.format === match[1].toLowerCase()) &&
(env.resolveContentVersion(match[2]) !== null);
} else {
return true;
}
} else if (t.type === '*/*' ||
(opts.format === 'html' && t.type === 'text/*')) {
return true;
}
});
};
14 changes: 14 additions & 0 deletions lib/api/routes.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ require('../../core-upgrade.js');
var childProcess = require('child_process');
var corepath = require('path');
var qs = require('querystring');
var Negotiator = require('negotiator');

var pkg = require('../../package.json');
var apiUtils = require('./apiUtils.js');
Expand Down Expand Up @@ -106,6 +107,11 @@ module.exports = function(parsoidConfig, processLogger) {
}
}

// Just do the parsing here.
var negotiator = new Negotiator(req);
res.locals.acceptableTypes =
negotiator.mediaTypes(undefined, { detailed: true });

res.locals.opts = opts;
next();
};
Expand Down Expand Up @@ -373,6 +379,14 @@ module.exports = function(parsoidConfig, processLogger) {
var stats = env.conf.parsoid.stats;
var startTimers = new Map();

// Validate the content version
if (!apiUtils.validateContentVersion(res)) {
var text = env.availableVersions.reduce(function(prev, curr) {
return prev + apiUtils[opts.format + 'ContentType'](env) + '\n';
}, 'Not acceptable.\n');
return apiUtils.fatalRequest(env, text, 406);
}

var p = Promise.method(function() {
// Check early if we have a wt string.
if (typeof wt === 'string') {
Expand Down
29 changes: 28 additions & 1 deletion lib/config/MWParserEnvironment.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use strict';
require('../../core-upgrade.js');

var semver = require('semver');
var Promise = require('../utils/promise.js');
var WikiConfig = require('./WikiConfig.js').WikiConfig;
var ConfigRequest = require('../mw/ApiRequest.js').ConfigRequest;
Expand Down Expand Up @@ -582,9 +583,35 @@ MWParserEnvironment.prototype.newAboutId = function() {
return "#" + this.newObjectId();
};

// Apply extra normalizations before serializing DOM.
/**
* Apply extra normalizations before serializing DOM.
*/
MWParserEnvironment.prototype.scrubWikitext = false;

/**
* The content versions Parsoid knows how to produce.
*/
MWParserEnvironment.prototype.availableVersions = ['1.2.1']; // env.conf.parsoid.HTML_VERSION

/**
* See if any content version Parsoid knows how to produce satisfies the
* the supplied version, when interpreted with semver caret semantics.
* This will allow us to make backwards compatible changes, without the need
* for clients to bump the version in their headers all the time.
*
* @method
* @param {String} v
* @return {String|null}
*/
MWParserEnvironment.prototype.resolveContentVersion = function(v) {
for (var i = 0; i < this.availableVersions.length; i++) {
var a = this.availableVersions[i];
if (semver.satisfies(a, '^' + v)) { return a; }
}
return null;
};


if (typeof module === "object") {
module.exports.MWParserEnvironment = MWParserEnvironment;
}
5 changes: 5 additions & 0 deletions npm-shrinkwrap.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"finalhandler": "^0.4.0",
"gelf-stream": "^0.2.4",
"html5": "^1.0.5",
"negotiator": "git+https://github.com/ethanresnick/negotiator#full-parse-access",
"node-txstatsd": "^0.1.5",
"node-uuid": "^1.4.7",
"pegjs": "git+https://github.com/tstarling/pegjs#fork",
Expand Down
131 changes: 131 additions & 0 deletions tests/mocha/api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,137 @@ describe('Parsoid API', function() {

}); // formats

describe('accepts', function() {

var acceptableHtmlResponse = function(profile) {
return function(res) {
res.statusCode.should.equal(200);
res.headers.should.have.property('content-type');
res.headers['content-type'].should.equal(
'text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/' + profile + '"'
);
res.text.should.not.equal('');
};
};

var acceptablePageBundleResponse = function(profile) {
return function(res) {
res.statusCode.should.equal(200);
res.body.should.have.property('html');
res.body.html.should.have.property('headers');
res.body.html.headers.should.have.property('content-type');
res.body.html.headers['content-type'].should.equal(
'text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/' + profile + '"'
);
res.body.html.should.have.property('body');
res.body.should.have.property('data-parsoid');
res.body['data-parsoid'].should.have.property('headers');
res.body['data-parsoid'].headers.should.have.property('content-type');
// FIXME: There's only one content version now, and they should all have it.
// res.body['data-parsoid'].headers['content-type'].should.equal(
// 'application/json; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/data-parsoid/' + profile + '"'
// );
res.body['data-parsoid'].should.have.property('body');
};
};

it('should not accept requests for older html versions', function(done) {
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/html/')
.set('Accept', 'text/html; profile="https://www.mediawiki.org/wiki/Specs/HTML/0.0.0"')
.send({ wikitext: '== h2 ==' })
.expect(406)
.end(done);
});

it('should not accept requests for older pagebundle versions', function(done) {
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/pagebundle/')
.set('Accept', 'application/json; profile="https://www.mediawiki.org/wiki/Specs/HTML/0.0.0"')
.send({ wikitext: '== h2 ==' })
.expect(406)
.end(done);
});

it('should not accept requests for other profiles', function(done) {
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/html/')
.set('Accept', 'text/html; profile="something different"')
.send({ wikitext: '== h2 ==' })
.expect(406)
.end(done);
});

it('should accept requests with no accept header', function(done) {
var profile = '1.2.1';
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/html/')
.send({ wikitext: '== h2 ==' })
.expect(200)
.expect(acceptableHtmlResponse(profile))
.end(done);
});

it('should accept wildcards', function(done) {
var profile = '1.2.1';
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/html/')
.set('Accept', '*/*')
.send({ wikitext: '== h2 ==' })
.expect(200)
.expect(acceptableHtmlResponse(profile))
.end(done);
});

it('should prefer higher quality', function(done) {
// FIXME: This test would be better if there were more available
// versions. Fix it in the followup.
var profile = '1.2.1';
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/html/')
.set('Accept', 'text/html; profile="https://www.mediawiki.org/wiki/Specs/HTML/0.0.0"; q=0.5,' +
'text/html; profile="https://www.mediawiki.org/wiki/Specs/HTML/' + profile + '"; q=0.8')
.send({ wikitext: '== h2 ==' })
.expect(200)
.expect(acceptableHtmlResponse(profile))
.end(done);
});

it('should accept requests for the latest html versions', function(done) {
var profile = '1.2.1';
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/html/')
.set('Accept', 'text/html; profile="https://www.mediawiki.org/wiki/Specs/HTML/' + profile + '"')
.send({ wikitext: '== h2 ==' })
.expect(200)
.expect(acceptableHtmlResponse(profile))
.end(done);
});

it('should accept requests for the latest pagebundle versions', function(done) {
var profile = '1.2.1';
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/pagebundle/')
.set('Accept', 'application/json; profile="https://www.mediawiki.org/wiki/Specs/pagebundle/' + profile + '"')
.send({ wikitext: '== h2 ==' })
.expect(200)
.expect(acceptablePageBundleResponse(profile))
.end(done);
});

it('should accept requests for the latest pagebundle versions', function(done) {
var profile = '1.2.1';
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/pagebundle/')
.set('Accept', 'application/json; profile="https://www.mediawiki.org/wiki/Specs/pagebundle/' + profile + '"')
.send({ wikitext: '== h2 ==' })
.expect(200)
.expect(acceptablePageBundleResponse(profile))
.end(done);
});

}); // accepts

describe("wt2html", function() {

var validHtmlResponse = function(expectFunc) {
Expand Down

0 comments on commit d562ea2

Please sign in to comment.