Skip to content

Commit

Permalink
Merge pull request #11 from HubSpot/guy_experiment_params
Browse files Browse the repository at this point in the history 
add experimentalParams function to the experiment class Fixes #10
  • Loading branch information
Guy committed Aug 30, 2015
2 parents 017e9ce + 4ff0425 commit c143542
Show file tree
Hide file tree
Showing 11 changed files with 808 additions and 745 deletions.
9 changes: 9 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,15 @@ class MyExperiment extends PlanOut.Experiment {
setup() {
//set experiment name, etc.
}

/*
This function should return a list of the possible parameter names that the assignment procedure may assign.
You can optionally override this function to always return this.getDefaultParamNames() which will analyze your program at runtime to determine what the range of possible experimental parameters are. Otherwise, simply return a fixed list of the experimental parameters that your assignment procedure may assign.
*/

getParamNames() {
return this.getDefaultParamNames();
}

assign(params, args) {
params.set('foo', new PlanOut.Ops.Random.UniformChoice({choices: ['a', 'b'], ‘unit’: args.userId}));
Expand Down
64 changes: 43 additions & 21 deletions __tests__/testExperiment.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ describe("Test the experiment module", function() {
log(stuff) {
globalLog.push(stuff);
}
getParamNames() {
return this.getDefaultParamNames();
}
previouslyLogged() {
return;
}
Expand All @@ -75,6 +78,9 @@ describe("Test the experiment module", function() {
previouslyLogged() {
return;
}
getParamNames() {
return this.getDefaultParamNames();
}
setup() {
this.name = 'test_name';
}
Expand All @@ -94,6 +100,10 @@ describe("Test the experiment module", function() {
return;
}

getParamNames() {
return this.getDefaultParamNames();
}

log(stuff) {
globalLog.push(stuff);
}
Expand Down Expand Up @@ -130,6 +140,10 @@ describe("Test the experiment module", function() {
return;
}

getParamNames() {
return this.getDefaultParamNames();
}

log(stuff) {
globalLog.push(stuff);
}
Expand Down Expand Up @@ -163,6 +177,10 @@ describe("Test the experiment module", function() {
return;
}

getParamNames() {
return this.getDefaultParamNames();
}

setup() {
this.name = 'test_name';
}
Expand All @@ -173,9 +191,9 @@ describe("Test the experiment module", function() {
}

var e = new TestAssignmentRetrieval();
expect(e.experimentParameters()).toEqual(['foo', 'boo']);
expect(e.getParamNames()).toEqual(['foo', 'boo']);
var f = new TestAssignmentRetrieval2();
expect(f.experimentParameters()).toEqual([]);
expect(f.getParamNames()).toEqual([]);
});

it('should work with an interpreted experiment', function() {
Expand All @@ -189,32 +207,36 @@ describe("Test the experiment module", function() {
previouslyLogged() {
return;
}

getParamNames() {
return this.getDefaultParamNames();
}
setup() {
this.name = 'test_name';
}

assign(params, args) {
var compiled =
{"op":"seq",
"seq": [
{"op":"set",
"var":"foo",
"value":{
"choices":["a","b"],
"op":"uniformChoice",
"unit": {"op": "get", "var": "i"}
}
},
{"op":"set",
"var":"bar",
"value": 41
}
]};
var proc = new Interpreter(compiled, this.getSalt(), args, params);
var par = proc.getParams();
Object.keys(par).forEach(function(param) {
params.set(param, par[param]);
});
"seq": [
{"op":"set",
"var":"foo",
"value":{
"choices":["a","b"],
"op":"uniformChoice",
"unit": {"op": "get", "var": "i"}
}
},
{"op":"set",
"var":"bar",
"value": 41
}
]};
var proc = new Interpreter(compiled, this.getSalt(), args, params);
var par = proc.getParams();
Object.keys(par).forEach(function(param) {
params.set(param, par[param]);
});
}
};
experimentTester(TestInterpretedExperiment);
Expand Down
136 changes: 75 additions & 61 deletions __tests__/testNamespace.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
var Namespace = require('../es6/namespace.js');
var Experiment = require('../es6/experiment.js');
var Utils = require('../es6/lib/utils.js');

var globalLog = [];
class Experiment1 extends Experiment {
Expand All @@ -12,7 +13,11 @@ class Experiment1 extends Experiment {
}

previouslyLogged() {
return true;
return;
}

getParamNames() {
return this.getDefaultParamNames();
}

setup() {
Expand All @@ -33,12 +38,16 @@ class Experiment2 extends Experiment {
this.name = 'test_name';
}

previouslyLogged() {
return;
}

log(data) {
globalLog.push(data);
}

previouslyLogged() {
return true;
getParamNames() {
return this.getDefaultParamNames();
}

assign(params, args) {
Expand All @@ -55,19 +64,34 @@ class Experiment3 extends Experiment {
this.name = 'test_name';
}

previouslyLogged() {
return;
}

log(data) {
globalLog.push(data);
}

previouslyLogged() {
return true;
getParamNames() {
return this.getDefaultParamNames();
}

assign(params, args) {
params.set("test2", 3)
}
}

class BaseTestNamespace extends Namespace.SimpleNamespace {
setup() {
this.setName('test');
this.setPrimaryUnit('userid');
}

setupDefaults() {
this.numSegments = 100;
}
};


describe("Test namespace module", function() {
var validateLog;
Expand All @@ -81,41 +105,24 @@ describe("Test namespace module", function() {
globalLog = [];
});
it('Adds segment correctly', function() {
class TestNamespace extends Namespace.SimpleNamespace {
setup() {
this.name = "test";
this.setPrimaryUnit('userid');
}

setupDefaults() {
this.numSegments = 100;
}

class TestNamespace extends BaseTestNamespace {
setupExperiments() {
this.addExperiment('Experiment1', Experiment1, 100);
}
}
};
var namespace = new TestNamespace({'userid': 'blah'});
expect(namespace.get('test')).toEqual(1);
validateLog("Experiment1");
});

it('Adds two segments correctly', function() {
class TestNamespace extends Namespace.SimpleNamespace {
setup() {
this.name = "test";
this.setPrimaryUnit('userid');
}

setupDefaults() {
this.numSegments = 100;
}

class TestNamespace extends BaseTestNamespace {
setupExperiments() {
this.addExperiment('Experiment1', Experiment1, 50);
this.addExperiment('Experiment2', Experiment2, 50);
}
}
};

var namespace = new TestNamespace({'userid': 'blah'});
expect(namespace.get('test')).toEqual(1);
validateLog("Experiment1");
Expand All @@ -126,12 +133,7 @@ describe("Test namespace module", function() {
});

it('Can remove segment correctly', function() {
class TestNamespace extends Namespace.SimpleNamespace {
setup() {
this.name = "test";
this.setPrimaryUnit('userid');
}

class TestNamespace extends BaseTestNamespace {
setupDefaults() {
this.numSegments = 10;
}
Expand All @@ -141,7 +143,8 @@ describe("Test namespace module", function() {
this.removeExperiment('Experiment1');
this.addExperiment('Experiment2', Experiment2, 10);
}
}
};

var str = "bla";
for(var i = 0; i < 100; i++) {
str += "h";
Expand All @@ -150,14 +153,9 @@ describe("Test namespace module", function() {
validateLog("Experiment2");
}
});

it('Should only log exposure when user could be in experiment', function() {
class TestNamespace extends Namespace.SimpleNamespace {
setup() {
this.name = "test";
this.setPrimaryUnit('userid');
}

class TestNamespace extends BaseTestNamespace {
setupDefaults() {
this.numSegments = 10;
}
Expand All @@ -176,16 +174,7 @@ describe("Test namespace module", function() {
});

it('Allow experiment overrides in SimpleNamespace', function() {
class TestNamespace extends Namespace.SimpleNamespace {
setup() {
this.name = "test";
this.setPrimaryUnit('userid');
}

setupDefaults() {
this.numSegments = 100;
}

class TestNamespace extends BaseTestNamespace {
setupExperiments() {
this.addExperiment('Experiment1', Experiment1, 50);
this.addExperiment('Experiment3', Experiment3, 50);
Expand Down Expand Up @@ -216,7 +205,7 @@ describe("Test namespace module", function() {
expect(namespace.get('test2')).toEqual('overridden2');
validateLog('Experiment3');
});

it('should respect auto exposure logging being set to off', function() {
class ExperimentNoExposure extends Experiment {
configureLogger() {
Expand All @@ -228,34 +217,59 @@ describe("Test namespace module", function() {
}

previouslyLogged() {
return true;
return false;
}

setup() {
this.setAutoExposureLogging(false);
this.name = 'test_name';
}

getParamNames() {
return this.getDefaultParamNames();
}

assign(params, args) {
params.set('test', 1)
}
};
class TestNamespace extends Namespace.SimpleNamespace {
setup() {
this.name = "test";
this.setPrimaryUnit('userid');
class TestNamespace extends BaseTestNamespace {
setupExperiments() {
this.addExperiment('ExperimentNoExposure', ExperimentNoExposure, 100);
}
};

setupDefaults() {
this.numSegments = 100;
var namespace = new TestNamespace({'userid': 'hi'});
namespace.get('test');
expect(globalLog.length).toEqual(0);
});

it('should respect dynamic getParamNames', function() {
class ExperimentParamTest extends Experiment1 {

assign(params, args) {
let clonedArgs = Utils.shallowCopy(args);
delete clonedArgs.userid;
let keys = Object.keys(clonedArgs);
Utils.forEach(keys, function(key) {
params.set(key, 1);
});
}

getParamNames() {
return ['foo', 'bar'];
}
};
class TestNamespace extends BaseTestNamespace {
setupExperiments() {
this.addExperiment('ExperimentNoExposure', ExperimentNoExposure, 100);
this.addExperiment('ExperimentParamTest', ExperimentParamTest, 100);
}
};
var namespace = new TestNamespace({'userid': 'hi'});
var namespace = new TestNamespace({'userid': 'hi', 'foo': 1, 'bar': 1});
namespace.get('test');
expect(globalLog.length).toEqual(0);
namespace.get('foo');
expect(globalLog.length).toEqual(1);
});

});
Loading

0 comments on commit c143542

Please sign in to comment.