Skip to content

Commit

Permalink
New Materialized View about JobSuccess (#80)
Browse files Browse the repository at this point in the history
* new materialized view jobSuccess

* fix about last materialized view

* v1 generateJobs

* seeds test ok

* generateJobs v2

* build needed

* sequelize close

* adjustement

* builded

* change about date format Materialized View

* index added to job table

* fix [skip ci]

* Materialized View jobSuccess updated to jobStatsByDay [skip ci]

* coalesce materialized view [skip ci]

* incoherence retryJob with jobStatus

* build needed

* fix

* test updated + new pubsub link for retryJob [skip ci]

* first changes about new worker structure

* New ping/pong additionnal subscriptions



* fundation workerUpdate

* workerUpdate on

* workerMonitor works with DB

* getNewClient updated to welcome subscription

* Ping ok now return workers status + type added

* package.json ok

* Model jobSuccessRating added

---------

Co-authored-by: janfi <janfi@users.noreply.github.com>
  • Loading branch information
genouf and janfi authored Oct 11, 2023
1 parent e123d68 commit ca86a4e
Show file tree
Hide file tree
Showing 54 changed files with 4,102 additions and 2,533 deletions.
17 changes: 17 additions & 0 deletions config/config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"development": {
"dialect": "sqlite",
"storage": "data/main.db",
"logging": false
},
"production": {
"dialect": "sqlite",
"storage": "data/main.db",
"logging": false
},
"test": {
"dialect": "sqlite",
"storage": ":memory:",
"logging": false
}
}
35 changes: 35 additions & 0 deletions lib/gnj.js
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ exports.__esModule = true;
var commander_1 = __importDefault(require("commander"));
var migrate_1 = __importDefault(require("./migrate"));
var models_1 = __importDefault(require("./models"));
var generateJobs_1 = require("./scripts/generateJobs");
commander_1["default"]
.command('migrate <configPath>')
.option('-d --dbhash <dbhash>', 'The database hash in case of shared database')
Expand Down Expand Up @@ -74,5 +75,39 @@ commander_1["default"]
});
});
});
commander_1["default"]
.command('seed <configPath> <nbDays> <nbJobsPerDay>')
.description('Seed the job table')
.action(function (configPath, nbDays, nbJobsPerDay) {
return __awaiter(this, void 0, void 0, function () {
var config, models;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (process.env.NODE_ENV !== 'development') {
throw new Error('This command is only available in development mode');
}
config = null;
try {
config = require(configPath);
}
catch (e) {
throw new Error('Could not load the given config.' + e.message);
}
return [4 /*yield*/, (0, models_1["default"])(config, '')];
case 1:
models = _a.sent();
return [4 /*yield*/, (0, generateJobs_1.generateJobs)(models, nbDays, nbJobsPerDay)];
case 2:
_a.sent();
return [4 /*yield*/, models.sequelize.close()];
case 3:
_a.sent();
console.log('Seeding Done');
return [2 /*return*/];
}
});
});
});
commander_1["default"].command('help', 'Display the help');
commander_1["default"].parse(process.argv);
11 changes: 7 additions & 4 deletions lib/graphql/getApolloServer.js
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ var batch_1 = __importDefault(require("./batch"));
var pipeline_1 = __importDefault(require("./pipeline"));
var pipelineStep_1 = __importDefault(require("./pipelineStep"));
var jobHoldType_1 = __importDefault(require("./jobHoldType"));
var workerMonitoring_1 = __importDefault(require("./workerMonitoring"));
var jobSuccessRating_1 = require("./jobSuccessRating");
var workerMonitoring_1 = require("./workerMonitoring");
/**
* @param dbConfig Sequelize database configuration object
* @param gsgParams Params from graphql-sequelize-generator that overwrite the default ones.
Expand All @@ -67,7 +68,7 @@ function getApolloServer(dbConfig, gsgParams, customMutations, onJobFail, wsServ
if (customMutations === void 0) { customMutations = {}; }
if (wsServer === void 0) { wsServer = null; }
return __awaiter(this, void 0, void 0, function () {
var models, types, jobsFail, _i, jobsFail_1, job_2, graphqlSchemaDeclaration;
var models, types, jobsFail, _i, jobsFail_1, job_2, pubSub, graphqlSchemaDeclaration;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, (0, models_1["default"])(dbConfig)];
Expand Down Expand Up @@ -97,13 +98,15 @@ function getApolloServer(dbConfig, gsgParams, customMutations, onJobFail, wsServ
_i++;
return [3 /*break*/, 4];
case 7:
pubSub = gsgParams.pubSubInstance;
graphqlSchemaDeclaration = {
job: (0, job_1["default"])(types, models, onJobFail),
job: (0, job_1["default"])(types, models, pubSub, onJobFail),
batch: (0, batch_1["default"])(types, models),
pipeline: (0, pipeline_1["default"])(types, models),
pipelineStep: (0, pipelineStep_1["default"])(types, models),
jobHoldType: (0, jobHoldType_1["default"])(types, models),
workerMonitoring: (0, workerMonitoring_1["default"])(types, models)
workerMonitoring: (0, workerMonitoring_1.workerMonitoring)(models, pubSub),
jobSuccessRating: (0, jobSuccessRating_1.jobSuccessRating)(models)
};
return [2 /*return*/, (0, graphql_sequelize_generator_1.generateApolloServer)(__assign({ graphqlSchemaDeclaration: graphqlSchemaDeclaration, types: types, models: models, globalPreCallback: function () { }, wsServer: wsServer, apolloServerOptions: {
playground: true,
Expand Down
5 changes: 3 additions & 2 deletions lib/graphql/job.js
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,17 @@ function getInstanceOfDebounceBatch(batchId) {
}
return instance[0].debounce;
}
function JobConfiguration(graphqlTypes, models, onFail) {
function JobConfiguration(graphqlTypes, models, pubSubInstance, onFail) {
var _this = this;
if (pubSubInstance === void 0) { pubSubInstance = null; }
return {
model: models.job,
actions: ['list', 'update', 'create', 'count'],
subscriptions: ['create', 'update', 'delete'],
additionalMutations: {
acquireJob: (0, acquire_1["default"])(graphqlTypes, models),
recover: (0, recover_1["default"])(graphqlTypes, models),
retryJob: (0, retry_1["default"])(graphqlTypes, models)
retryJob: (0, retry_1["default"])(graphqlTypes, models, pubSubInstance)
},
list: {
before: function (findOptions) {
Expand Down
65 changes: 4 additions & 61 deletions lib/graphql/job/acquire.js
Original file line number Diff line number Diff line change
Expand Up @@ -35,34 +35,9 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
exports.__esModule = true;
var graphql_1 = require("graphql");
var sequelize_1 = require("sequelize");
var debounce_1 = __importDefault(require("debounce"));
var allInstanceOfDebounceWorker = [];
function getInstanceOfDebounceWorker(workerId) {
var _this = this;
var instance = allInstanceOfDebounceWorker.filter(function (instance) { return instance.workerId === workerId; });
if (!instance.length) {
allInstanceOfDebounceWorker.push({
workerId: workerId,
debounce: (0, debounce_1["default"])(function (callback) { return callback(); }, 50)
});
return process.env.NO_ASYNC === 'true'
? function (callback) { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
return [2 /*return*/, callback()];
}); }); }
: allInstanceOfDebounceWorker.filter(function (instance) { return instance.workerId === workerId; })[0].debounce;
}
return process.env.NO_ASYNC === 'true'
? function (callback) { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
return [2 /*return*/, callback()];
}); }); }
: instance[0].debounce;
}
function AcquireJobDefinition(graphqlTypes, models) {
var _this = this;
return {
Expand All @@ -72,12 +47,12 @@ function AcquireJobDefinition(graphqlTypes, models) {
typeList: {
type: new graphql_1.GraphQLNonNull(new graphql_1.GraphQLList(new graphql_1.GraphQLNonNull(graphql_1.GraphQLString)))
},
workerId: { type: graphql_1.GraphQLString }
workerId: { type: graphql_1.GraphQLString },
workerType: { type: graphql_1.GraphQLString }
},
resolve: function (source, args, context) { return __awaiter(_this, void 0, void 0, function () {
var transaction, allJobHoldType, heldTypes, conditions, job, debounceWorker;
var transaction, allJobHoldType, heldTypes, conditions, job;
var _a, _b, _c, _d;
var _this = this;
return __generator(this, function (_e) {
switch (_e.label) {
case 0: return [4 /*yield*/, models.sequelize.transaction()];
Expand Down Expand Up @@ -130,39 +105,7 @@ function AcquireJobDefinition(graphqlTypes, models) {
return [4 /*yield*/, transaction.commit()];
case 7:
_e.sent();
if (!args.workerId) return [3 /*break*/, 9];
debounceWorker = getInstanceOfDebounceWorker(args.workerId);
return [4 /*yield*/, debounceWorker(function () { return __awaiter(_this, void 0, void 0, function () {
var workerMonitoring;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, models.workerMonitoring.findOne({
where: { workerId: args.workerId }
})];
case 1:
workerMonitoring = _a.sent();
if (!workerMonitoring) return [3 /*break*/, 3];
return [4 /*yield*/, workerMonitoring.update({
lastCalledAt: new Date()
})];
case 2:
_a.sent();
return [3 /*break*/, 5];
case 3: return [4 /*yield*/, models.workerMonitoring.create({
workerId: args.workerId,
lastCalledAt: new Date()
})];
case 4:
_a.sent();
_a.label = 5;
case 5: return [2 /*return*/];
}
});
}); })];
case 8:
_e.sent();
_e.label = 9;
case 9: return [2 /*return*/, job];
return [2 /*return*/, job];
}
});
}); }
Expand Down
23 changes: 15 additions & 8 deletions lib/graphql/job/retry.js
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,9 @@ var status = [
'cancel-requested',
'cancelled',
];
function RetryJob(graphqlTypes, models) {
function RetryJob(graphqlTypes, models, pubSubInstance) {
var _this = this;
if (pubSubInstance === void 0) { pubSubInstance = null; }
return {
type: graphqlTypes.outputTypes.job,
description: 'Retry a job which fail',
Expand All @@ -77,16 +78,17 @@ function RetryJob(graphqlTypes, models) {
},
resolve: function (source, args, context) { return __awaiter(_this, void 0, void 0, function () {
var job, attributesToDelete, oldJobAttributes, attributes, newJob;
return __generator(this, function (_a) {
switch (_a.label) {
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0: return [4 /*yield*/, models.job.findByPk(args.id)];
case 1:
job = _a.sent();
job = _b.sent();
if (!job) {
throw new Error('The job does not exist.');
}
if (job.status !== 'failed') {
throw new Error('The job must be failed.');
if (job.status !== 'failed' && job.status !== 'cancelled') {
throw new Error('The job must be failed or cancelled.');
}
attributesToDelete = [
'id',
Expand All @@ -103,7 +105,7 @@ function RetryJob(graphqlTypes, models) {
];
oldJobAttributes = Object.keys(job.dataValues).reduce(function (acc, attribute) {
if (!attributesToDelete.includes(attribute)) {
if (attribute === 'output') {
if (attribute === 'output' && job.status !== 'cancelled') {
// We do not keep errors raised by a worker.
if (job.dataValues[attribute].error) {
var _a = job.dataValues[attribute], otherAttributes = _a.otherAttributes, error = __rest(_a, ["otherAttributes"]);
Expand All @@ -118,7 +120,12 @@ function RetryJob(graphqlTypes, models) {
attributes = __assign(__assign({}, oldJobAttributes), { retryOfJobId: job.id });
return [4 /*yield*/, models.job.create(attributes)];
case 2:
newJob = _a.sent();
newJob = _b.sent();
if (pubSubInstance) {
pubSubInstance.publish("jobCreated", (_a = {},
_a["jobCreated"] = newJob.get(),
_a));
}
return [2 /*return*/, newJob];
}
});
Expand Down
10 changes: 10 additions & 0 deletions lib/graphql/jobSuccessRating.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
"use strict";
exports.__esModule = true;
exports.jobSuccessRating = void 0;
function jobSuccessRating(models) {
return {
model: models.jobSuccessRating,
actions: ['list', 'count']
};
}
exports.jobSuccessRating = jobSuccessRating;
21 changes: 19 additions & 2 deletions lib/graphql/workerMonitoring.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,31 @@
"use strict";
exports.__esModule = true;
function WorkerMonitoringConfiguration(types, models) {
exports.workerMonitoring = void 0;
var pinged_1 = require("./workerMonitoring/subscriptions/pinged");
var ponged_1 = require("./workerMonitoring/subscriptions/ponged");
var ping_1 = require("./workerMonitoring/mutations/ping");
var pong_1 = require("./workerMonitoring/mutations/pong");
var workerMonitoringUpdated_1 = require("./workerMonitoring/subscriptions/workerMonitoringUpdated");
var workerMonitorUpdate_1 = require("./workerMonitoring/mutations/workerMonitorUpdate");
function workerMonitoring(models, pubSubInstance) {
return {
model: models.workerMonitoring,
actions: ['list'],
list: {
before: function (findOptions) {
return findOptions;
}
},
additionalMutations: {
ping: (0, ping_1.ping)(pubSubInstance),
pong: (0, pong_1.pong)(pubSubInstance),
workerMonitoringUpdate: (0, workerMonitorUpdate_1.workerMonitorUpdate)(pubSubInstance, models)
},
additionalSubscriptions: {
pinged: (0, pinged_1.pinged)(pubSubInstance),
ponged: (0, ponged_1.ponged)(pubSubInstance),
workerMonitoringUpdated: (0, workerMonitoringUpdated_1.workerMonitorUpdated)(pubSubInstance)
}
};
}
exports["default"] = WorkerMonitoringConfiguration;
exports.workerMonitoring = workerMonitoring;
62 changes: 62 additions & 0 deletions lib/graphql/workerMonitoring/mutations/ping.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
exports.__esModule = true;
exports.ping = void 0;
var type_1 = require("../type");
function ping(pubSubInstance) {
var _this = this;
return {
type: type_1.successType,
description: 'Ping the server.',
args: {},
resolve: function () { return __awaiter(_this, void 0, void 0, function () {
return __generator(this, function (_a) {
try {
pubSubInstance.publish('Pinged', { success: true });
return [2 /*return*/, { success: true }];
}
catch (error) {
console.error('Une erreur est survenue lors du ping:', error);
return [2 /*return*/, { success: false }];
}
return [2 /*return*/];
});
}); }
};
}
exports.ping = ping;
Loading

0 comments on commit ca86a4e

Please sign in to comment.