duktape/testrunner/server-node/githubutil.js

/*
 *  Github API and Webhook handling.
 */

var fs = require('fs');
var path = require('path');

var dbutil = require('./dbutil');
var util = require('./util');
var assert = util.assert;

// Send a JSON reply to an Express response.
function sendJsonReply(res, obj) {
    var repData = new Buffer(JSON.stringify(obj), 'utf8');
    res.send(repData);
}

// Create a new github status entry and mark it dirty for background push.
function createGithubStatus(state, status) {
    var db = assert(state.db);
    var github = assert(state.github);

    db.find({
        type: 'github_status',
        user: assert(status.user),
        repo: assert(status.repo),
        sha: assert(status.sha),
        context: assert(status.context)
    }, function (err, docs) {
        var doc;
        if (err) { console.log(err); return; }
        if (docs && docs.length > 0) {
            console.log('github-status already exists');
            return;
        }

        db.insert({
            type: 'github_status',
            user: assert(status.user),
            repo: assert(status.repo),
            sha: assert(status.sha),
            state: assert(status.state),
            target_url: assert(status.target_url),
            description: assert(status.description),
            context: assert(status.context),
            dirty: true
        });
    });
}

// Update an existing github status and mark it dirty.
function updateGithubStatus(state, status) {
    var db = assert(state.db);
    var github = assert(state.github);

    db.find({
        type: 'github_status',
        user: assert(status.user),
        repo: assert(status.repo),
        sha: assert(status.sha),
        context: assert(status.context)
    }, function (err, docs) {
        var doc;
        if (err) { console.log(err); return; }
        if (!docs || docs.length <= 0) {
            console.log('github-status not found');
            return;
        }
        if (docs.length >= 2) {
            console.log('more than one github-status matches, unexpected');
        }
        doc = docs[docs.length - 1];

        var setValues = { dirty: true };
        function check(key) {
            if (typeof status[key] !== 'undefined') {
                setValues[key] = status[key];
            }
        }
        check('state');
        check('target_url');
        check('description');

        db.update({
            _id: doc._id
        }, {
            $set: setValues
        }, function (err, numReplaced) {
            if (err) { throw err; }
        });
    });
}

// Push dirty github status items.  Rate limit Github operations to avoid
// behaving badly even if something goes wrong and we end up retrying a
// push indefinitely.
function pushGithubStatuses(state) {
    var db = assert(state.db);
    var github = assert(state.github);

    db.find({
        type: 'github_status',
        dirty: true
    }, function (err, docs) {
        if (err) { console.log(err); return; }
        if (!docs || docs.length <= 0) { return; }

        docs.forEach(function (doc) {
            state.githubLimiter.removeTokens(1, function (err, remainingTokens) {
                if (err) {
                    console.log(err);
                    return;
                }
                if (remainingTokens < 0) {
                    console.log('skip github status push, rate limited');
                    return;
                }

                console.log('push github status for repo ' + doc.user + '/' + doc.repo +
                            ', commit ' + doc.sha + ', context ' + doc.context +
                            ', state ' + doc.state + '; tokens left: ' + remainingTokens);

                github.authenticate({
                    type: 'basic',
                    username: assert(state.githubAuthUsername),
                    password: assert(state.githubAuthPassword)
                });
                github.statuses.create({
                    user: assert(doc.user),
                    repo: assert(doc.repo),
                    sha: assert(doc.sha),
                    state: assert(doc.state),
                    target_url: assert(doc.target_url),
                    description: assert(doc.description),
                    context: assert(doc.context),
                }, function statusCreated(err, ret) {
                    //console.log('status created:', err, ret);
                    if (err) {
                        console.log(err);

                        // If commit is not found, Github returns something like:
                        // { message: '{"message":"No commit found for SHA: 239e1a240b70576ca123aa14f75c7a5781f6e8c5","documentation_url":"https://developer.github.com/v3/repos/statuses/"}',
                        //   code: 422 }
                        //
                        // Don't retry indefinitely.
                        // XXX: Better fix is a backoff and try count.

                        if (err.code === 422) {
                            console.log('commit no longer exists, don\'t retry status push');
                            // fall through and mark non-dirty
                        } else {
                            return;
                        }
                    }

                    db.update({
                        _id: doc._id
                    }, {
                        $set: {
                            dirty: false
                        },
                    }, function (err, numReplaced) {
                        if (err) { throw err; }
                    });
                });
            });
        });
    });
}

// Tracking table for hanging get-commit-simple, i.e. clients waiting for a
// job to execute.  Clients will be hanging most of the time which avoids
// slow response time due to periodic polling.
var getCommitRequests = [];
var getCommitRequestsCount = null;  // for logging

// Recheck pending get-commit-simple requests: if new matching jobs are
// available, respond to the client and remove the tracking state.  Also
// handles request timeouts.
function handleGetCommitRequests(state) {
    var db = assert(state.db);
    var github = assert(state.github);
    var now = Date.now();

    // XXX: Quite inefficient but good enough for now.

    db.find({
        type: 'commit_simple'
    }, function (err, docs) {
        if (err) { console.log(err); return; }
        if (!docs || docs.length <= 0) { return; }

        docs.forEach(function (doc) {
            if (now - doc.time > 3 * 24 * 3600e3) {
                return;  // ignore webhooks several days old
            }

            getCommitRequests = getCommitRequests.filter(function (client) {
                var i, j, ctx, run, found;

                // XXX: This is racy at the moment, rework to only send a
                // client response when the database has been updated.

                if (now - client.time >= 300e3) {
                    console.log('client request for simple commit job timed out');
                    sendJsonReply(client.res, {
                        error_code: 'TIMEOUT'
                    });  // XXX
                    return false;  // remove
                }

                for (i = 0; i < client.contexts.length; i++) {
                    ctx = client.contexts[i];
                    found = false;
                    for (j = 0; j < doc.runs.length; j++) {
                        run = doc.runs[j];
                        if (run.context === ctx) {
                            found = true;
                        }
                    }
                    if (!found) {
                        // Context ctx not found in runs already, add to runs
                        // and respond to client.

                        doc.runs.push({
                            start_time: Date.now(),
                            end_time: null,
                            context: ctx
                        });

                        db.update({
                            _id: doc._id
                        }, {
                            $set: {
                                runs: doc.runs
                            }
                        }, function (err, numReplaced) {
                            if (err) { throw err; }
                        });

                        console.log('start simple commit job for sha ' + doc.sha + ', context ' + ctx);
                        sendJsonReply(client.res, {
                            repo: assert(doc.repo),
                            repo_full: assert(doc.repo_full),
                            repo_clone_url: assert(doc.repo_clone_url),
                            sha: assert(doc.sha),
                            context: ctx
                        });

                        createGithubStatus(state, {
                            user: assert(state.githubStatusUsername),
                            repo: assert(doc.repo),
                            sha: assert(doc.sha),
                            context: assert(ctx),
                            state: 'pending',
                            target_url: 'http://duktape.org/',  // XXX: useful temporary URI? web UI job status?
                            description: 'Running... (' + (client.client_name || 'no client name') + ')'
                        });

                        // XXX: error recovery / restart; check for start_time
                        // age over sanity (24h?) and remove/reassign job

                        return false;  // no longer pending
                    }
                }

                // No context found, keep in pending state.
                return true;
            });
        });

        var pendingAfter = getCommitRequests.length;
        if (pendingAfter !== getCommitRequestsCount) {
            console.log('pending clients: ' + getCommitRequestsCount + ' -> ' + pendingAfter);
        }
        getCommitRequestsCount = pendingAfter;
    });
}

// Handle a 'push' webhook.
function handleGithubPush(req, res, state) {
    var db = assert(state.db);
    var github = assert(state.github);
    var trustedAuthors = assert(state.githubTrustedAuthors);
    var allowedRepos = assert(state.githubRepos);

    // Careful to avoid automatic test runs unless repo/author is
    // whitelisted.

    var body = req.body;
    var commitHash = body.after;
    var committerName = body.head_commit && body.head_commit.committer && body.head_commit.committer.username;
    var repoName = body.repository && body.repository.name;
    var repoFullName = body.repository && body.repository.full_name;
    if (!repoFullName || !repoName) {
        console.log('ignoring github webhook "push", missing repo name');
        return;
    }
    if (trustedAuthors.indexOf(committerName) < 0) {
        console.log('ignoring github webhook "push" from untrusted committer: ' + committerName);
        return;
    }
    if (allowedRepos.indexOf(repoFullName) < 0) {
        console.log('ignoring github webhook "push" for non-allowed repo: ' + repoFullName);
        return;
    }

    console.log('github webhook "push" to repo ' + repoFullName + ' from trusted committer ' + committerName + ', add automatic jobs');

    // XXX: add a commit_simple UUID for exact matching for get/finish?

    // Tracking object for commit related simple test runs identified by
    // run name, with 'runs' tracking the individual runs and their status.
    // A named run maps directly to a Github status item.
    db.insert({
        type: 'commit_simple',
        time: Date.now(),
        runs: [],
        repo: repoName,
        repo_full: repoFullName,
        repo_clone_url: assert(body.repository.clone_url),
        committer: committerName,
        sha: assert(commitHash)
    });

    handleGetCommitRequests(state);
}

// Create a github-webhook handler.
function makeGithubWebhookHandler(state) {
    var db = assert(state.db);
    var github = assert(state.github);

    return function githubWebhookHandler(req, res) {
        var body = req.body;
        if (typeof body !== 'object') {
           throw new Error('invalid POST body, perhaps client is missing Content-Type?');
        }
        var ghEvent = req.get('X-Github-Event');
        var ghDelivery = req.get('X-Github-Delivery');

        console.log('github webhook, event: ' + ghEvent + ', delivery: ' + ghDelivery);

        // Raw record of webhook requests received.
        db.insert({
            type: 'github_webhook',
            x_github_event: ghEvent,
            x_github_delivery: ghDelivery,
            data: assert(body)
        });

        if (ghEvent === 'push') {
            handleGithubPush(req, res, state);
        } else {
            console.log('unhandled github webhook: ' + ghEvent);
        }

        var rep = {};
        var repData = new Buffer(JSON.stringify(rep), 'utf8');
        res.setHeader('content-type', 'application/json');
        res.send(repData);
    };
}

// Create a get-commit-simple handler.
function makeGetCommitSimpleHandler(state) {
    return function getCommitSimpleHandler(req, res) {
        var body = req.body;
        if (typeof body !== 'object') {
           throw new Error('invalid POST body, perhaps client is missing Content-Type?');
        }

        // body.contexts: list of contexts supported

        getCommitRequests.push({
            time: Date.now(),
            res: res,
            contexts: assert(body.contexts),
            client_name: body.client_name
        });
        handleGetCommitRequests(state);
    };
}

// Create a finish-commit-simple handler.
function makeFinishCommitSimpleHandler(state) {
    var db = assert(state.db);
    var github = assert(state.github);

    return function finishCommitSimpleHandler(req, res) {
        var body = req.body;
        if (typeof body !== 'object') {
           throw new Error('invalid POST body, perhaps client is missing Content-Type?');
        }

        // XXX: add an explicit webhook identifier to update the exact 'commit_simple'
        // instead of the awkward scan below?

        // body.repo_full
        // body.sha
        // body.context
        // body.state        success/failure
        // body.description  oneline description
        // body.text         text, automatically served, github status URI will point to this text file

        function fail(code, desc) {
            var rep = { error_code: code, error_description: desc };
            var repData = new Buffer(JSON.stringify(rep), 'utf8');
            res.setHeader('content-type', 'application/json');
            res.send(repData);
        }
        dbutil.find(db, {
            type: 'commit_simple',
            repo_full: assert(body.repo_full),
            sha: assert(body.sha)
        }).then(function (docs) {
            var doc, i, run;

            // XXX: This is racy now.  Client should also be allowed to retry
            // persistently even if we respond but the response is lost.

            if (!docs || docs.length <= 0) {
                throw new Error('target webhook commit not found');
            }
            if (docs.length > 1) {
                console.log('more than one commit_simple docs found');
            }
            doc = docs[docs.length - 1];

            var output = new Buffer(assert(body.text), 'base64');
            var outputSha = util.sha1sum(output);
            var outputFn = path.join(state.dataDumpDirectory, outputSha);
            var outputUri = assert(state.webBaseUri) + '/out/' + outputSha;
            fs.writeFileSync(outputFn, output);
            console.log('wrote output data to ' + outputFn + ', ' + output.length + ' bytes' +
                        ', link is ' + outputUri);

            for (i = 0; i < doc.runs.length; i++) {
                run = doc.runs[i];
                if (run.context === body.context) {
                    if (run.end_time !== null) {
                        console.log('finish-commit-job already finished, ignoring');
                    } else {
                        run.end_time = Date.now();
                        console.log('finish-commit-job for sha ' + body.sha + ', context ' + body.context + '; took ' +
                                    (run.end_time - run.start_time) / 60e3 + ' mins');

                        db.update({
                            _id: doc.id
                        }, {
                            $set: {
                                runs: doc.runs
                            }
                        }, function (err, numReplaced) {
                            if (err) { throw err; }
                        });
                    }

                    sendJsonReply(res, {});

                    updateGithubStatus(state, {
                        user: assert(state.githubStatusUsername),
                        repo: assert(doc.repo),
                        sha: assert(doc.sha),
                        context: assert(body.context),
                        state: assert(body.state),
                        description: assert(body.description),
                        target_url: assert(outputUri)
                    });

                    return;
                }
            }

            throw new Error('cannot find internal tracking state for context');
        }).catch(function (err) {
            console.log(err);
            fail('INTERNAL_ERROR', String(err));
        });
    }
}

exports.createGithubStatus = createGithubStatus;
exports.updateGithubStatus = updateGithubStatus;
exports.pushGithubStatuses = pushGithubStatuses;
exports.handleGetCommitRequests = handleGetCommitRequests;
exports.makeGithubWebhookHandler = makeGithubWebhookHandler;
exports.makeGetCommitSimpleHandler = makeGetCommitSimpleHandler;
exports.makeFinishCommitSimpleHandler = makeFinishCommitSimpleHandler;