performance benchmark of async-design-patterns - recursive-callbacks vs. promises vs. async/await

classic Classic list List threaded Threaded
3 messages Options
Reply | Threaded
Open this post in threaded view
|

performance benchmark of async-design-patterns - recursive-callbacks vs. promises vs. async/await

kai zhu
fyi, here are some benchmark results of nodejs' client-based http-request throughput, employing various async-design-patterns (on a 4gb linode box).  overall, recursive-callbacks seem to ~15% faster than both async/await and promises (~3000 vs ~2600 client-http-request/s).

```shell
$ REQUESTS_PER_TICK=10 node example.js

state 1 - node (v9.11.1)
state 2 - http-server listening on port 3000
...
state 3 - clientHttpRequestWithRecursiveCallback - flooding http-server with request "http://localhost:3000"
state 5 - clientHttpRequestWithRecursiveCallback - testRun #99
state 5 - clientHttpRequestWithRecursiveCallback - requestsTotal = 14690 (in 5009 ms)
state 5 - clientHttpRequestWithRecursiveCallback - requestsPassed = 7349
state 5 - clientHttpRequestWithRecursiveCallback - requestsFailed = 7341 ({
    "statusCode - 500": true
})
state 5 - clientHttpRequestWithRecursiveCallback - 2933 requests / second
state 5 - mean requests / second = {
    "clientHttpRequestWithRecursiveCallback": "3059 (156 sigma)",
    "clientHttpRequestWithPromise": "2615 (106 sigma)",
    "clientHttpRequestWithAsyncAwait": "2591 (71 sigma)"
}
```


you can reproduce the benchmark-results by running this zero-dependency/zero-config, standalone nodejs script below:


```js
/*
 * example.js
 *
 * this zero-dependency example will benchmark nodejs' client-based http-requests throughput,
 * using recursive-callback/promise/async-await design-patterns.
 *
 * the program will make 100 test-runs (randomly picking a design-pattern per test-run),
 * measuring client-based http-requests/seconde over a 5000 ms interval.
 * it will save the 16 most recent test-runs for each design-pattern,
 * and print the mean and standard deviation.
 * any test-run with unusual errors (timeouts, econnreset, etc),
 * will be discarded and not used in calculations
 *
 * the script accepts one env variable $REQUESTS_PER_TICK, which defaults to 10
 * (you can try increasing it if you have a high-performance machine)
 *
 *
 *
 * example usage:
 * $ REQUESTS_PER_TICK=10 node example.js
 *
 * example output:
 *
 * state 1 - node (v9.11.1)
 * state 2 - http-server listening on port 3000
 * ...
 * state 3 - clientHttpRequestWithRecursiveCallback - flooding http-server with request "http://localhost:3000"
 * state 5 - clientHttpRequestWithRecursiveCallback - testRun #99
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsTotal = 14690 (in 5009 ms)
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsPassed = 7349
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsFailed = 7341 ({
 *     "statusCode - 500": true
 * })
 * state 5 - clientHttpRequestWithRecursiveCallback - 2933 requests / second
 * state 5 - mean requests / second = {
 *     "clientHttpRequestWithRecursiveCallback": "3059 (156 sigma)",
 *     "clientHttpRequestWithPromise": "2615 (106 sigma)",
 *     "clientHttpRequestWithAsyncAwait": "2591 (71 sigma)"
 * }
 *
 * state 6 - process.exit(0)
 */

/*jslint
    bitwise: true,
    browser: true,
    maxerr: 4,
    maxlen: 100,
    node: true,
    nomen: true,
    regexp: true,
    stupid: true
*/

(function () {
    'use strict';
    var local;
    local = {};

    // require modules
    local.http = require('http');
    local.url = require('url');

    /* jslint-ignore-begin */
    local.clientHttpRequestWithAsyncAwait = async function (url, onError) {
    /*
     * this function will make an http-request using async/await design-pattern
     */
        var request, response, timerTimeout;
        try {
            response = await new Promise(function (resolve, reject) {
                // init timeout
                timerTimeout = setTimeout(function () {
                    reject(new Error('timeout - 2000 ms'));
                }, 2000);
                request = local.http.request(local.url.parse(url), resolve);
                request.on('error', reject);
                request.end();
            });
            await new Promise(function (resolve, reject) {
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    reject(new Error('statusCode - ' + response.statusCode));
                    return;
                }
                response.on('end', resolve);
                response.on('error', reject);
            });
        } catch (error) {
            // cleanup timerTimeout
            clearTimeout(timerTimeout);
            // cleanup request and response
            if (request) {
                request.destroy();
            }
            if (response) {
                response.destroy();
            }
            onError(error);
            return;
        }
        onError();
    };
    /* jslint-ignore-end */

    local.clientHttpRequestWithPromise = function (url, onError) {
    /*
     * this function will make an http-request using promise design-pattern
     */
        var request, response, timerTimeout;
        new Promise(function (resolve, reject) {
            // init timeout
            timerTimeout = setTimeout(function () {
                reject(new Error('timeout - 2000 ms'));
            }, 2000);
            request = local.http.request(local.url.parse(url), resolve);
            request.on('error', reject);
            request.end();
        }).then(function (result) {
            return new Promise(function (resolve, reject) {
                response = result;
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    reject(new Error('statusCode - ' + response.statusCode));
                    return;
                }
                response.on('end', resolve);
                response.on('error', reject);
            });
        }).then(onError).catch(function (error) {
            // cleanup timerTimeout
            clearTimeout(timerTimeout);
            // cleanup request and response
            if (request) {
                request.destroy();
            }
            if (response) {
                response.destroy();
            }
            onError(error);
        });
    };

    local.clientHttpRequestWithRecursiveCallback = function (url, onError) {
    /*
     * this function will make an http-request using recursive-callback design-pattern
     */
        var isDone, modeNext, request, response, onNext, timerTimeout;
        onNext = function (error) {
            modeNext += error instanceof Error
                ? Infinity
                : 1;
            switch (modeNext) {
            case 1:
                // init timeout
                timerTimeout = setTimeout(function () {
                    onNext(new Error('timeout - 2000 ms'));
                }, 2000);
                request = local.http.request(local.url.parse(url), onNext);
                request.on('error', onNext);
                request.end();
                break;
            case 2:
                response = error;
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    onNext(new Error('statusCode - ' + response.statusCode));
                }
                response.on('end', onNext);
                response.on('error', onNext);
                break;
            default:
                if (isDone) {
                    return;
                }
                // cleanup timerTimeout
                clearTimeout(timerTimeout);
                // cleanup request and response
                if (request) {
                    request.destroy();
                }
                if (response) {
                    response.destroy();
                }
                isDone = true;
                onError(error);
            }
        };
        modeNext = 0;
        onNext();
    };

    local.clientHttpRequestOnError = function (error) {
    /*
     * this function is the callback for clientHttpRequest
     */
        if (error) {
            local.errorDict[error.message] = true;
            local.requestsFailed += 1;
        } else {
            local.requestsPassed += 1;
        }
        if (local.timeElapsed >= 5000 &&
                (local.requestsFailed + local.requestsPassed) === local.requestsTotal) {
            local.main();
        }
    };

    local.nop = function () {
    /*
     * this function will do nothing
     */
        return;
    };

    local.templateRenderAndPrint = function (template) {
    /*
     * this function render simple double-mustache templates with the local dict,
     * and print to stderr
     */
        console.error(template.replace((/\{\{.*?\}\}/g), function (match0) {
            return local[match0.slice(2, -2)];
        }));
    };

    local.main = function (error) {
    /*
     * this function will fun the main-loop
     */
        local.state += error
            ? Infinity
            : 1;
        switch (local.state) {
        case 1:
            // init local var
            local.clientHttpRequestUrl = 'http://localhost:3000';
            local.version = process.version;
            local.templateRenderAndPrint('state {{state}} - node ({{version}})');
            // create simple http-server that responds with random 200 or 500 statusCode
            local.http.createServer(function (request, response) {
                request
                    // ignore stream-data
                    .on('data', local.nop)
                    .on('error', console.error);
                // respond randomly with either 200 or 500 statusCode
                response.statusCode = Math.random() < 0.5
                    ? 200
                    : 500;
                response
                    .on('error', console.error)
                    .end();
            // listen on port 3000
            }).listen(3000, local.main);
            break;
        case 2:
            local.templateRenderAndPrint('state {{state}} - http-server listening on port 3000');
            local.main();
            break;
        case 3:
            local.clientHttpRequestState = local.clientHttpRequestState || 0;
            local.clientHttpRequestState += 1;
            if (local.clientHttpRequestState < 100) {
                switch (Math.floor(Math.random() * 3)) {
                case 0:
                    local.clientHttpRequest = 'clientHttpRequestWithAsyncAwait';
                    break;
                case 1:
                    local.clientHttpRequest = 'clientHttpRequestWithPromise';
                    break;
                case 2:
                    local.clientHttpRequest = 'clientHttpRequestWithRecursiveCallback';
                    break;
                }
            } else {
                local.state += 2;
                local.main();
                return;
            }
            local.templateRenderAndPrint('\nstate {{state}} - {{clientHttpRequest}} - ' +
                'flooding http-server with request "{{clientHttpRequestUrl}}"');
            local.errorDict = {};
            local.requestsFailed = 0;
            local.requestsPassed = 0;
            local.requestsTotal = 0;
            local.timeElapsed = 0;
            local.timeStart = Date.now();
            local.main();
            break;
        case 4:
            setTimeout(function () {
                for (local.ii = 0;
                        // configurable REQUESTS_PER_TICK
                        local.ii < (Number(process.env.REQUESTS_PER_TICK) || 10);
                        local.ii += 1) {
                    local.requestsTotal += 1;
                    local[local.clientHttpRequest](
                        local.clientHttpRequestUrl,
                        local.clientHttpRequestOnError
                    );
                }
                // recurse / repeat this step for 5000 ms
                local.timeElapsed = Date.now() - local.timeStart;
                if (local.timeElapsed < 5000) {
                    local.state -= 1;
                    local.main();
                }
            });
            break;
        case 5:
            local.timeElapsed = Date.now() - local.timeStart;
            local.requestsPerSecond = Math.round(1000 * local.requestsTotal / local.timeElapsed);
            local.errorDictJson = JSON.stringify(local.errorDict, null, 4);
            local.resultList = local.resultList || {};
            local.resultMean = local.resultMean || {};
            // only save result if no unusual errors occurred
            if (Object.keys(local.errorDict).length <= 1) {
                local.resultList[local.clientHttpRequest] =
                    local.resultList[local.clientHttpRequest] || [];
                local.resultList[local.clientHttpRequest].push(local.requestsPerSecond);
                // remove old data
                if (local.resultList[local.clientHttpRequest].length > 16) {
                    local.resultList[local.clientHttpRequest].shift();
                }
                // calculate mean
                local.resultMean[local.clientHttpRequest] = Math.round(
                    local.resultList[local.clientHttpRequest].reduce(function (aa, bb) {
                        return aa + (bb || 0);
                    }, 0) / local.resultList[local.clientHttpRequest].length
                );
                // calculate sigma
                local.resultMean[local.clientHttpRequest] += ' (' + Math.round(Math.sqrt(
                    local.resultList[local.clientHttpRequest].reduce(function (aa, bb) {
                        return aa + Math.pow(
                            (bb || 0) - local.resultMean[local.clientHttpRequest],
                            2
                        );
                    }, 0) / (local.resultList[local.clientHttpRequest].length - 1)
                )) + ' sigma)';
            }
            local.resultJson = JSON.stringify(local.resultMean, null, 4);
            local.templateRenderAndPrint(
/* jslint-ignore-begin */
'\
state {{state}} - {{clientHttpRequest}} - testRun #{{clientHttpRequestState}}\n\
state {{state}} - {{clientHttpRequest}} - requestsTotal = {{requestsTotal}} (in {{timeElapsed}} ms)\n\
state {{state}} - {{clientHttpRequest}} - requestsPassed = {{requestsPassed}}\n\
state {{state}} - {{clientHttpRequest}} - requestsFailed = {{requestsFailed}} ({{errorDictJson}})\n\
state {{state}} - {{clientHttpRequest}} - {{requestsPerSecond}} requests / second\n\
state {{state}} - mean requests / second = {{resultJson}}\n\
',
/* jslint-ignore-end */
            );
            // repeat test with other design-patterns
            local.state -= 3;
            local.main();
            break;
        default:
            if (error) {
                console.error(error);
            }
            local.exitCode = Number(!!error);
            local.templateRenderAndPrint('state {{state}} - process.exit({{exitCode}})');
            process.exit(local.exitCode);
        }
    };
    // run main-loop
    local.state = 0;
    local.main();
}());
```



_______________________________________________
es-discuss mailing list
[hidden email]
https://mail.mozilla.org/listinfo/es-discuss
Reply | Threaded
Open this post in threaded view
|

Re: performance benchmark of async-design-patterns - recursive-callbacks vs. promises vs. async/await

Michael J. Ryan
Nice... And not really surprising.  I am slightly surprised async/await is so close to promises.  Which means that improving promises performance should probably be a priority.  I still feel the easier to reason with code is well worth it, given many apps now scale horizontally.

On Sun, Apr 29, 2018, 10:31 kai zhu <[hidden email]> wrote:
fyi, here are some benchmark results of nodejs' client-based http-request throughput, employing various async-design-patterns (on a 4gb linode box).  overall, recursive-callbacks seem to ~15% faster than both async/await and promises (~3000 vs ~2600 client-http-request/s).

```shell
$ REQUESTS_PER_TICK=10 node example.js

state 1 - node (v9.11.1)
state 2 - http-server listening on port 3000
...
state 3 - clientHttpRequestWithRecursiveCallback - flooding http-server with request "http://localhost:3000"
state 5 - clientHttpRequestWithRecursiveCallback - testRun #99
state 5 - clientHttpRequestWithRecursiveCallback - requestsTotal = 14690 (in 5009 ms)
state 5 - clientHttpRequestWithRecursiveCallback - requestsPassed = 7349
state 5 - clientHttpRequestWithRecursiveCallback - requestsFailed = 7341 ({
    "statusCode - 500": true
})
state 5 - clientHttpRequestWithRecursiveCallback - 2933 requests / second
state 5 - mean requests / second = {
    "clientHttpRequestWithRecursiveCallback": "3059 (156 sigma)",
    "clientHttpRequestWithPromise": "2615 (106 sigma)",
    "clientHttpRequestWithAsyncAwait": "2591 (71 sigma)"
}
```


you can reproduce the benchmark-results by running this zero-dependency/zero-config, standalone nodejs script below:


```js
/*
 * example.js
 *
 * this zero-dependency example will benchmark nodejs' client-based http-requests throughput,
 * using recursive-callback/promise/async-await design-patterns.
 *
 * the program will make 100 test-runs (randomly picking a design-pattern per test-run),
 * measuring client-based http-requests/seconde over a 5000 ms interval.
 * it will save the 16 most recent test-runs for each design-pattern,
 * and print the mean and standard deviation.
 * any test-run with unusual errors (timeouts, econnreset, etc),
 * will be discarded and not used in calculations
 *
 * the script accepts one env variable $REQUESTS_PER_TICK, which defaults to 10
 * (you can try increasing it if you have a high-performance machine)
 *
 *
 *
 * example usage:
 * $ REQUESTS_PER_TICK=10 node example.js
 *
 * example output:
 *
 * state 1 - node (v9.11.1)
 * state 2 - http-server listening on port 3000
 * ...
 * state 3 - clientHttpRequestWithRecursiveCallback - flooding http-server with request "http://localhost:3000"
 * state 5 - clientHttpRequestWithRecursiveCallback - testRun #99
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsTotal = 14690 (in 5009 ms)
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsPassed = 7349
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsFailed = 7341 ({
 *     "statusCode - 500": true
 * })
 * state 5 - clientHttpRequestWithRecursiveCallback - 2933 requests / second
 * state 5 - mean requests / second = {
 *     "clientHttpRequestWithRecursiveCallback": "3059 (156 sigma)",
 *     "clientHttpRequestWithPromise": "2615 (106 sigma)",
 *     "clientHttpRequestWithAsyncAwait": "2591 (71 sigma)"
 * }
 *
 * state 6 - process.exit(0)
 */

/*jslint
    bitwise: true,
    browser: true,
    maxerr: 4,
    maxlen: 100,
    node: true,
    nomen: true,
    regexp: true,
    stupid: true
*/

(function () {
    'use strict';
    var local;
    local = {};

    // require modules
    local.http = require('http');
    local.url = require('url');

    /* jslint-ignore-begin */
    local.clientHttpRequestWithAsyncAwait = async function (url, onError) {
    /*
     * this function will make an http-request using async/await design-pattern
     */
        var request, response, timerTimeout;
        try {
            response = await new Promise(function (resolve, reject) {
                // init timeout
                timerTimeout = setTimeout(function () {
                    reject(new Error('timeout - 2000 ms'));
                }, 2000);
                request = local.http.request(local.url.parse(url), resolve);
                request.on('error', reject);
                request.end();
            });
            await new Promise(function (resolve, reject) {
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    reject(new Error('statusCode - ' + response.statusCode));
                    return;
                }
                response.on('end', resolve);
                response.on('error', reject);
            });
        } catch (error) {
            // cleanup timerTimeout
            clearTimeout(timerTimeout);
            // cleanup request and response
            if (request) {
                request.destroy();
            }
            if (response) {
                response.destroy();
            }
            onError(error);
            return;
        }
        onError();
    };
    /* jslint-ignore-end */

    local.clientHttpRequestWithPromise = function (url, onError) {
    /*
     * this function will make an http-request using promise design-pattern
     */
        var request, response, timerTimeout;
        new Promise(function (resolve, reject) {
            // init timeout
            timerTimeout = setTimeout(function () {
                reject(new Error('timeout - 2000 ms'));
            }, 2000);
            request = local.http.request(local.url.parse(url), resolve);
            request.on('error', reject);
            request.end();
        }).then(function (result) {
            return new Promise(function (resolve, reject) {
                response = result;
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    reject(new Error('statusCode - ' + response.statusCode));
                    return;
                }
                response.on('end', resolve);
                response.on('error', reject);
            });
        }).then(onError).catch(function (error) {
            // cleanup timerTimeout
            clearTimeout(timerTimeout);
            // cleanup request and response
            if (request) {
                request.destroy();
            }
            if (response) {
                response.destroy();
            }
            onError(error);
        });
    };

    local.clientHttpRequestWithRecursiveCallback = function (url, onError) {
    /*
     * this function will make an http-request using recursive-callback design-pattern
     */
        var isDone, modeNext, request, response, onNext, timerTimeout;
        onNext = function (error) {
            modeNext += error instanceof Error
                ? Infinity
                : 1;
            switch (modeNext) {
            case 1:
                // init timeout
                timerTimeout = setTimeout(function () {
                    onNext(new Error('timeout - 2000 ms'));
                }, 2000);
                request = local.http.request(local.url.parse(url), onNext);
                request.on('error', onNext);
                request.end();
                break;
            case 2:
                response = error;
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    onNext(new Error('statusCode - ' + response.statusCode));
                }
                response.on('end', onNext);
                response.on('error', onNext);
                break;
            default:
                if (isDone) {
                    return;
                }
                // cleanup timerTimeout
                clearTimeout(timerTimeout);
                // cleanup request and response
                if (request) {
                    request.destroy();
                }
                if (response) {
                    response.destroy();
                }
                isDone = true;
                onError(error);
            }
        };
        modeNext = 0;
        onNext();
    };

    local.clientHttpRequestOnError = function (error) {
    /*
     * this function is the callback for clientHttpRequest
     */
        if (error) {
            local.errorDict[error.message] = true;
            local.requestsFailed += 1;
        } else {
            local.requestsPassed += 1;
        }
        if (local.timeElapsed >= 5000 &&
                (local.requestsFailed + local.requestsPassed) === local.requestsTotal) {
            local.main();
        }
    };

    local.nop = function () {
    /*
     * this function will do nothing
     */
        return;
    };

    local.templateRenderAndPrint = function (template) {
    /*
     * this function render simple double-mustache templates with the local dict,
     * and print to stderr
     */
        console.error(template.replace((/\{\{.*?\}\}/g), function (match0) {
            return local[match0.slice(2, -2)];
        }));
    };

    local.main = function (error) {
    /*
     * this function will fun the main-loop
     */
        local.state += error
            ? Infinity
            : 1;
        switch (local.state) {
        case 1:
            // init local var
            local.clientHttpRequestUrl = 'http://localhost:3000';
            local.version = process.version;
            local.templateRenderAndPrint('state {{state}} - node ({{version}})');
            // create simple http-server that responds with random 200 or 500 statusCode
            local.http.createServer(function (request, response) {
                request
                    // ignore stream-data
                    .on('data', local.nop)
                    .on('error', console.error);
                // respond randomly with either 200 or 500 statusCode
                response.statusCode = Math.random() < 0.5
                    ? 200
                    : 500;
                response
                    .on('error', console.error)
                    .end();
            // listen on port 3000
            }).listen(3000, local.main);
            break;
        case 2:
            local.templateRenderAndPrint('state {{state}} - http-server listening on port 3000');
            local.main();
            break;
        case 3:
            local.clientHttpRequestState = local.clientHttpRequestState || 0;
            local.clientHttpRequestState += 1;
            if (local.clientHttpRequestState < 100) {
                switch (Math.floor(Math.random() * 3)) {
                case 0:
                    local.clientHttpRequest = 'clientHttpRequestWithAsyncAwait';
                    break;
                case 1:
                    local.clientHttpRequest = 'clientHttpRequestWithPromise';
                    break;
                case 2:
                    local.clientHttpRequest = 'clientHttpRequestWithRecursiveCallback';
                    break;
                }
            } else {
                local.state += 2;
                local.main();
                return;
            }
            local.templateRenderAndPrint('\nstate {{state}} - {{clientHttpRequest}} - ' +
                'flooding http-server with request "{{clientHttpRequestUrl}}"');
            local.errorDict = {};
            local.requestsFailed = 0;
            local.requestsPassed = 0;
            local.requestsTotal = 0;
            local.timeElapsed = 0;
            local.timeStart = Date.now();
            local.main();
            break;
        case 4:
            setTimeout(function () {
                for (local.ii = 0;
                        // configurable REQUESTS_PER_TICK
                        local.ii < (Number(process.env.REQUESTS_PER_TICK) || 10);
                        local.ii += 1) {
                    local.requestsTotal += 1;
                    local[local.clientHttpRequest](
                        local.clientHttpRequestUrl,
                        local.clientHttpRequestOnError
                    );
                }
                // recurse / repeat this step for 5000 ms
                local.timeElapsed = Date.now() - local.timeStart;
                if (local.timeElapsed < 5000) {
                    local.state -= 1;
                    local.main();
                }
            });
            break;
        case 5:
            local.timeElapsed = Date.now() - local.timeStart;
            local.requestsPerSecond = Math.round(1000 * local.requestsTotal / local.timeElapsed);
            local.errorDictJson = JSON.stringify(local.errorDict, null, 4);
            local.resultList = local.resultList || {};
            local.resultMean = local.resultMean || {};
            // only save result if no unusual errors occurred
            if (Object.keys(local.errorDict).length <= 1) {
                local.resultList[local.clientHttpRequest] =
                    local.resultList[local.clientHttpRequest] || [];
                local.resultList[local.clientHttpRequest].push(local.requestsPerSecond);
                // remove old data
                if (local.resultList[local.clientHttpRequest].length > 16) {
                    local.resultList[local.clientHttpRequest].shift();
                }
                // calculate mean
                local.resultMean[local.clientHttpRequest] = Math.round(
                    local.resultList[local.clientHttpRequest].reduce(function (aa, bb) {
                        return aa + (bb || 0);
                    }, 0) / local.resultList[local.clientHttpRequest].length
                );
                // calculate sigma
                local.resultMean[local.clientHttpRequest] += ' (' + Math.round(Math.sqrt(
                    local.resultList[local.clientHttpRequest].reduce(function (aa, bb) {
                        return aa + Math.pow(
                            (bb || 0) - local.resultMean[local.clientHttpRequest],
                            2
                        );
                    }, 0) / (local.resultList[local.clientHttpRequest].length - 1)
                )) + ' sigma)';
            }
            local.resultJson = JSON.stringify(local.resultMean, null, 4);
            local.templateRenderAndPrint(
/* jslint-ignore-begin */
'\
state {{state}} - {{clientHttpRequest}} - testRun #{{clientHttpRequestState}}\n\
state {{state}} - {{clientHttpRequest}} - requestsTotal = {{requestsTotal}} (in {{timeElapsed}} ms)\n\
state {{state}} - {{clientHttpRequest}} - requestsPassed = {{requestsPassed}}\n\
state {{state}} - {{clientHttpRequest}} - requestsFailed = {{requestsFailed}} ({{errorDictJson}})\n\
state {{state}} - {{clientHttpRequest}} - {{requestsPerSecond}} requests / second\n\
state {{state}} - mean requests / second = {{resultJson}}\n\
',
/* jslint-ignore-end */
            );
            // repeat test with other design-patterns
            local.state -= 3;
            local.main();
            break;
        default:
            if (error) {
                console.error(error);
            }
            local.exitCode = Number(!!error);
            local.templateRenderAndPrint('state {{state}} - process.exit({{exitCode}})');
            process.exit(local.exitCode);
        }
    };
    // run main-loop
    local.state = 0;
    local.main();
}());
```


_______________________________________________
es-discuss mailing list
[hidden email]
https://mail.mozilla.org/listinfo/es-discuss

_______________________________________________
es-discuss mailing list
[hidden email]
https://mail.mozilla.org/listinfo/es-discuss
Reply | Threaded
Open this post in threaded view
|

Re: [v8-dev] performance benchmark of async-design-patterns - recursive-callbacks vs. promises vs. async/await

kai zhu
i ran the test-script on travis-ci against various v8/nodejs versions (including v8-6.6/node-10) [1] [2].

here are the raw data and visualization [3] of recursive-callback/promise/async-await vs v8/nodejs-versions.  not-surprisingly, recursive-callbacks are consistently faster, but only a small margin (7-13%) than promises across historical nodejs versions on travis-ci.

of note:
- the standard deviation error-bars should be taken with a grain of salt, as the travis-ci rerun indicates higher inconsistency
- the low numbers from node-0.9 and node-0.10 are meaningless, as the sockets seem to constantly hang and timeout according to travis logs.
- note the sudden drop for the travis-ci rerun of node-9.11.1 (don’t have explanation, and the travis-logs shows the low numbers are quite consistent within the rerun)
- node-10.0.0 is slower than node-8.11.1, including the travis rerun
- async/await is only available on node-7 and higher


[1] travis-ci build logs #20

[2] travis-ci build logs #21 (rerun to verify consistency)

[3] visualisation



```json
[
    {
        "version": "v8-3.14.5.8<br>node-0.9.12",
        "clientHttpRequestWithRecursiveCallback": "456 (83 sigma)",
        "recursiveCallbackVsPromiseRatio": null
    },
    {
        "version": "v8-3.14.5.11<br>node-0.10.48",
        "clientHttpRequestWithRecursiveCallback": "49 (1 sigma)",
        "recursiveCallbackVsPromiseRatio": null
    },
    {
        "version": "v8-3.28.73<br>node-0.11.16",
        "clientHttpRequestWithRecursiveCallback": "1997 (41 sigma)",
        "clientHttpRequestWithPromise": "1854 (67 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0771305285868393
    },
    {
        "version": "v8-3.28.71.20<br>node-0.12.18",
        "clientHttpRequestWithRecursiveCallback": "1964 (14 sigma)",
        "clientHttpRequestWithPromise": "1829 (15 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0738108255877528
    },
    {
        "version": "v8-4.5.103.53<br>node-4.9.1",
        "clientHttpRequestWithRecursiveCallback": "2355 (31 sigma)",
        "clientHttpRequestWithPromise": "2170 (22 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0852534562211982
    },
    {
        "version": "v8-4.6.85.32<br>node-5.12.0",
        "clientHttpRequestWithRecursiveCallback": "2471 (69 sigma)",
        "clientHttpRequestWithPromise": "2319 (76 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0655454937473048
    },
    {
        "version": "v8-5.1.281.111<br>node-6.14.1",
        "clientHttpRequestWithRecursiveCallback": "2589 (37 sigma)",
        "clientHttpRequestWithPromise": "2374 (38 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0905644481887111
    },
    {
        "version": "v8-5.5.372.43<br>node-7.10.1",
        "clientHttpRequestWithRecursiveCallback": "3238 (53 sigma)",
        "clientHttpRequestWithPromise": "2846 (35 sigma)",
        "clientHttpRequestWithAsyncAwait": "2860 (31 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.1377371749824314
    },
    {
        "version": "v8-6.2.414.50<br>node-8.11.1",
        "clientHttpRequestWithRecursiveCallback": "4699 (74 sigma)",
        "clientHttpRequestWithPromise": "4305 (88 sigma)",
        "clientHttpRequestWithAsyncAwait": "4294 (52 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.091521486643438
    },
    {
        "version": "v8-6.2.414.46-node.23<br>node-9.11.1",
        "clientHttpRequestWithRecursiveCallback": "4809 (80 sigma)",
        "clientHttpRequestWithPromise": "4423 (94 sigma)",
        "clientHttpRequestWithAsyncAwait": "4404 (81 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.087271082975356
    },
    {
        "version": "v8-6.6.346.24-node.5<br>node-10.0.0",
        "clientHttpRequestWithRecursiveCallback": "4100 (69 sigma)",
        "clientHttpRequestWithPromise": "3836 (98 sigma)",
        "clientHttpRequestWithAsyncAwait": "3837 (46 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0688216892596454
    },
    {
        "version": "v8-3.14.5.8<br>node-0.9.12",
        "clientHttpRequestWithRecursiveCallback-rerun": "477 (46 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": null
    },
    {
        "version": "v8-3.14.5.11<br>node-0.10.48",
        "clientHttpRequestWithRecursiveCallback-rerun": "48 (2 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": null
    },
    {
        "version": "v8-3.28.73<br>node-0.11.16",
        "clientHttpRequestWithRecursiveCallback-rerun": "1887 (25 sigma)",
        "clientHttpRequestWithPromise-rerun": "1763 (25 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0771305285868393
    },
    {
        "version": "v8-3.28.71.20<br>node-0.12.18",
        "clientHttpRequestWithRecursiveCallback-rerun": "2494 (23 sigma)",
        "clientHttpRequestWithPromise-rerun": "2282 (35 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0738108255877528
    },
    {
        "version": "v8-4.5.103.53<br>node-4.9.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "3246 (99 sigma)",
        "clientHttpRequestWithPromise-rerun": "3008 (91 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0852534562211982
    },
    {
        "version": "v8-4.6.85.32<br>node-5.12.0",
        "clientHttpRequestWithRecursiveCallback-rerun": "2709 (26 sigma)",
        "clientHttpRequestWithPromise-rerun": "2507 (30 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0655454937473048
    },
    {
        "version": "v8-5.1.281.111<br>node-6.14.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "2401 (38 sigma)",
        "clientHttpRequestWithPromise-rerun": "2193 (55 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0905644481887111
    },
    {
        "version": "v8-5.5.372.43<br>node-7.10.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "4065 (83 sigma)",
        "clientHttpRequestWithPromise-rerun": "3565 (91 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "3620 (89 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.1377371749824314
    },
    {
        "version": "v8-6.2.414.50<br>node-8.11.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "4915 (56 sigma)",
        "clientHttpRequestWithPromise-rerun": "4537 (49 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "4476 (44 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.091521486643438
    },
    {
        "version": "v8-6.2.414.46-node.23<br>node-9.11.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "2141 (67 sigma)",
        "clientHttpRequestWithPromise-rerun": "2011 (67 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "2019 (43 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.087271082975356
    },
    {
        "version": "v8-6.6.346.24-node.5<br>node-10.0.0",
        "clientHttpRequestWithRecursiveCallback-rerun": "3609 (84 sigma)",
        "clientHttpRequestWithPromise-rerun": "3350 (47 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "3353 (65 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0688216892596454
    }
]
```


On 30 Apr 2018, at 2:08 PM, Benedikt Meurer <[hidden email]> wrote:

Only 15% slower Promise (or async/await) based version sounds awesome. I'm not sure we can squeeze a lot more out of this. But having a benchmark setup on Node Infrastructure would be very much welcome.

-- Benedikt

On Sun, Apr 29, 2018 at 10:52 PM Caitlin Potter <[hidden email]> wrote:
I’d be very interested to see how this runs in node 10.x or Canary, which have some major optimizations to Promises and RunMicrotasks.

Hopefully the async/await and Promise numbers do a bit better than before.
On Apr 29, 2018, at 4:24 PM, Michael J. Ryan <[hidden email]> wrote:

Nice... And not really surprising.  I am slightly surprised async/await is so close to promises.  Which means that improving promises performance should probably be a priority.  I still feel the easier to reason with code is well worth it, given many apps now scale horizontally.

On Sun, Apr 29, 2018, 10:31 kai zhu <[hidden email]> wrote:
fyi, here are some benchmark results of nodejs' client-based http-request throughput, employing various async-design-patterns (on a 4gb linode box).  overall, recursive-callbacks seem to ~15% faster than both async/await and promises (~3000 vs ~2600 client-http-request/s).

```shell
$ REQUESTS_PER_TICK=10 node example.js

state 1 - node (v9.11.1)
state 2 - http-server listening on port 3000
...
state 3 - clientHttpRequestWithRecursiveCallback - flooding http-server with request "http://localhost:3000"
state 5 - clientHttpRequestWithRecursiveCallback - testRun #99
state 5 - clientHttpRequestWithRecursiveCallback - requestsTotal = 14690 (in 5009 ms)
state 5 - clientHttpRequestWithRecursiveCallback - requestsPassed = 7349
state 5 - clientHttpRequestWithRecursiveCallback - requestsFailed = 7341 ({
    "statusCode - 500": true
})
state 5 - clientHttpRequestWithRecursiveCallback - 2933 requests / second
state 5 - mean requests / second = {
    "clientHttpRequestWithRecursiveCallback": "3059 (156 sigma)",
    "clientHttpRequestWithPromise": "2615 (106 sigma)",
    "clientHttpRequestWithAsyncAwait": "2591 (71 sigma)"
}
```


you can reproduce the benchmark-results by running this zero-dependency/zero-config, standalone nodejs script below:


```js
/*
 * example.js
 *
 * this zero-dependency example will benchmark nodejs' client-based http-requests throughput,
 * using recursive-callback/promise/async-await design-patterns.
 *
 * the program will make 100 test-runs (randomly picking a design-pattern per test-run),
 * measuring client-based http-requests/seconde over a 5000 ms interval.
 * it will save the 16 most recent test-runs for each design-pattern,
 * and print the mean and standard deviation.
 * any test-run with unusual errors (timeouts, econnreset, etc),
 * will be discarded and not used in calculations
 *
 * the script accepts one env variable $REQUESTS_PER_TICK, which defaults to 10
 * (you can try increasing it if you have a high-performance machine)
 *
 *
 *
 * example usage:
 * $ REQUESTS_PER_TICK=10 node example.js
 *
 * example output:
 *
 * state 1 - node (v9.11.1)
 * state 2 - http-server listening on port 3000
 * ...
 * state 3 - clientHttpRequestWithRecursiveCallback - flooding http-server with request "http://localhost:3000"
 * state 5 - clientHttpRequestWithRecursiveCallback - testRun #99
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsTotal = 14690 (in 5009 ms)
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsPassed = 7349
 * state 5 - clientHttpRequestWithRecursiveCallback - requestsFailed = 7341 ({
 *     "statusCode - 500": true
 * })
 * state 5 - clientHttpRequestWithRecursiveCallback - 2933 requests / second
 * state 5 - mean requests / second = {
 *     "clientHttpRequestWithRecursiveCallback": "3059 (156 sigma)",
 *     "clientHttpRequestWithPromise": "2615 (106 sigma)",
 *     "clientHttpRequestWithAsyncAwait": "2591 (71 sigma)"
 * }
 *
 * state 6 - process.exit(0)
 */

/*jslint
    bitwise: true,
    browser: true,
    maxerr: 4,
    maxlen: 100,
    node: true,
    nomen: true,
    regexp: true,
    stupid: true
*/

(function () {
    'use strict';
    var local;
    local = {};

    // require modules
    local.http = require('http');
    local.url = require('url');

    /* jslint-ignore-begin */
    local.clientHttpRequestWithAsyncAwait = async function (url, onError) {
    /*
     * this function will make an http-request using async/await design-pattern
     */
        var request, response, timerTimeout;
        try {
            response = await new Promise(function (resolve, reject) {
                // init timeout
                timerTimeout = setTimeout(function () {
                    reject(new Error('timeout - 2000 ms'));
                }, 2000);
                request = local.http.request(local.url.parse(url), resolve);
                request.on('error', reject);
                request.end();
            });
            await new Promise(function (resolve, reject) {
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    reject(new Error('statusCode - ' + response.statusCode));
                    return;
                }
                response.on('end', resolve);
                response.on('error', reject);
            });
        } catch (error) {
            // cleanup timerTimeout
            clearTimeout(timerTimeout);
            // cleanup request and response
            if (request) {
                request.destroy();
            }
            if (response) {
                response.destroy();
            }
            onError(error);
            return;
        }
        onError();
    };
    /* jslint-ignore-end */

    local.clientHttpRequestWithPromise = function (url, onError) {
    /*
     * this function will make an http-request using promise design-pattern
     */
        var request, response, timerTimeout;
        new Promise(function (resolve, reject) {
            // init timeout
            timerTimeout = setTimeout(function () {
                reject(new Error('timeout - 2000 ms'));
            }, 2000);
            request = local.http.request(local.url.parse(url), resolve);
            request.on('error', reject);
            request.end();
        }).then(function (result) {
            return new Promise(function (resolve, reject) {
                response = result;
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    reject(new Error('statusCode - ' + response.statusCode));
                    return;
                }
                response.on('end', resolve);
                response.on('error', reject);
            });
        }).then(onError).catch(function (error) {
            // cleanup timerTimeout
            clearTimeout(timerTimeout);
            // cleanup request and response
            if (request) {
                request.destroy();
            }
            if (response) {
                response.destroy();
            }
            onError(error);
        });
    };

    local.clientHttpRequestWithRecursiveCallback = function (url, onError) {
    /*
     * this function will make an http-request using recursive-callback design-pattern
     */
        var isDone, modeNext, request, response, onNext, timerTimeout;
        onNext = function (error) {
            modeNext += error instanceof Error
                ? Infinity
                : 1;
            switch (modeNext) {
            case 1:
                // init timeout
                timerTimeout = setTimeout(function () {
                    onNext(new Error('timeout - 2000 ms'));
                }, 2000);
                request = local.http.request(local.url.parse(url), onNext);
                request.on('error', onNext);
                request.end();
                break;
            case 2:
                response = error;
                // ignore stream-data
                response.on('data', local.nop);
                if (response.statusCode >= 400) {
                    onNext(new Error('statusCode - ' + response.statusCode));
                }
                response.on('end', onNext);
                response.on('error', onNext);
                break;
            default:
                if (isDone) {
                    return;
                }
                // cleanup timerTimeout
                clearTimeout(timerTimeout);
                // cleanup request and response
                if (request) {
                    request.destroy();
                }
                if (response) {
                    response.destroy();
                }
                isDone = true;
                onError(error);
            }
        };
        modeNext = 0;
        onNext();
    };

    local.clientHttpRequestOnError = function (error) {
    /*
     * this function is the callback for clientHttpRequest
     */
        if (error) {
            local.errorDict[error.message] = true;
            local.requestsFailed += 1;
        } else {
            local.requestsPassed += 1;
        }
        if (local.timeElapsed >= 5000 &&
                (local.requestsFailed + local.requestsPassed) === local.requestsTotal) {
            local.main();
        }
    };

    local.nop = function () {
    /*
     * this function will do nothing
     */
        return;
    };

    local.templateRenderAndPrint = function (template) {
    /*
     * this function render simple double-mustache templates with the local dict,
     * and print to stderr
     */
        console.error(template.replace((/\{\{.*?\}\}/g), function (match0) {
            return local[match0.slice(2, -2)];
        }));
    };

    local.main = function (error) {
    /*
     * this function will fun the main-loop
     */
        local.state += error
            ? Infinity
            : 1;
        switch (local.state) {
        case 1:
            // init local var
            local.clientHttpRequestUrl = 'http://localhost:3000';
            local.version = process.version;
            local.templateRenderAndPrint('state {{state}} - node ({{version}})');
            // create simple http-server that responds with random 200 or 500 statusCode
            local.http.createServer(function (request, response) {
                request
                    // ignore stream-data
                    .on('data', local.nop)
                    .on('error', console.error);
                // respond randomly with either 200 or 500 statusCode
                response.statusCode = Math.random() < 0.5
                    ? 200
                    : 500;
                response
                    .on('error', console.error)
                    .end();
            // listen on port 3000
            }).listen(3000, local.main);
            break;
        case 2:
            local.templateRenderAndPrint('state {{state}} - http-server listening on port 3000');
            local.main();
            break;
        case 3:
            local.clientHttpRequestState = local.clientHttpRequestState || 0;
            local.clientHttpRequestState += 1;
            if (local.clientHttpRequestState < 100) {
                switch (Math.floor(Math.random() * 3)) {
                case 0:
                    local.clientHttpRequest = 'clientHttpRequestWithAsyncAwait';
                    break;
                case 1:
                    local.clientHttpRequest = 'clientHttpRequestWithPromise';
                    break;
                case 2:
                    local.clientHttpRequest = 'clientHttpRequestWithRecursiveCallback';
                    break;
                }
            } else {
                local.state += 2;
                local.main();
                return;
            }
            local.templateRenderAndPrint('\nstate {{state}} - {{clientHttpRequest}} - ' +
                'flooding http-server with request "{{clientHttpRequestUrl}}"');
            local.errorDict = {};
            local.requestsFailed = 0;
            local.requestsPassed = 0;
            local.requestsTotal = 0;
            local.timeElapsed = 0;
            local.timeStart = Date.now();
            local.main();
            break;
        case 4:
            setTimeout(function () {
                for (local.ii = 0;
                        // configurable REQUESTS_PER_TICK
                        local.ii < (Number(process.env.REQUESTS_PER_TICK) || 10);
                        local.ii += 1) {
                    local.requestsTotal += 1;
                    local[local.clientHttpRequest](
                        local.clientHttpRequestUrl,
                        local.clientHttpRequestOnError
                    );
                }
                // recurse / repeat this step for 5000 ms
                local.timeElapsed = Date.now() - local.timeStart;
                if (local.timeElapsed < 5000) {
                    local.state -= 1;
                    local.main();
                }
            });
            break;
        case 5:
            local.timeElapsed = Date.now() - local.timeStart;
            local.requestsPerSecond = Math.round(1000 * local.requestsTotal / local.timeElapsed);
            local.errorDictJson = JSON.stringify(local.errorDict, null, 4);
            local.resultList = local.resultList || {};
            local.resultMean = local.resultMean || {};
            // only save result if no unusual errors occurred
            if (Object.keys(local.errorDict).length <= 1) {
                local.resultList[local.clientHttpRequest] =
                    local.resultList[local.clientHttpRequest] || [];
                local.resultList[local.clientHttpRequest].push(local.requestsPerSecond);
                // remove old data
                if (local.resultList[local.clientHttpRequest].length > 16) {
                    local.resultList[local.clientHttpRequest].shift();
                }
                // calculate mean
                local.resultMean[local.clientHttpRequest] = Math.round(
                    local.resultList[local.clientHttpRequest].reduce(function (aa, bb) {
                        return aa + (bb || 0);
                    }, 0) / local.resultList[local.clientHttpRequest].length
                );
                // calculate sigma
                local.resultMean[local.clientHttpRequest] += ' (' + Math.round(Math.sqrt(
                    local.resultList[local.clientHttpRequest].reduce(function (aa, bb) {
                        return aa + Math.pow(
                            (bb || 0) - local.resultMean[local.clientHttpRequest],
                            2
                        );
                    }, 0) / (local.resultList[local.clientHttpRequest].length - 1)
                )) + ' sigma)';
            }
            local.resultJson = JSON.stringify(local.resultMean, null, 4);
            local.templateRenderAndPrint(
/* jslint-ignore-begin */
'\
state {{state}} - {{clientHttpRequest}} - testRun #{{clientHttpRequestState}}\n\
state {{state}} - {{clientHttpRequest}} - requestsTotal = {{requestsTotal}} (in {{timeElapsed}} ms)\n\
state {{state}} - {{clientHttpRequest}} - requestsPassed = {{requestsPassed}}\n\
state {{state}} - {{clientHttpRequest}} - requestsFailed = {{requestsFailed}} ({{errorDictJson}})\n\
state {{state}} - {{clientHttpRequest}} - {{requestsPerSecond}} requests / second\n\
state {{state}} - mean requests / second = {{resultJson}}\n\
',
/* jslint-ignore-end */
            );
            // repeat test with other design-patterns
            local.state -= 3;
            local.main();
            break;
        default:
            if (error) {
                console.error(error);
            }
            local.exitCode = Number(!!error);
            local.templateRenderAndPrint('state {{state}} - process.exit({{exitCode}})');
            process.exit(local.exitCode);
        }
    };
    // run main-loop
    local.state = 0;
    local.main();
}());
```


_______________________________________________
es-discuss mailing list
[hidden email]
https://mail.mozilla.org/listinfo/es-discuss

--
--
v8-dev mailing list
[hidden email]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [hidden email].
For more options, visit https://groups.google.com/d/optout.

--
--
v8-dev mailing list
[hidden email]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [hidden email].
For more options, visit https://groups.google.com/d/optout.
--



 •  Benedikt Meurer
 •  Google Germany GmbH
 •  Erika-Mann-Str. 33
 •  80636 Munich
 •  [hidden email]

Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg
Diese E-Mail ist vertraulich. Wenn Sie nicht der richtige Adressat sind, leiten Sie diese bitte nicht weiter, informieren Sie den Absender und löschen Sie die E-Mail und alle Anhänge. Vielen Dank. This e-mail is confidential. If you are not the right addressee please do not forward it, please inform the sender, and please erase this e-mail including any attachments. Thanks.


_______________________________________________
es-discuss mailing list
[hidden email]
https://mail.mozilla.org/listinfo/es-discuss