Merge pull request #253 from jamestalmage/disallow-end-callback-on-non-async

Breaking: `t.end` requires `test.async`, and `t.plan` will not auto end tests. Fixes #244.

lib/runner.js

@@ -32,7 +32,8 @@ function Runner(opts) {
 		type: 'test',
 		serial: false,
 		exclusive: false,
-		skipped: false
+		skipped: false,
+		async: false
 	}, this._addFn.bind(this));
 }
 
@@ -46,7 +47,8 @@ var chainableFunctions = {
 	skip: {skipped: true},
 	only: {exclusive: true},
 	beforeEach: {type: 'beforeEach'},
-	afterEach: {type: 'afterEach'}
+	afterEach: {type: 'afterEach'},
+	async: {async: true}
 };
 
 function makeChain(defaults, parentAdd) {

lib/test.js

@@ -6,6 +6,7 @@ var fnName = require('fn-name');
 var co = require('co-with-promise');
 var observableToPromise = require('observable-to-promise');
 var isPromise = require('is-promise');
+var isObservable = require('is-observable');
 var assert = require('./assert');
 var globals = require('./globals');
 
@@ -28,6 +29,10 @@ function Test(title, fn) {
 	this.duration = null;
 	this.assertError = undefined;
 
+	// TODO(jamestalmage): make this an optional constructor arg instead of having Runner set it after the fact.
+	// metadata should just always exist, otherwise it requires a bunch of ugly checks all over the place.
+	this.metadata = {};
+
 	// store the time point before test execution
 	// to calculate the total time spent in test
 	this._timeStart = null;
@@ -46,10 +51,6 @@ module.exports = Test;
 
 Test.prototype._assert = function () {
 	this.assertCount++;
-
-	if (this.assertCount === this.planCount) {
-		globals.setImmediate(this.exit.bind(this));
-	}
 };
 
 // patch assert methods to increase assert count and store errors
@@ -133,14 +134,23 @@ Test.prototype.run = function () {
 		this.exit();
 	}
 
-	ret = observableToPromise(ret);
+	var asyncType = 'promises';
+
+	if (isObservable(ret)) {
+		asyncType = 'observables';
+		ret = observableToPromise(ret);
+	}
 
 	if (isPromise(ret)) {
+		if (this.metadata.async) {
+			self._setAssertError(new Error('Do not return ' + asyncType + ' from tests declared via `test.async(...)`, if you want to return a promise simply declare the test via `test(...)`'));
+			this.exit();
+			return this.promise.promise;
+		}
+
 		ret
 			.then(function () {
-				if (!self.planCount || self.planCount === self.assertCount) {
-					self.exit();
-				}
+				self.exit();
 			})
 			.catch(function (err) {
 				self._setAssertError(new assert.AssertionError({
@@ -151,12 +161,23 @@ Test.prototype.run = function () {
 
 				self.exit();
 			});
+	} else if (!this.metadata.async) {
+		this.exit();
 	}
 
 	return this.promise.promise;
 };
 
-Test.prototype.end = function (err) {
+Object.defineProperty(Test.prototype, 'end', {
+	get: function () {
+		if (this.metadata.async) {
+			return this._end;
+		}
+		throw new Error('t.end is not supported in this context. To use t.end as a callback, you must explicitly declare the test asynchronous via `test.async(testName, fn)` ');
+	}
+});
+
+Test.prototype._end = function (err) {
 	if (err) {
 		this._setAssertError(new assert.AssertionError({
 			actual: err,

package.json

@@ -93,6 +93,7 @@
     "has-flag": "^1.0.0",
     "has-generator": "^1.0.0",
     "is-generator-fn": "^1.0.0",
+    "is-observable": "^0.1.0",
     "is-promise": "^2.1.0",
     "loud-rejection": "^1.2.0",
     "max-timeout": "^1.0.0",

readme.md

@@ -67,19 +67,18 @@ Install AVA globally `$ npm install --global ava` and run `$ ava --init` (with a
 
 ```js
 import test from 'ava';
+import delay from 'delay';
 
 test('foo', t => {
 	t.pass();
-	t.end();
 });
 
-test('bar', t => {
-	t.plan(2);
+const bar = Promise.resolve('bar').then(delay(200));
 
-	setTimeout(() => {
-		t.is('bar', 'bar');
-		t.same(['a', 'b'], ['a', 'b']);
-	}, 100);
+test('bar', async t => {
+	t.plan(2);
+	
+  t.is(await bar, 'bar');
 });
 ```
 
@@ -121,9 +120,13 @@ Files starting with `_` are ignored. This can be useful for having helpers in th
 
 ## Documentation
 
-Tests are run asynchronously and require you to either set planned assertions `t.plan(1)`, explicitly end the test when done `t.end()`, or return a promise. [Async functions](#async-function-support) already returns a promise implicitly, so no need for you to explicitly return a promise in that case.
+Tests are run asynchronously and require you to return a supported async object (a promise, or [observable](https://github.com/zenparsing/zen-observable)). We *highly* recommend the use of [async functions](#async-function-support); They make async code concise and readable, and they implicitly return a promise, so you don't need to. 
 
-You have to define all tests synchronously, meaning you can't define a test in the next tick, e.g. inside a `setTimeout`.
+If you do not return one of the supported async objects mentioned above, the test is considered to be synchronous and ended immediately. 
+
+If you are unable to use promises or other supported async objects, you may enable legacy async support by defining your test with `test.async([title', fn)`. Tests declared this way **must** be manually ended with `t.end()`.
+
+You must define all tests synchronously. They can't be defined inside `setTimeout`, `setImmediate`, etc.
 
 Test files are run from their current directory, so [`process.cwd()`](https://nodejs.org/api/process.html#process_process_cwd) is always the same as [`__dirname`](https://nodejs.org/api/globals.html#globals_dirname). You can just use relative paths instead of doing `path.join(__dirname, 'relative/path')`.
 
@@ -134,7 +137,6 @@ To create a test, you call the `test` function you `require`d from AVA and pass
 ```js
 test('name', t => {
 	t.pass();
-	t.end();
 });
 ```
 
@@ -144,7 +146,7 @@ Naming a test is optional, but you're recommended to use one if you have more th
 
 ```js
 test(t => {
-	t.end();
+	t.pass();
 });
 ```
 
@@ -152,13 +154,15 @@ You can also choose to use a named function instead:
 
 ```js
 test(function name(t) {
-	t.end();
+	t.pass();
 });
 ```
 
-### Planned assertions
+### Assertion plan
 
-Planned assertions are useful for being able to assert that all async actions happened. It also comes with the benefit of not having to manually end the test.
+An assertion plan can be used to ensure a specific number of assertions are made.
+ In the most common scenario, it validates that the test did not exit before executing the expected number of assertions.
+ It also fails the test if too many assertions are executed (Useful if you have assertions inside callbacks or loops). 
 
 This will result in a passed test:
 
@@ -166,19 +170,62 @@ This will result in a passed test:
 test(t => {
 	t.plan(1);
 
+	return Promise.resolve(3).then(n => {
+		t.is(n, 3);
+	});
+});
+
+test.async(t => {
 	setTimeout(() => {
 		t.pass();
+		t.end();
 	}, 100);
 });
 ```
 
+#### WARNING: Recent breaking change.
+
+AVA no longer supports automatically ending tests via `t.plan(...)`. 
+ This helps prevent false positives if you add assertions, but forget to increase your plan count.
+
+```js
+// This no longer works
+
+test('auto ending is dangerous', t => {
+	t.plan(2);
+	
+	t.pass();
+	t.pass();
+	
+	// auto-ending after reaching the planned two assertions will miss this final one
+	setTimeout(() => t.fail(), 10000);
+});
+```
+
+For this to work, you now must use the legacy `async` test mode, and explicitly call `t.end()`.
+  
+```js
+test('explicitly end your tests', t => {
+	t.plan(2);
+	
+	t.pass();
+	t.pass();
+	
+	setTimeout(() => {
+		// This failure is now reliably caught.
+		t.fail();
+		t.end();
+	}, 1000);
+});
+```
+
 ### Serial-tests
 
 While concurrency is awesome, there are some things that can't be done concurrently. In these rare cases, you can call `test.serial`, which will force those tests to run serially before the concurrent ones.
 
 ```js
 test.serial(t => {
-	t.end();
+	t.pass();
 });
 ```
 
@@ -189,12 +236,10 @@ Only-tests enforces only those tests to be run. This can be useful for running o
 ```js
 test('will not be run', t => {
 	t.fail();
-	t.end();
 })
 
 test.only('will be run', t => {
 	t.pass();
-	t.end();
 });
 ```
 
@@ -203,8 +248,8 @@ test.only('will be run', t => {
 Skip-tests are shown in the output as skipped but never run.
 
 ```js
-test.skip('unicorn', t => {
-	t.end();
+test.skip('will not be run', t => {
+  t.fail();
 });
 ```
 
@@ -217,32 +262,42 @@ used in the same manner as `test()`. The test function given to `test.before()`
 ```js
 test.before(t => {
 	// this runs before all tests
-	t.end();
 });
 
 test.before(t => {
 	// this runs after the above, but before tests
-	t.end();
 });
 
 test.after('cleanup', t => {
 	// this runs after all tests
-	t.end();
 });
 
 test.beforeEach(t => {
 	// this runs before each test
-	t.end();
 });
 
 test.afterEach(t => {
 	// this runs after each test
-	t.end();
 });
 
 test(t => {
 	// regular test
-	t.end();
+});
+```
+
+Both modern and legacy async support are available for hooks
+
+```js
+test.before(async t => {
+  await promiseFn();
+});
+
+test.async.beforeEach(t => {
+  setTimeout(t.end);
+});
+
+test.afterEach.async(t => {
+  setTimeout(t.end);
 });
 ```
 
@@ -251,12 +306,10 @@ The `beforeEach` & `afterEach` hooks can share context with the test:
 ```js
 test.beforeEach(t => {
 	t.context.data = generateUniqueData();
-	t.end();
 });
 
 test(t => {
 	t.is(t.context.data + 'bar', 'foobar');
-	t.end();
 });
 ```
 
@@ -265,12 +318,10 @@ The context is by default an object, but it can also be directly assigned:
 ```js
 test.beforeEach(t => {
 	t.context = 'unicorn';
-	t.end();
 });
 
 test(t => {
 	t.is(t.context, 'unicorn');
-	t.end();
 });
 ```
 
@@ -296,7 +347,6 @@ import assert from 'assert';
 
 test(t => {
 	assert(true);
-	t.end();
 });
 ```
 
@@ -309,7 +359,6 @@ Just write your tests in ES2015. No extra setup needed.
 ```js
 test(t => {
 	t.pass();
-	t.end();
 });
 ```
 
@@ -337,7 +386,6 @@ import foo from './foo'; // <-- foo can be written in ES2015!
 
 test('foo bar', t => {
 	t.same('baz', foo('bar'));
-	t.end();
 });
 ```
 
@@ -385,31 +433,32 @@ test(async t => {
 });
 ```
 
-*You don't have to manually call `t.end()`.*
-
 ### Observable support
 
 AVA comes with builtin support for [observables](https://github.com/zenparsing/es-observable).
+If you return an observable from a test, AVA will automatically consume it to completion before ending the test.
+*You don't have to use legacy `test.async` mode or `t.end()`.*
 
 ```js
 test(t => {
-	return Observable.of(1, 2, 3).map(n => {
-		t.true(n > 0);
-		return n * n;
-	});
+	t.plan(3);
+	return Observable.of(1, 2, 3, 4, 5, 6)
+		.filter(n => {
+			// only even numbers
+			return n % 2 === 0;
+		})
+		.map(() => t.pass());
 });
 ```
 
-*You don't have to manually call `t.end()`.*
-
 ### Callback support
 
 AVA supports using `t.end` as the final callback when using node-style
 error-first callback APIs. AVA will consider any truthy value passed as
-the first argument to `t.end` to be an error.
+the first argument to `t.end` to be an error. Note that `t.end` requires legacy `test.async` mode. 
 
 ```js
-test(t => {
+test.async(t => {
 	// t.end automatically checks for error as first argument
 	fs.readFile('data.txt', t.end);
 });
@@ -461,7 +510,6 @@ Assertions are mixed into the test [context](#context):
 ```js
 test(t => {
 	t.ok('unicorn'); // assertion
-	t.end();
 });
 ```
 
@@ -536,7 +584,6 @@ The following test:
 test(t => {
 	const x = 'foo';
 	t.ok(x === 'bar');
-	t.end();
 });
 ```
 
@@ -564,7 +611,6 @@ test(t => {
 	const b = 'bar';
 	const c = 'baz';
 	t.ok(a.test(b) || b === c);
-	t.end();
 });
 ```
 

test/fixture/circular-reference-on-assertion.js

@@ -4,5 +4,4 @@ test(t => {
 	var circular = ['a', 'b'];
 	circular.push(circular);
 	t.same([circular, 'c'], [circular, 'd']);
-	t.end();
 });

test/fixture/es2015.js

@@ -2,5 +2,4 @@ import test from '../../';
 
 test(t => {
 	t.pass();
-	t.end();
 });

test/fixture/fail-fast.js

@@ -2,10 +2,8 @@ import test from '../../';
 
 test(t => {
 	t.fail();
-	t.end();
 });
 
 test(t => {
 	t.pass();
-	t.end();
 });

test/fixture/hooks-failing.js

@@ -4,10 +4,8 @@ test.beforeEach(fail);
 test(pass);
 
 function pass(t) {
-	t.end();
 }
 
 function fail(t) {
 	t.fail();
-	t.end();
 }

test/fixture/hooks-passing.js

@@ -7,5 +7,4 @@ test.afterEach(pass);
 test(pass);
 
 function pass(t) {
-	t.end();
 }

test/fixture/long-running.js

@@ -2,7 +2,7 @@
 const test = require('../../');
 var onExit = require('signal-exit');
 
-test('long running', function (t) {
+test.async('long running', function (t) {
 	t.plan(1);
 
 	onExit(function () {
@@ -20,6 +20,7 @@ test('long running', function (t) {
 
 	setTimeout(function () {
 		t.ok(true);
+		t.end();
 	});
 
 	setTimeout(function () {

test/fixture/loud-rejection.js

@@ -1,6 +1,6 @@
 const test = require('../../');
 
-test('creates an unhandled rejection', t => {
+test.async('creates an unhandled rejection', t => {
 	Promise.reject(new Error(`You can't handle this!`));
 
 	setTimeout(function () {

test/fixture/one-pass-one-fail.js

@@ -2,10 +2,8 @@ const test = require('../../');
 
 test('this is a passing test', t => {
 	t.ok(true);
-	t.end();
 });
 
 test('this is a failing test', t => {
 	t.ok(false);
-	t.end();
 });

test/fixture/power-assert.js

@@ -4,5 +4,4 @@ test(t => {
 	const a = 'foo';
 
 	t.ok(a === 'bar');
-	t.end();
 });

test/fixture/process-cwd.js

@@ -3,5 +3,4 @@ const test = require('../../');
 
 test(t => {
 	t.is(process.cwd(), __dirname);
-	t.end();
 });

test/fixture/serial.js

@@ -6,14 +6,14 @@ function randomDelay() {
 
 const tests = [];
 
-test('first', t => {
+test.async('first', t => {
 	setTimeout(() => {
 		tests.push('first');
 		t.end();
 	}, randomDelay());
 });
 
-test('second', t => {
+test.async('second', t => {
 	setTimeout(() => {
 		tests.push('second');
 		t.end();
@@ -22,5 +22,4 @@ test('second', t => {
 
 test(t => {
 	t.same(tests, ['first', 'second']);
-	t.end();
 });

test/hooks.js

@@ -10,14 +10,12 @@ test('before', function (t) {
 	var runner = new Runner();
 	var arr = [];
 
-	runner.before(function (a) {
+	runner.before(function () {
 		arr.push('a');
-		a.end();
 	});
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr.push('b');
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -26,22 +24,22 @@ test('before', function (t) {
 });
 
 test('after', function (t) {
-	t.plan(1);
+	t.plan(3);
 
 	var runner = new Runner();
 	var arr = [];
 
-	runner.after(function (a) {
+	runner.after(function () {
 		arr.push('b');
-		a.end();
 	});
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr.push('a');
-		a.end();
 	});
 
 	runner.run().then(function () {
+		t.is(runner.stats.passCount, 1);
+		t.is(runner.stats.failCount, 0);
 		t.same(arr, ['a', 'b']);
 		t.end();
 	});
@@ -53,9 +51,8 @@ test('stop if before hooks failed', function (t) {
 	var runner = new Runner();
 	var arr = [];
 
-	runner.before(function (a) {
+	runner.before(function () {
 		arr.push('a');
-		a.end();
 	});
 
 	runner.before(function () {
@@ -81,24 +78,20 @@ test('before each with concurrent tests', function (t) {
 	var i = 0;
 	var k = 0;
 
-	runner.beforeEach(function (a) {
+	runner.beforeEach(function () {
 		arr[i++].push('a');
-		a.end();
 	});
 
-	runner.beforeEach(function (a) {
+	runner.beforeEach(function () {
 		arr[k++].push('b');
-		a.end();
 	});
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr[0].push('c');
-		a.end();
 	});
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr[1].push('d');
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -113,24 +106,20 @@ test('before each with serial tests', function (t) {
 	var runner = new Runner();
 	var arr = [];
 
-	runner.beforeEach(function (a) {
+	runner.beforeEach(function () {
 		arr.push('a');
-		a.end();
 	});
 
-	runner.beforeEach(function (a) {
+	runner.beforeEach(function () {
 		arr.push('b');
-		a.end();
 	});
 
-	runner.serial(function (a) {
+	runner.serial(function () {
 		arr.push('c');
-		a.end();
 	});
 
-	runner.serial(function (a) {
+	runner.serial(function () {
 		arr.push('d');
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -148,13 +137,11 @@ test('fail if beforeEach hook fails', function (t) {
 	runner.beforeEach(function (a) {
 		arr.push('a');
 		a.fail();
-		a.end();
 	});
 
 	runner.test(function (a) {
 		arr.push('b');
 		a.pass();
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -172,24 +159,20 @@ test('after each with concurrent tests', function (t) {
 	var i = 0;
 	var k = 0;
 
-	runner.afterEach(function (a) {
+	runner.afterEach(function () {
 		arr[i++].push('a');
-		a.end();
 	});
 
-	runner.afterEach(function (a) {
+	runner.afterEach(function () {
 		arr[k++].push('b');
-		a.end();
 	});
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr[0].push('c');
-		a.end();
 	});
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr[1].push('d');
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -204,24 +187,20 @@ test('after each with serial tests', function (t) {
 	var runner = new Runner();
 	var arr = [];
 
-	runner.afterEach(function (a) {
+	runner.afterEach(function () {
 		arr.push('a');
-		a.end();
 	});
 
-	runner.afterEach(function (a) {
+	runner.afterEach(function () {
 		arr.push('b');
-		a.end();
 	});
 
-	runner.serial(function (a) {
+	runner.serial(function () {
 		arr.push('c');
-		a.end();
 	});
 
-	runner.serial(function (a) {
+	runner.serial(function () {
 		arr.push('d');
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -236,29 +215,24 @@ test('ensure hooks run only around tests', function (t) {
 	var runner = new Runner();
 	var arr = [];
 
-	runner.beforeEach(function (a) {
+	runner.beforeEach(function () {
 		arr.push('beforeEach');
-		a.end();
 	});
 
-	runner.before(function (a) {
+	runner.before(function () {
 		arr.push('before');
-		a.end();
 	});
 
-	runner.afterEach(function (a) {
+	runner.afterEach(function () {
 		arr.push('afterEach');
-		a.end();
 	});
 
-	runner.after(function (a) {
+	runner.after(function () {
 		arr.push('after');
-		a.end();
 	});
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr.push('test');
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -275,29 +249,24 @@ test('shared context', function (t) {
 	runner.before(function (a) {
 		a.is(a.context, undefined);
 		a.context = {arr: []};
-		a.end();
 	});
 
 	runner.after(function (a) {
 		a.is(a.context, undefined);
-		a.end();
 	});
 
 	runner.beforeEach(function (a) {
 		a.context.arr = ['a'];
-		a.end();
 	});
 
 	runner.test(function (a) {
 		a.context.arr.push('b');
 		a.same(a.context.arr, ['a', 'b']);
-		a.end();
 	});
 
 	runner.afterEach(function (a) {
 		a.context.arr.push('c');
 		a.same(a.context.arr, ['a', 'b', 'c']);
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -313,12 +282,10 @@ test('shared context of any type', function (t) {
 
 	runner.beforeEach(function (a) {
 		a.context = 'foo';
-		a.end();
 	});
 
 	runner.test(function (a) {
 		a.is(a.context, 'foo');
-		a.end();
 	});
 
 	runner.run().then(function () {

test/observable.js

@@ -1,10 +1,22 @@
 'use strict';
 var test = require('tap').test;
-var ava = require('../lib/test');
+var _ava = require('../lib/test');
 var Observable = require('./fixture/observable');
 
-test('plan assertions', function (t) {
-	ava(function (a) {
+function ava(fn) {
+	var a = _ava(fn);
+	a.metadata = {async: false};
+	return a;
+}
+
+ava.async = function (fn) {
+	var a = _ava(fn);
+	a.metadata = {async: true};
+	return a;
+};
+
+test('returning an observable from a legacy async fn is an error', function (t) {
+	ava.async(function (a) {
 		a.plan(2);
 
 		var observable = Observable.of();
@@ -12,12 +24,12 @@ test('plan assertions', function (t) {
 		setTimeout(function () {
 			a.pass();
 			a.pass();
+			a.end();
 		}, 200);
 
 		return observable;
-	}).run().then(function (a) {
-		t.is(a.planCount, 2);
-		t.is(a.assertCount, 2);
+	}).run().catch(function (err) {
+		t.match(err.message, /Do not return observables/);
 		t.end();
 	});
 });
@@ -30,7 +42,7 @@ test('handle throws with thrown observable', function (t) {
 			observer.error(new Error());
 		});
 
-		a.throws(observable);
+		return a.throws(observable);
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -47,7 +59,7 @@ test('handle throws with long running thrown observable', function (t) {
 			}, 2000);
 		});
 
-		a.throws(observable, /abc/);
+		return a.throws(observable, /abc/);
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -59,7 +71,7 @@ test('handle throws with completed observable', function (t) {
 		a.plan(1);
 
 		var observable = Observable.of();
-		a.throws(observable);
+		return a.throws(observable);
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.is(err.name, 'AssertionError');
@@ -75,7 +87,7 @@ test('handle throws with regex', function (t) {
 			observer.error(new Error('abc'));
 		});
 
-		a.throws(observable, /abc/);
+		return a.throws(observable, /abc/);
 	}).run().then(function (a) {
 		t.notOk(a.assertionError);
 		t.end();
@@ -90,7 +102,7 @@ test('handle throws with string', function (t) {
 			observer.error(new Error('abc'));
 		});
 
-		a.throws(observable, 'abc');
+		return a.throws(observable, 'abc');
 	}).run().then(function (a) {
 		t.notOk(a.assertionError);
 		t.end();
@@ -106,7 +118,7 @@ test('handle throws with false-positive observable', function (t) {
 			observer.complete();
 		});
 
-		a.throws(observable);
+		return a.throws(observable);
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.is(err.name, 'AssertionError');
@@ -119,7 +131,7 @@ test('handle doesNotThrow with completed observable', function (t) {
 		a.plan(1);
 
 		var observable = Observable.of();
-		a.doesNotThrow(observable);
+		return a.doesNotThrow(observable);
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -134,7 +146,7 @@ test('handle doesNotThrow with thrown observable', function (t) {
 			observer.error(new Error());
 		});
 
-		a.doesNotThrow(observable);
+		return a.doesNotThrow(observable);
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.is(err.name, 'AssertionError');

test/promise.js

@@ -1,7 +1,19 @@
 'use strict';
 var Promise = require('bluebird');
 var test = require('tap').test;
-var ava = require('../lib/test');
+var _ava = require('../lib/test');
+
+function ava(fn) {
+	var a = _ava(fn);
+	a.metadata = {async: false};
+	return a;
+}
+
+ava.async = function (fn) {
+	var a = _ava(fn);
+	a.metadata = {async: true};
+	return a;
+};
 
 function pass() {
 	return new Promise(function (resolve) {
@@ -17,21 +29,84 @@ function fail() {
 	});
 }
 
-test('plan assertions', function (t) {
+test('returning a promise from a legacy async fn is an error', function (t) {
+	ava.async(function (a) {
+		a.plan(1);
+
+		return Promise.resolve(true).then(function () {
+			a.pass();
+			a.end();
+		});
+	}).run().catch(function (err) {
+		t.match(err.message, /Do not return promises/);
+		t.end();
+	});
+});
+
+test('assertion plan is tested after returned promise resolves', function (t) {
+	var start = Date.now();
 	ava(function (a) {
 		a.plan(2);
 
-		var promise = Promise.resolve();
+		var defer = Promise.defer();
 
 		setTimeout(function () {
-			a.pass();
-			a.pass();
-		}, 200);
+			defer.resolve();
+		}, 500);
 
-		return promise;
+		a.pass();
+		a.pass();
+
+		return defer.promise;
 	}).run().then(function (a) {
 		t.is(a.planCount, 2);
 		t.is(a.assertCount, 2);
+		t.true(Date.now() - start > 500);
+		t.end();
+	});
+});
+
+test('missing assertion will fail the test', function (t) {
+	ava(function (a) {
+		a.plan(2);
+
+		var defer = Promise.defer();
+
+		setTimeout(function () {
+			a.pass();
+			defer.resolve();
+		}, 200);
+
+		return defer.promise;
+	}).run().catch(function (err) {
+		t.ok(err);
+		t.is(err.expected, 2);
+		t.is(err.actual, 1);
+		t.end();
+	});
+});
+
+test('extra assertion will fail the test', function (t) {
+	ava(function (a) {
+		a.plan(2);
+
+		var defer = Promise.defer();
+
+		setTimeout(function () {
+			a.pass();
+			a.pass();
+		}, 200);
+
+		setTimeout(function () {
+			a.pass();
+			defer.resolve();
+		}, 500);
+
+		return defer.promise;
+	}).run().catch(function (err) {
+		t.ok(err);
+		t.is(err.expected, 2);
+		t.is(err.actual, 3);
 		t.end();
 	});
 });
@@ -41,7 +116,7 @@ test('handle throws with rejected promise', function (t) {
 		a.plan(1);
 
 		var promise = Promise.reject(new Error());
-		a.throws(promise);
+		return a.throws(promise);
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -58,7 +133,7 @@ test('handle throws with long running rejected promise', function (t) {
 			}, 2000);
 		});
 
-		a.throws(promise, /abc/);
+		return a.throws(promise, /abc/);
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -70,7 +145,7 @@ test('handle throws with resolved promise', function (t) {
 		a.plan(1);
 
 		var promise = Promise.resolve();
-		a.throws(promise);
+		return a.throws(promise);
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.is(err.name, 'AssertionError');
@@ -83,7 +158,7 @@ test('handle throws with regex', function (t) {
 		a.plan(1);
 
 		var promise = Promise.reject(new Error('abc'));
-		a.throws(promise, /abc/);
+		return a.throws(promise, /abc/);
 	}).run().then(function (a) {
 		t.notOk(a.assertionError);
 		t.end();
@@ -95,7 +170,7 @@ test('handle throws with string', function (t) {
 		a.plan(1);
 
 		var promise = Promise.reject(new Error('abc'));
-		a.throws(promise, 'abc');
+		return a.throws(promise, 'abc');
 	}).run().then(function (a) {
 		t.notOk(a.assertionError);
 		t.end();
@@ -107,7 +182,7 @@ test('handle throws with false-positive promise', function (t) {
 		a.plan(1);
 
 		var promise = Promise.resolve(new Error());
-		a.throws(promise);
+		return a.throws(promise);
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.is(err.name, 'AssertionError');
@@ -120,7 +195,7 @@ test('handle doesNotThrow with resolved promise', function (t) {
 		a.plan(1);
 
 		var promise = Promise.resolve();
-		a.doesNotThrow(promise);
+		return a.doesNotThrow(promise);
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -132,7 +207,7 @@ test('handle doesNotThrow with rejected promise', function (t) {
 		a.plan(1);
 
 		var promise = Promise.reject(new Error());
-		a.doesNotThrow(promise);
+		return a.doesNotThrow(promise);
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.is(err.name, 'AssertionError');

test/runner.js

@@ -136,7 +136,7 @@ test('runner emits a "test" event', function (t) {
 	var runner = new Runner();
 
 	runner.test(function foo(a) {
-		a.end();
+		a.pass();
 	});
 
 	runner.on('test', function (props) {
@@ -179,9 +179,8 @@ test('anything can be skipped', function (t) {
 	var arr = [];
 
 	function pusher(title) {
-		return function (a) {
+		return function () {
 			arr.push(title);
-			a.end();
 		};
 	}
 
@@ -230,7 +229,7 @@ test('test types and titles', function (t) {
 	runner.test('test', pass);
 
 	function pass(a) {
-		a.end();
+		a.pass();
 	}
 
 	var tests = [
@@ -257,14 +256,12 @@ test('skip test', function (t) {
 	var runner = new Runner();
 	var arr = [];
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr.push('a');
-		a.end();
 	});
 
-	runner.skip(function (a) {
+	runner.skip(function () {
 		arr.push('b');
-		a.end();
 	});
 
 	runner.run().then(function () {
@@ -281,14 +278,12 @@ test('only test', function (t) {
 	var runner = new Runner();
 	var arr = [];
 
-	runner.test(function (a) {
+	runner.test(function () {
 		arr.push('a');
-		a.end();
 	});
 
-	runner.only(function (a) {
+	runner.only(function () {
 		arr.push('b');
-		a.end();
 	});
 
 	runner.run().then(function () {

test/test.js

@@ -1,6 +1,18 @@
 'use strict';
 var test = require('tap').test;
-var ava = require('../lib/test');
+var _ava = require('../lib/test');
+
+function ava() {
+	var t = _ava.apply(null, arguments);
+	t.metadata = {async: false};
+	return t;
+}
+
+ava.async = function () {
+	var t = _ava.apply(null, arguments);
+	t.metadata = {async: true};
+	return t;
+};
 
 test('run test', function (t) {
 	ava('foo', function (a) {
@@ -14,7 +26,7 @@ test('run test', function (t) {
 
 test('title is optional', function (t) {
 	ava(function (a) {
-		a.end();
+		a.pass();
 	}).run().then(function (a) {
 		t.is(a.title, '[anonymous]');
 		t.end();
@@ -35,7 +47,7 @@ test('callback is required', function (t) {
 
 test('infer name from function', function (t) {
 	ava(function foo(a) {
-		a.end();
+		a.pass();
 	}).run().then(function (a) {
 		t.is(a.title, 'foo');
 		t.end();
@@ -47,7 +59,6 @@ test('multiple asserts', function (t) {
 		a.pass();
 		a.pass();
 		a.pass();
-		a.end();
 	}).run().then(function (a) {
 		t.is(a.assertCount, 3);
 		t.end();
@@ -90,7 +101,7 @@ test('handle non-assertion errors', function (t) {
 });
 
 test('end can be used as callback without maintaining thisArg', function (t) {
-	ava(function (a) {
+	ava.async(function (a) {
 		setTimeout(a.end);
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
@@ -121,7 +132,6 @@ test('handle non-assertion errors even when planned', function (t) {
 test('handle testing of arrays', function (t) {
 	ava(function (a) {
 		a.same(['foo', 'bar'], ['foo', 'bar']);
-		a.end();
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -131,7 +141,6 @@ test('handle testing of arrays', function (t) {
 test('handle falsy testing of arrays', function (t) {
 	ava(function (a) {
 		a.notSame(['foo', 'bar'], ['foo', 'bar', 'cat']);
-		a.end();
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -141,7 +150,6 @@ test('handle falsy testing of arrays', function (t) {
 test('handle testing of objects', function (t) {
 	ava(function (a) {
 		a.same({foo: 'foo', bar: 'bar'}, {foo: 'foo', bar: 'bar'});
-		a.end();
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -151,7 +159,6 @@ test('handle testing of objects', function (t) {
 test('handle falsy testing of objects', function (t) {
 	ava(function (a) {
 		a.notSame({foo: 'foo', bar: 'bar'}, {foo: 'foo', bar: 'bar', cat: 'cake'});
-		a.end();
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -163,8 +170,6 @@ test('handle throws with error', function (t) {
 		a.throws(function () {
 			throw new Error('foo');
 		});
-
-		a.end();
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -176,8 +181,6 @@ test('handle throws without error', function (t) {
 		a.throws(function () {
 			return;
 		});
-
-		a.end();
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.end();
@@ -189,8 +192,6 @@ test('handle doesNotThrow with error', function (t) {
 		a.doesNotThrow(function () {
 			throw new Error('foo');
 		});
-
-		a.end();
 	}).run().catch(function (err) {
 		t.ok(err);
 		t.is(err.name, 'AssertionError');
@@ -203,8 +204,6 @@ test('handle doesNotThrow without error', function (t) {
 		a.doesNotThrow(function () {
 			return;
 		});
-
-		a.end();
 	}).run().then(function (a) {
 		t.notOk(a.assertError);
 		t.end();
@@ -227,17 +226,11 @@ test('run functions after last planned assertion', function (t) {
 test('run async functions after last planned assertion', function (t) {
 	var i = 0;
 
-	ava(function (a) {
+	ava.async(function (a) {
 		a.plan(1);
-
-		function foo(cb) {
-			a.pass();
-			cb();
-		}
-
-		foo(function () {
-			i++;
-		});
+		a.pass();
+		a.end();
+		i++;
 	}).run().then(function () {
 		t.is(i, 1);
 		t.end();
@@ -245,11 +238,12 @@ test('run async functions after last planned assertion', function (t) {
 });
 
 test('planned async assertion', function (t) {
-	ava(function (a) {
+	ava.async(function (a) {
 		a.plan(1);
 
 		setTimeout(function () {
 			a.pass();
+			a.end();
 		}, 100);
 	}).run().then(function (a) {
 		t.ifError(a.assertError);
@@ -258,7 +252,7 @@ test('planned async assertion', function (t) {
 });
 
 test('async assertion with `.end()`', function (t) {
-	ava(function (a) {
+	ava.async(function (a) {
 		setTimeout(function () {
 			a.pass();
 			a.end();
@@ -282,11 +276,12 @@ test('more assertions than planned should emit an assertion error', function (t)
 });
 
 test('record test duration', function (t) {
-	ava(function (a) {
+	ava.async(function (a) {
 		a.plan(1);
 
 		setTimeout(function () {
 			a.true(true);
+			a.end();
 		}, 1234);
 	}).run().then(function (a) {
 		t.true(a.duration >= 1234);
@@ -297,7 +292,7 @@ test('record test duration', function (t) {
 test('wait for test to end', function (t) {
 	var avaTest;
 
-	ava(function (a) {
+	ava.async(function (a) {
 		a.plan(1);
 
 		avaTest = a;
@@ -310,6 +305,7 @@ test('wait for test to end', function (t) {
 
 	setTimeout(function () {
 		avaTest.pass();
+		avaTest.end();
 	}, 1234);
 });