Merge pull request #3399 from halemmerich/gpstests

Implements more asserts and reading testcases from json files

apps/android/test.json

@@ -0,0 +1,41 @@
+{
+  "app" : "android",
+  "setup" : [{
+    "id": "default",
+    "steps" : [
+      {"t":"cmd", "js": "Bangle.setGPSPower=(isOn, appID)=>{if (!appID) appID='?';if (!Bangle._PWR) Bangle._PWR={};if (!Bangle._PWR.GPS) Bangle._PWR.GPS=[];if (isOn && !Bangle._PWR.GPS.includes(appID)) Bangle._PWR.GPS.push(appID);if (!isOn && Bangle._PWR.GPS.includes(appID)) Bangle._PWR.GPS.splice(Bangle._PWR.GPS.indexOf(appID),1);return Bangle._PWR.GPS.length>0;};", "text": "Fake the setGPSPower"},
+      {"t":"wrap", "fn": "Bangle.setGPSPower", "id": "gpspower"},
+      {"t":"cmd", "js": "NRF.getSecurityStatus = () => { return { connected: false };}", "text": "Control the security status"},
+      {"t":"cmd", "js": "Serial1.println = () => { }", "text": "Fake the serial port println"},
+      {"t":"cmd", "js": "Bluetooth.println = () => { }", "text": "Fake the Bluetooth println"},
+      {"t":"cmd", "js": "Bangle._PWR={}", "text": "Prepare an empty _PWR for following asserts"},
+      {"t":"cmd", "js": "require('Storage').writeJSON('android.settings.json', {overwriteGps: true})", "text": "Enable GPS overwrite"},
+      {"t":"cmd", "js": "eval(require('Storage').read('android.boot.js'))", "text": "Load the boot code"}
+    ]
+  }],
+  "tests" : [{
+    "description": "Check setGPSPower is replaced", 
+    "steps" : [
+      {"t":"cmd", "js": "Serial1.println = () => { }", "text": "Fake the serial port"},
+      {"t":"cmd", "js": "Bluetooth.println = () => { }", "text": "Fake the Bluetooth println"},
+      {"t":"cmd", "js": "require('Storage').writeJSON('android.settings.json', {overwriteGps: true})", "text": "Enable GPS overwrite"},
+      {"t":"cmd", "js": "eval(require('Storage').read('android.boot.js'))", "text": "Load the boot code"},
+      {"t":"assert", "js": "Bangle.setGPSPower.toString().includes('native')", "is":"false", "text": "setGPSPower has been replaced"}
+    ]
+  },{
+    "description": "Test switching hardware GPS on and off", 
+    "steps" : [
+      {"t":"setup", "id": "default"},
+      {"t":"assertArray", "js": "Bangle._PWR.GPS", "is":"undefinedOrEmpty", "text": "No GPS clients"},
+      {"t":"assert", "js": "Bangle.isGPSOn()", "is":"falsy", "text": "isGPSOn shows GPS as off"},
+      {"t":"assert", "js": "Bangle.setGPSPower(1, 'test')", "is":"truthy", "text": "setGPSPower returns truthy when switching on"},
+      {"t":"assertArray", "js": "Bangle._PWR.GPS", "is":"notEmpty", "text": "GPS clients"},
+      {"t":"assert", "js": "Bangle.isGPSOn()", "is":"truthy", "text": "isGPSOn shows GPS as on"},
+      {"t":"assertCall", "id": "gpspower", "count": 1, "argAsserts": [ { "t": "assert", "arg": "0", "is": "equal", "to": 1 } ] , "text": "internal GPS switched on"},
+      {"t":"assert", "js": "Bangle.setGPSPower(0, 'test')", "is":"falsy", "text": "setGPSPower returns falsy when switching off"},
+      {"t":"assertArray", "js": "Bangle._PWR.GPS", "is":"undefinedOrEmpty", "text": "No GPS clients"},
+      {"t":"assert", "js": "Bangle.isGPSOn()", "is":"falsy", "text": "isGPSOn shows GPS as off"},
+      {"t":"assertCall", "id": "gpspower", "count": 2, "argAsserts": [ { "t": "assert", "arg": "0", "is": "equal", "to": 0 } ] , "text": "internal GPS switched off"}
+    ]
+  }]
+}

apps/antonclk/test.json

@@ -0,0 +1,15 @@
+{
+  "app" : "antonclk",
+  "tests" : [{
+    "description": "Check memory usage after setUI",
+    "steps" : [
+      {"t":"cmd", "js": "Bangle.loadWidgets()"},
+      {"t":"cmd", "js": "eval(require('Storage').read('antonclk.app.js'))"},
+      {"t":"cmd", "js": "Bangle.setUI()"},
+      {"t":"saveMemoryUsage"},
+      {"t":"cmd", "js": "eval(require('Storage').read('antonclk.app.js'))"},
+      {"t":"cmd", "js":"Bangle.setUI()"},
+      {"t":"checkMemoryUsage"}
+    ]
+  }]
+}

apps/messagesoverlay/test.json

@@ -0,0 +1,32 @@
+{
+  "app" : "messagesoverlay",
+  "tests" : [{
+    "description": "Test handler backgrounding", 
+    "steps" : [
+      {"t":"upload", "file": "modules/widget_utils.js", "as": "widget_utils"},
+      {"t":"cmd", "js": "Bangle.loadWidgets()", "text": "Load widgets"},
+      {"t":"assertArray", "js": "Bangle['#onswipe']", "is":"undefinedOrEmpty", "text": "No swipe handlers"},
+      {"t":"cmd", "js": "require('widget_utils').swipeOn(0)", "text": "Store widgets in overlay"},
+      {"t":"assert", "js": "Bangle['#onswipe']", "is":"function", "text": "One swipe handler for widgets"},
+      {"t":"emit", "event":"swipe", "paramsArray": [ 0, 1 ], "text": "Show widgets"},
+      {"t":"assert", "js": "Bangle['#onswipe']", "is":"function", "text": "One swipe handler for widgets"},
+      {"t":"cmd", "js": "require('messagesoverlay').message('text', {src:'Messenger',t:'add',type:'text',id:Date.now().toFixed(0),title:'title',body:'body'})", "text": "Show a message overlay"},
+      {"t":"assertArray", "js": "Bangle['#onswipe']", "is":"undefinedOrEmpty", "text": "No swipe handlers while message overlay is on screen"},
+      {"t":"emit", "event":"touch", "paramsArray": [ 1, { "x": 10, "y": 10, "type": 0 } ], "text": "Close message"},
+      {"t":"assert", "js": "Bangle['#onswipe']", "is":"function", "text": "One swipe handler restored"}
+    ]
+  },{
+    "description": "Test handler backgrounding with fastloading (setUI)", 
+    "steps" : [
+      {"t":"cmd", "js": "Bangle.on('swipe',print)", "text": "Create listener for swipes"},
+      {"t":"cmd", "js": "Bangle.setUI({mode: 'clock',remove: ()=>{}})", "text": "Init UI for clock"},
+      {"t":"cmd", "js": "require('messagesoverlay').message('text', {src:'Messenger',t:'add',type:'text',id:Date.now().toFixed(0),title:'title',body:'body'})", "text": "Show a message overlay"},
+      {"t":"assertArray", "js": "Bangle['#onswipe']", "is":"undefinedOrEmpty", "text": "No swipe handlers while message overlay is on screen"},
+      {"t":"cmd", "js": "Bangle.setUI()", "text": "Trigger removal of UI"},
+      {"t":"assertArray", "js": "Bangle['#onswipe']", "is":"undefinedOrEmpty", "text": "Still no swipe handlers"},
+      {"t":"cmd", "js": "Bangle.on('touch', print)"},
+      {"t":"emit", "event":"touch", "paramsArray": [ 1, { "x": 10, "y": 10, "type": 0 } ], "text": "Close message"},
+      {"t":"assert", "js": "Bangle['#onswipe']", "is":"function", "text": "One swipe handler restored"}
+    ]
+  }]
+}

bin/runapptests.js

@@ -8,6 +8,8 @@ IT IS UNFINISHED
 It searches for `test.json` in each app's directory and will
 run them in sequence.
 
+The return code is the number of failed tests.
+
 TODO:
 
 * more code to test with
@@ -20,32 +22,45 @@ TODO:
 
 */
 
-// A simpletest
+const DEMOAPP = {
-/*var TEST = {
+  "id":"demoappfortestformat",
-  app : "android",
+  "name":"demo",
-  tests : [ {
+  "version":"0.01",
-    steps : [
+  "type":"app",
-      {t:"load", fn:"messagesgui.app.js"},
+  "supports":["BANGLEJS2"],
-      {t:"gb", "obj":{"t":"notify","id":1234,"src":"Twitter","title":"A Name","body":"message contents"}},
+  "storage":[],
-      {t:"cmd", "js":"X='hello';"},
-      {t:"eval", "js":"X", eq:"hello"}
-    ]
-  }]
-};*/
-var TEST = {
-  app : "antonclk",
-  tests : [ {
-    steps : [
-      {t:"cmd", "js": "Bangle.loadWidgets()"},
-      {t:"cmd", "js": "eval(require('Storage').read('antonclk.app.js'))"},
-      {t:"cmd", "js":"Bangle.setUI()"}, // load and free
-      {t:"saveMemoryUsage"},
-      {t:"cmd", "js": "eval(require('Storage').read('antonclk.app.js'))"},
-      {t:"cmd", "js":"Bangle.setUI()"}, // load and free
-      {t:"checkMemoryUsage"}, // check memory usage is the same
-    ]
-  }]
 };
+const DEMOTEST = {
+  "app" : "demoappfortestformat",
+  "setup" : [{
+    "id": "arbitraryid",
+    "steps" : [
+      {"t":"cmd", "js": "global.testfunction = ()=>{}", "text": "Runs some code on the device"},
+      {"t":"wrap", "fn": "global.testfunction", "id": "testfunc", text:"Wraps a function to count calls and store the last set of arguments on the device"}
+    ]
+  }],
+  "tests" : [{
+    "description": "Optional description of the test, will be shown in results table",
+    "steps" : [
+      {"t":"setup", "id": "arbitraryid", "text": "Calls a set of predefined steps"},
+      {"t":"eval", "js": "'test' + 'value'", "eq": "testvalue", "text": "Evals code on the device and compares the resulting string to the value in 'eq'"},
+//      {"t":"console", "text": "Starts an interactive console for debugging"},
+      {"t":"saveMemoryUsage", "text": "Gets and stores the current memory usage"},
+      {"t":"checkMemoryUsage", "text": "Checks the current memory to be equal to the stored value"},
+      {"t":"assert", "js": "0", "is":"falsy", "text": "Evaluates the content of 'js' on the device and asserts if the result is falsy"},
+      {"t":"assert", "js": "1", "is":"truthy", "text": "Evaluates the content of 'js' on the device and asserts if the result is truthy"},
+      {"t":"assert", "js": "false", "is":"false", "text": "Evaluates the content of 'js' on the device and asserts if the result is false"},
+      {"t":"assert", "js": "true", "is":"true", "text": "Evaluates the content of 'js' on the device and asserts if the result is true"},
+      {"t":"assert", "js": "()=>{}", "is":"function", "text": "Evaluates the content of 'js' and on the device and asserts if the result is a function"},
+      {"t":"assert", "js": "123", "is":"equal", "to": "123", "text": "Evaluates the content of 'js' and 'to' on the device and asserts if the result is equal"},
+      {"t":"assertArray", "js": "[]", "is":"undefinedOrEmpty", "text": "Evaluates the content of 'js' on the device and asserts if the result is undefined or an empty array"},
+      {"t":"assertArray", "js": "[1,2,3]", "is":"notEmpty", "text": "Evaluates the content of 'js' on the device and asserts if the result is an array with more than 0 entries"},
+      {"t":"cmd", "js": "global.testfunction(1)", "text": "Call function for the following asserts"},
+      {"t":"assertCall", "id": "testfunc", "argAsserts": [ { "t": "assert", "arg": "0", "is": "equal", "to": 1 } ] , "text": "Asserts if a wrapped function has been called with the expected arguments"},
+      {"t":"assertCall", "id": "testfunc", "count": 1 , "text": "Asserts if a wrapped function has been called the expected number of times"}
+    ]
+  }]
+}
 
 var EMULATOR = "banglejs2";
 var DEVICEID = "BANGLEJS2";
@@ -62,6 +77,9 @@ if (!require("fs").existsSync(DIR_IDE)) {
   process.exit(1);
 }
 
+const verbose = process.argv.includes("--verbose") || process.argv.includes("-v");
+
+var AppInfo = require(BASE_DIR+"/core/js/appinfo.js");
 var apploader = require(BASE_DIR+"/core/lib/apploader.js");
 apploader.init({
   DEVICEID : DEVICEID
@@ -71,97 +89,408 @@ var emu = require(BASE_DIR+"/core/lib/emulator.js");
 // Last set of text received
 var lastTxt;
 
+function getSanitizedLastLine(){
+  return emu.getLastLine().replaceAll("\r", "");
+}
+
 function ERROR(s) {
   console.error(s);
   process.exit(1);
 }
 
-function runTest(test) {
+function getValue(js){
-  var app = apploader.apps.find(a=>a.id==test.app);
+  if(verbose)
-  if (!app) ERROR(`App ${JSON.stringify(test.app)} not found`);
+    console.log(`> GETTING VALUE FOR \`${js}\``);
-  if (app.custom) ERROR(`App ${JSON.stringify(appId)} requires HTML customisation`);
+  emu.tx(`\x10print(JSON.stringify(${js}))\n`);
-  return apploader.getAppFilesString(app).then(command => {
+  var result = getSanitizedLastLine();
-    // What about dependencies??
+  
-    test.tests.forEach((subtest,subtestIdx) => {
+  if (verbose)
-      console.log(`==============================`);
+    console.log(`  GOT \`${result}\``);
-      console.log(`"${test.app}" Test ${subtestIdx}`);
+  return JSON.parse(result);
-      console.log(`==============================`);
+}
-      emu.factoryReset();
+
-      console.log("> Sending app "+test.app);
+function assertArray(step){
-      emu.tx(command);
+  console.log(`> ASSERT ARRAY ${step.js} IS`,step.is.toUpperCase(), step.text ? "- " + step.text : "");
-      console.log("> Sent app");
+  let isOK;
-      emu.tx("reset()\n");
+  switch (step.is.toLowerCase()){
-      console.log("> Reset.");
+    case "notempty": isOK = getValue(`${step.js} && ${step.js}.length > 0`); break;
-      var ok = true;
+    case "undefinedorempty": isOK = getValue(`!${step.js} || (${step.js} && ${step.js}.length === 0)`); break;
-      subtest.steps.forEach(step => {
+  }
-        if (ok) switch(step.t) {
+
+  if (isOK) {
+    if (verbose)
+      console.log("> OK -", `\`${step.js}\``);
+  } else
+    console.log("> FAIL -", `\`${step.js}\``);
+  return isOK;
+}
+
+function assertValue(step){
+  console.log("> ASSERT " + `\`${step.js}\``, "IS", step.is.toUpperCase(), step.to ? "TO " + `\`${step.js}\`` : "", step.text ? "- " + step.text : "");
+  let isOK;
+  switch (step.is.toLowerCase()){
+    case "truthy": isOK = getValue(`!!${step.js}`); break;
+    case "falsy": isOK = getValue(`!${step.js}`); break;
+    case "true": isOK = getValue(`${step.js} === true`); break;
+    case "false": isOK = getValue(`${step.js} === false`); break;
+    case "equal": isOK = getValue(`${step.js} == ${step.to}`); break;
+    case "function": isOK = getValue(`typeof ${step.js} === "function"`); break;
+  }
+
+  if (isOK){
+    if (verbose)
+      console.log("> OK - " + `\`${step.js}\``, "IS", step.is.toUpperCase(), step.to ? "TO " + `\`${step.js}\`` : "");
+  } else
+    console.log("> FAIL - " + `\`${step.js}\``, "IS", step.is.toUpperCase(), step.to ? "TO " + `\`${step.js}\`` : "");
+  return isOK;
+}
+
+function wrap(func, id){
+  console.log(`> WRAPPING \`${func}\` AS ${id}`);
+
+  let wrappingCode = `
+  if(!global.APPTESTS) global.APPTESTS={};
+  if(!global.APPTESTS.funcCalls) global.APPTESTS.funcCalls={};
+  if(!global.APPTESTS.funcArgs) global.APPTESTS.funcArgs={};
+  global.APPTESTS.funcCalls.${id}=0;
+  (function(o) {
+    ${func} = function() {
+      global.APPTESTS.funcCalls.${id}++;
+      global.APPTESTS.funcArgs.${id}=arguments;
+      return o.apply(this, arguments);
+    };
+  }(${func}));\n`;
+
+  emu.tx(wrappingCode);
+}
+
+function assertCall(step){
+  console.log("> ASSERT CALL", step.id, step.text ? "- " + step.text : "");
+  let isOK = true;
+  let id = step.id;
+  let args = step.argAsserts;
+  if (step.count !== undefined){
+    let calls = getValue(`global.APPTESTS.funcCalls.${id}`);
+    isOK = step.count == calls
+  }
+  if (args && args.length > 0){
+    let callArgs = getValue(`global.APPTESTS.funcArgs.${id}`);
+    for (let a of args){
+      let current = {
+        js: callArgs[a.arg],
+        is: a.is,
+        to: a.to,
+        text: step.text
+      };
+      switch(a.t){
+        case "assertArray":
+          isOK = isOK && assertArray(current);
+          break;
+        case "assert":
+          isOK = isOK && assertValue(current);
+          break;
+      }
+    }
+  }
+  if (isOK){
+    if (verbose)
+      console.log("OK - ASSERT CALL", step.text ? "- " + step.text : "");
+  } else
+    console.log("FAIL - ASSERT CALL", step.text ? "- " + step.text : "");
+  return isOK;
+}
+
+function runStep(step, subtest, test, state){
+  let p = Promise.resolve();
+  if (state.ok) switch(step.t) {
+    case "setup" :
+      test.setup.filter(e=>e.id==step.id)[0].steps.forEach(setupStep=>{
+        p = p.then(()=>{
+          let np = runStep(setupStep, subtest, test, state);
+          emu.idle();
+          return np;
+        });
+      });
+      break;
     case "load" :
-            console.log(`> Loading file "${step.fn}"`);
+      p = p.then(() => {
+        console.log(`> LOADING FILE "${step.fn}"`);
         emu.tx(`load(${JSON.stringify(step.fn)})\n`);
+      });
       break;
     case "cmd" :
-            console.log(`> Sending JS "${step.js}"`);
+      p = p.then(() => {
+        console.log(`> SENDING JS \`${step.js}\``, step.text ? "- " + step.text : "");
         emu.tx(`${step.js}\n`);
+      });
+      break;
+    case "wrap" :
+      p = p.then(() => {
+        wrap(step.fn, step.id);
+      });
+      break;
+    case "gb" : 
+      p = p.then(() => {
+        let obj = Object.apply({
+          src:'Messenger',
+          t: 'notify',
+          type: 'text',
+          id: Date.now().toFixed(0),
+          title:'title',
+          body:'body'
+        }, step.obj || {});
+        emu.tx(`GB(${JSON.stringify(obj)})\n`);
+      });
+      break;
+    case "emit" :
+      p = p.then(() => {
+        let parent = step.parent ? step.parent : "Bangle";
+        if (!step.paramsArray) step.paramsArray = [];
+        let args = JSON.stringify([step.event].concat(step.paramsArray));
+        console.log(`> EMIT "${step.event}" on ${parent} with parameters ${JSON.stringify(step.paramsArray, null, null)}`);
+        
+        emu.tx(`${parent}.emit.apply(${parent}, ${args})\n`);
+      });
       break;
-          case "gb" : emu.tx(`GB(${JSON.stringify(step.obj)})\n`); break;
-          case "tap" : emu.tx(`Bangle.emit(...)\n`); break;
     case "eval" :
-            console.log(`> Evaluate "${step.js}"`);
+      p = p.then(() => {
+        console.log(`> EVAL \`${step.js}\``);
         emu.tx(`\x10print(JSON.stringify(${step.js}))\n`);
-            var result = emu.getLastLine();
+        var result = getSanitizedLastLine();
         var expected = JSON.stringify(step.eq);
-            console.log("> GOT "+result);
+        if (verbose)
+          console.log("> GOT `"+result+"`");
         if (result!=expected) {
           console.log("> FAIL: EXPECTED "+expected);
-              ok = false;
+          state.ok = false;
+        } else if (verbose) {
+          console.log("> OK: EXPECTED "+expected);
         }
+      });
       break;
       // tap/touch/drag/button press
       // delay X milliseconds?
+    case "assertArray":
+      p = p.then(() => {
+        state.ok &= assertArray(step);
+      });
+      break;
+    case "assertCall":
+      p = p.then(() => {
+        state.ok &= assertCall(step);
+      });
+      break;
+    case "assert":
+      p = p.then(() => {
+        state.ok &= assertValue(step);
+      });
+      break;
     case "screenshot" :
+      p = p.then(() => {
         console.log(`> Compare screenshots - UNIMPLEMENTED`);
+      });
       break;
     case "saveMemoryUsage" :
+      p = p.then(() => {
         emu.tx(`\x10print(process.memory().usage)\n`);
-            subtest.memUsage = parseInt( emu.getLastLine());
+        subtest.memUsage = parseInt(getSanitizedLastLine());
-            console.log("> CURRENT MEMORY USAGE", subtest.memUsage);
+        console.log("> SAVED MEMORY USAGE", subtest.memUsage);
+      });
       break;
     case "checkMemoryUsage" :
+      p = p.then(() => {
         emu.tx(`\x10print(process.memory().usage)\n`);
-            var memUsage =  emu.getLastLine();
+        var memUsage =  parseInt(getSanitizedLastLine());
-            console.log("> CURRENT MEMORY USAGE", memUsage);
+        console.log("> COMPARE MEMORY USAGE", memUsage);
         if (subtest.memUsage != memUsage ) {
           console.log("> FAIL: EXPECTED MEMORY USAGE OF "+subtest.memUsage);
-              ok = false;
+          state.ok = false;
         }
+      });
+      break;
+    case "sleep" :
+      p = p.then(()=>{
+        return new Promise(resolve => {
+          setTimeout(()=>{
+            console.log("> WAITED FOR", step.ms);
+            resolve();
+          }, step.ms);
+        })
+      });
+      break;
+    case "upload" :
+      p = p.then(()=>{
+        console.log("> UPLOADING" + (step.load ? " AND LOADING" : ""), step.file);
+        emu.tx(AppInfo.getFileUploadCommands(step.as, require("fs").readFileSync(BASE_DIR + "/" + step.file).toString()));
+        if (step.load){
+          emu.tx(`\x10load("${step.as}")\n`);
+        }
+      });
+      break;
+    case "console" :
+      p = p.then(()=>{
+        return new Promise(resolve => {
+          if (process.stdin.isTTY){
+            console.log("> STARTING INTERACTIVE CONSOLE");
+
+            let shutdownHandler = function (code) {
+              console.log(" STOPPING INTERACTIVE CONSOLE");
+              process.stdin.removeListener("readable", stdinlistener)
+              process.stdin.setRawMode(false);
+              handleRx = ()=>{};
+              handleConsoleOutput = handleConsoleOutputCurrent;
+              resolve();
+            }
+
+            let stdinlistener = () => {
+              while ((chunk = process.stdin.read()) !== null) {
+                if (chunk === '\x03') {
+                  shutdownHandler();
+                }
+                emu.tx(chunk.toString());
+              }
+            };
+
+            handleRx = (c) => {
+              process.stdout.write(String.fromCharCode(c));
+            }
+
+            let handleConsoleOutputCurrent = handleConsoleOutput;
+            handleConsoleOutput = () => {};
+
+            process.stdin.setRawMode(true);
+            process.stdin.setEncoding('ASCII');
+            process.stdin.on("readable", stdinlistener);
+
+            process.stdout.write(">");
+          } else {
+            console.log("> TERMINAL NEEDS TO BE A TTY FOR INTERACTIVE CONSOLE");
+            resolve();
+          }
+        })
+      });
       break;
     default: ERROR("Unknown step type "+step.t);
   }
+  p = p.then(()=> {
     emu.idle();
   });
+  return p;
+}
+
+function runTest(test, testState) {
+  apploader.reset();
+  var app = apploader.apps.find(a=>a.id==test.app);
+  if (!app) ERROR(`App ${JSON.stringify(test.app)} not found`);
+  if (app.custom) ERROR(`App ${JSON.stringify(appId)} requires HTML customisation`);
+  
+  return apploader.getAppFilesString(app).then(command => {
+    let p = Promise.resolve();
+    test.tests.forEach((subtest,subtestIdx) => {
+      let state = { ok: true};
+      p = p.then(()=>{
+        console.log(`==============================`);
+        console.log(`"${test.app}" Test ${subtestIdx}`);
+        if (test.description)
+          console.log(`"${test.description}`);
+        console.log(`==============================`);
+        emu.factoryReset();
+        console.log("> SENDING APP "+test.app);
+        emu.tx(command);
+        if (verbose)
+          console.log("> SENT APP");
+        emu.tx("reset()\n");
+        console.log("> RESET");
+
       });
+
+      subtest.steps.forEach(step => {
+        p = p.then(()=>{
+          return runStep(step, subtest, test, state).catch((e)=>{
+            console.log("> STEP FAILED:", e, step);
+            state.ok = false;
+          })
+        });
+      });
+
+      p = p.finally(()=>{
+        console.log("> RESULT -",  (state.ok ? "OK": "FAIL") , "- " + test.app + (subtest.description ? (" - " + subtest.description) : ""));
+        testState.push({
+          app: test.app,
+          number: subtestIdx,
+          result: state.ok ? "SUCCESS": "FAILURE",
+          description: subtest.description
+        });
+      });
+    });
+    p = p.then(()=>{
       emu.stopIdle();
     });
+    return p;
+  });
 }
 
 
+let handleRx = ()=>{};
+let handleConsoleOutput = () => {};
+if (verbose){
+  handleConsoleOutput = (d) => {
+    console.log("<", d);
+  }
+}
+
+let testState = [];
+
 emu.init({
   EMULATOR : EMULATOR,
-  DEVICEID : DEVICEID
+  DEVICEID : DEVICEID,
+  rxCallback : (ch)=>{
+    handleRx(ch);
+  },
+  consoleOutputCallback: (d)=>{
+    handleConsoleOutput(d);
+  }
 }).then(function() {
   // Emulator is now loaded
   console.log("Loading tests");
-  var tests = [];
+  let p = Promise.resolve();
-  apploader.apps.forEach(app => {
+  let apps = apploader.apps;
-    var testFile = APP_DIR+"/"+app.id+"/test.json";
+
+  apps.push(DEMOAPP);
+
+  if (process.argv.includes("--id")) {
+    let f = process.argv[process.argv.indexOf("--id") + 1];
+    apps = apps.filter(e=>e.id==f);
+    if (apps.length == 0){
+      console.log("No apps left after filtering for " + f);
+      process.exitCode(255);
+    }
+  }
+
+  apps.forEach(app => {
+    let test = DEMOTEST;
+    if (app.id != DEMOAPP.id){
+      let testFile = APP_DIR+"/"+app.id+"/test.json";
       if (!require("fs").existsSync(testFile)) return;
-    var test = JSON.parse(require("fs").readFileSync(testFile).toString());
+      test = JSON.parse(require("fs").readFileSync(testFile).toString());
       test.app = app.id;
-    tests.push(test);
+    }
+    p = p.then(()=>{
+      return runTest(test, testState);
     });
-  // Running tests
+  });
-  runTest(TEST);
+  p.finally(()=>{
+    console.log("\n\n");
+    console.log("Overall results:");
+    console.table(testState);
+
+    process.exit(testState.reduce((a,c)=>{
+      return a || ((c.result == "SUCCESS") ? 0 : 1);
+    }, 0))
+  });
+  return p;
 });
+
 /*
   if (erroredApps.length) {
     erroredApps.forEach(app => {