Add a "data" section to apps.json, with data files to clean on uninstall

These are added to `appid.info` as "dataFiles" or "storageFiles", and
can contain wildcards.

README.md

@@ -202,6 +202,13 @@ and which gives information about the app for the Launcher.
   "files:"file1,file2,file3",
      // added by BangleApps loader on upload - lists all files
      // that belong to the app so it can be deleted
+  "dataFiles":"appid.data.json,appid.data?.json"
+     // added by BangleApps loader on upload - lists files that 
+     // the app might write, so they can be deleted on uninstall
+     // typically these files are not uploaded, but created by the app
+     // these can include '*' or '?' wildcards
+  "storageFiles":"
+     // same as "dataFiles", except the app handles these as storageFile
 }
 ```
 
@@ -240,16 +247,27 @@ and which gives information about the app for the Launcher.
      "evaluate":true          // if supplied, data isn't quoted into a String before upload
                               // (eg it's evaluated as JS)
     },
+  ]
+  "data": [                   // list of files the app writes to
+    {"name":"appid.data.json",  // filename used in storage
+     "storageFile":true       // if supplied, file is treated as storageFile
+    },
+    {"wildcard":"appid.data.*" // wildcard of filenames used in storage
+    },                         // this is mutually exclusive with using "name" 
+  ],
   "sortorder" : 0,            // optional - choose where in the list this goes.
                               // this should only really be used to put system
                               // stuff at the top
-  ]
 }
 ```
 
 * name, icon and description present the app in the app loader.
 * tags is used for grouping apps in the library, separate multiple entries by comma. Known tags are `tool`, `system`, `clock`, `game`, `sound`, `gps`, `widget`, `launcher` or empty.
 * storage is used to identify the app files and how to handle them
+* data is used to clean up files when the app is uninstalled
+  (If the app has settings but no data section, it is assumed settings are 
+  stored in `appid.settings.json`, so there is no need to add a data section 
+  containing only that file)
 
 ### `apps.json`: `custom` element
 
@@ -351,19 +369,16 @@ Example `settings.js`
   E.showMenu(appMenu)
 })
 ```
-In this example the app needs to add both `app.settings.js` and
-`app.settings.json` to `apps.json`:
+In this example the app needs to add `app.settings.js` to `apps.json`:
 ```json
   { "id": "app",
     ...
     "storage": [
       ...
       {"name":"app.settings.js","url":"settings.js"},
-      {"name":"app.settings.json","content":"{}"}
     ]
   },
 ```
-That way removing the app also cleans up `app.settings.json`.
 
 ## Coding hints
 

bin/sanitycheck.js

@@ -74,6 +74,8 @@ apps.forEach((app,appIdx) => {
   var fileNames = [];
   app.storage.forEach((file) => {
     if (!file.name) ERROR(`App ${app.id} has a file with no name`);
+    if (file.name.includes('?') || file.name.includes('*'))
+      ERROR(`App ${app.id} storage file ${file.name} contains wildcards`);
     if (fileNames.includes(file.name))
       ERROR(`App ${app.id} file ${file.name} is a duplicate`);
     fileNames.push(file.name);
@@ -115,6 +117,37 @@ apps.forEach((app,appIdx) => {
       if (!STORAGE_KEYS.includes(key)) ERROR(`App ${app.id}'s ${file.name} has unknown key ${key}`);
     }
   });
+  let dataNames = [];
+  (app.data||[]).forEach((data)=>{
+    if (!data.name && !data.wildcard) ERROR(`App ${app.id} has a data file with no name`);
+    if (dataNames.includes(data.name||data.wildcard))
+      ERROR(`App ${app.id} data file ${data.name||data.wildcard} is a duplicate`);
+    dataNames.push(data.name||data.wildcard)
+    if ('name' in data && 'wildcard' in data)
+      ERROR(`App ${app.id} data file ${data.name} has both name and wildcard`);
+    if (data.name) {
+      if (data.name.includes('?') || data.name.includes('*'))
+        ERROR(`App ${app.id} data file name ${data.name} contains wildcards`);
+    }
+    if (data.wildcard) {
+      if (!data.wildcard.includes('?') && !data.wildcard.includes('*'))
+        ERROR(`App ${app.id} data file wildcard ${data.wildcard} does not actually contains wildcard`);
+      if (data.wildcard.replace(/\?|\*/g,'') === '')
+        ERROR(`App ${app.id} data file wildcard ${data.wildcard} does not contain regular characters`);
+      else if (data.wildcard.replace(/\?|\*/g,'').length < 3)
+        WARN(`App ${app.id} data file wildcard ${data.wildcard} is very broad`);
+      else if (!data.wildcard.includes(app.id))
+        WARN(`App ${app.id} data file wildcard ${data.wildcard} does not include app ID`);
+    }
+    if ('storageFile' in data && typeof data.storageFile !== 'boolean')
+      ERROR(`App ${app.id} data file ${data.name||data.wildcard} has non-boolean value for "storageFile"`);
+    for (const key in data) {
+      if (!['name','wildcard','storageFile'].includes(key))
+        ERROR(`App ${app.id} data file ${data.name||data.wildcard} has unknown property "${key}"`);
+    }
+  });
+  if (fileNames.includes(app.id+".settings.js") && dataNames.length===1 && dataNames[0] === app.id+'.settings.json')
+    WARN(`App ${app.id} has settings, so does not need to declare data file ${app.id+'.settings.json'}`)
   //console.log(fileNames);
   if (isApp && !fileNames.includes(app.id+".app.js")) ERROR(`App ${app.id} has no entrypoint`);
   if (isApp && !fileNames.includes(app.id+".img")) ERROR(`App ${app.id} has no JS icon`);

js/appinfo.js

@@ -69,6 +69,19 @@ var AppInfo = {
       var fileList = fileContents.map(storageFile=>storageFile.name);
       fileList.unshift(appJSONName); // do we want this? makes life easier!
       json.files = fileList.join(",");
+      let dataFileList = [], storageFileList = [];
+      if ('data' in app) {
+        // add "data" files to appropriate list
+        app.data.forEach(d=>{
+          if (d.storageFile) storageFileList.push(d.name||d.wildcard)
+          else dataFileList.push(d.name||d.wildcard)
+        })
+      } else if (json.settings) {
+        // settings but no data files: assume app uses <appid>.settings.json file
+        dataFileList.push(app.id + '.settings.json')
+      }
+      if (dataFileList.length) json.dataFiles = dataFileList.join(",");
+      if (storageFileList.length) json.storageFiles = storageFileList.join(",");
       fileContents.push({
         name : appJSONName,
         content : JSON.stringify(json)

js/comms.js

@@ -94,10 +94,28 @@ getInstalledApps : () => {
   });
 },
 removeApp : app => { // expects an appid.info structure (i.e. with `files`)
-  if (app.files === '') return Promise.resolve(); // nothing to erase
+  if (!app.files && !app.dataFiles && !app.storageFiles) return Promise.resolve(); // nothing to erase
   Progress.show({title:`Removing ${app.name}`,sticky:true});
-  var cmds = app.files.split(',').map(file=>{
-    return `\x10require("Storage").erase(${toJS(file)});\n`;
+  let cmds = '\x10const s=require("Storage");\n';
+  // remove App files (regular files, exact names only)
+  cmds += app.files.split(',').map(file => `\x10s.erase(${toJS(file)});\n`).join("");
+  // remove Data files (regular files, can use wildcards)
+  cmds += (app.dataFiles||[]).split(',').map(file => {
+    const isGlob = (file.includes('*') || file.includes('?'))
+    if (!isGlob) return `\x10s.erase(${toJS(file)});\n`;
+    const regex = new RegExp(globToRegex(file))
+    return `\x10s.list(${regex}).forEach(f=>s.erase(f));\n`;
+  }).join("");
+  // remove Storage files (storageFiles, can use wildcards)
+  cmds += (app.storageFiles||[]).split(',').map(file => {
+    const isGlob = (file.includes('*') || file.includes('?'))
+    if (!isGlob) return `\x10s.open(${toJS(file)},'r').erase();\n`;
+    // storageFiles have a chunk number appended to their real name
+    const regex = globToRegex(file+'\u0001')
+    // open() doesn't want the chunk number though
+    let cmd = `\x10s.list(${regex}).forEach(f=>s.open(f.substring(0,f.length-1),'r').erase());\n`
+    // using a literal \u0001 char fails (not sure why), so escape it
+    return cmd.replace('\u0001', '\\x01')
   }).join("");
   console.log("removeApp", cmds);
   return Comms.reset().then(new Promise((resolve,reject) => {

js/index.js

@@ -349,6 +349,19 @@ function updateApp(app) {
       .filter(f => f !== app.id + '.info')
       .filter(f => !app.storage.some(s => s.name === f))
       .join(',');
+    let dataFiles = (remove.dataFiles||'').split(','),
+      storageFiles = (remove.storageFiles||'').split(',')
+    if ('data' in app) {
+      // keep declared (in new version) data files
+      dataFiles = dataFiles.filter(f => app.data.some(d => (d.name||d.wildcard) === f))
+      storageFiles = storageFiles.filter(f => app.data.some(d => (d.name||d.wildcard) === f))
+    }
+    else if (remove.settings || app.settings) {
+      // app with settings but no data files declared: keep <appid>.settings.json
+      dataFiles = dataFiles.filter(f => f !== (app.id + '.settings.json'))
+    }
+    if (dataFiles.length) remove.dataFiles = dataFiles.join(',');
+    if (storageFiles.length) remove.storageFiles = storageFiles.join(',')
     return Comms.removeApp(remove);
   }).then(()=>{
     showToast(`Updating ${app.name}...`);

js/utils.js

@@ -8,6 +8,18 @@ function escapeHtml(text) {
   };
   return text.replace(/[&<>"']/g, function(m) { return map[m]; });
 }
+// simple glob to regex conversion, only supports "*" and "?" wildcards
+function globToRegex(pattern) {
+  const ESCAPE = '.*+-?^${}()|[]\\';
+  const regex = pattern.replace(/./g, c => {
+    switch (c) {
+      case '?': return '.';
+      case '*': return '.*';
+      default: return ESCAPE.includes(c) ? ('\\' + c) : c;
+    }
+  });
+  return new RegExp('^'+regex+'$');
+}
 function htmlToArray(collection) {
   return [].slice.call(collection);
 }