FOSS
/
BangleApps
mirror of https://github.com/espruino/BangleApps
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Making appinfo language scan work under node.js moving language_scan and other libs into AppLoader core from BangleApps

pull/2871/head
Gordon Williams 2023-07-07 15:19:32 +01:00
parent 456d2c7293
commit 2e6574f6ed
11 changed files with 10 additions and 630 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/apploader.js
View File

@ -38,7 +38,7 @@ function ERROR(msg) {
var deviceId = "BANGLEJS2";
var apploader = require("./lib/apploader.js");
var apploader = require("../core/lib/apploader.js");
var args = process.argv;
var bangleParam = args.findIndex(arg => /-b\d/.test(arg));

5
bin/firmwaremaker.js
View File

@ -14,9 +14,10 @@ var APPS = [ // IDs of apps to install
var MINIFY = true;
var fs = require("fs");
var apploader = require("./lib/apploader.js");
var apploader = require("../core/lib/apploader.js");
apploader.init({
DEVICEID : DEVICEID
DEVICEID : DEVICEID,
//language : "lang/de_DE.json"
});
var appfiles = [];

2
bin/firmwaremaker_c.js
View File

@ -34,7 +34,7 @@ if (DEVICEID=="BANGLEJS") {
}
console.log("Device = ",DEVICEID);
var apploader = require("./lib/apploader.js");
var apploader = require("../core/lib/apploader.js");
apploader.init({
DEVICEID : DEVICEID
});

64
bin/language_render.js
View File

@ -1,64 +0,0 @@
#!/bin/node
/*
Takes language files that have been written with unicode chars that Bangle.js cannot render
with its built-in fonts, and pre-render them.
*/
//const FONT_SIZE = 18;
//const FONT_NAME = 'Sans';
const FONT_SIZE = 16; // 12pt
const FONT_NAME = '"Unifont Regular"'; // or just 'Sans'
var createCanvas, registerFont;
try {
createCanvas = require("canvas").createCanvas;
registerFont = require("canvas").registerFont;
} catch(e) {
console.log("ERROR: needc canvas library");
console.log("Try: npm install canvas");
process.exit(1);
}
// Use font from https://unifoundry.com/unifont/ as it scales well at 16px high
registerFont(__dirname+'/unifont-15.0.01.ttf', { family: 'Unifont Regular' })
var imageconverter = require(__dirname+"/../webtools/imageconverter.js");
const canvas = createCanvas(200, 20)
const ctx = canvas.getContext('2d')
function renderText(txt) {
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.font = FONT_SIZE+'px '+FONT_NAME;
ctx.fillStyle = "white";
ctx.fillText(txt, 0, FONT_SIZE);
var str = imageconverter.canvastoString(canvas, { autoCrop:true, output:"raw", mode:"1bit", transparent:true } );
// for testing:
// console.log(txt);
// console.log("g.drawImage(",imageconverter.canvastoString(canvas, { autoCrop:true, output:"string", mode:"1bit" } ),");");
// process.exit(1);
return "\0"+str;
}
function renderLangFile(file) {
var fileIn = __dirname + "/../lang/unicode-based/"+file;
var fileOut = __dirname + "/../lang/"+file;
console.log("Reading",fileIn);
var inJSON = JSON.parse(require("fs").readFileSync(fileIn));
var outJSON = { "// created with bin/language_render.js" : ""};
for (var categoryName in inJSON) {
if (categoryName.includes("//")) continue;
var category = inJSON[categoryName];
outJSON[categoryName] = {};
for (var english in category) {
if (english.includes("//")) continue;
var translated = category[english];
//console.log(english,"=>",translated);
outJSON[categoryName][english] = renderText(translated);
}
}
require("fs").writeFileSync(fileOut, JSON.stringify(outJSON,null,2));
console.log("Written",fileOut);
}
renderLangFile("ja_JA.json");

327
bin/language_scan.js
View File

@ -1,327 +0,0 @@
#!/usr/bin/env node
/* Scans for strings that may be in English in each app, and
outputs a list of strings that have been found.
See https://github.com/espruino/BangleApps/issues/1311
Needs old 'translate':
npm install translate@1.4.1
For actual translation you need to sign up for a free Deepl API at https://www.deepl.com/
```
# show status
bin/language_scan.js -r
# add missing keys for all languages (in english)
bin/language_scan.js -r
# for translation
bin/language_scan.js --deepl YOUR_API_KEY --turl https://api-free.deepl.com
*/
var childProcess = require('child_process');
let refresh = false;
function handleCliParameters ()
{
let usage = "USAGE: language_scan.js [options]";
let die = function (message) {
console.log(usage);
console.log(message);
process.exit(3);
};
let hadTURL = false,
hadDEEPL = false;
for(let i = 2; i < process.argv.length; i++)
{
const param = process.argv[i];
switch(param)
{
case '-r':
case '--refresh':
refresh = true;
break;
case '--deepl':
i++;
let KEY = process.argv[i];
if(KEY === '' || KEY === null || KEY === undefined)
{
die('--deepl requires a parameter: the API key to use');
}
process.env.DEEPL = KEY;
hadDEEPL = true;
break;
case '--turl':
i++;
let URL = process.argv[i];
if(URL === '' || URL === null || URL === undefined)
{
die('--turl requires a parameter: the URL to use');
}
process.env.TURL = URL;
hadTURL = true;
break;
case '-h':
case '--help':
console.log(usage+"\n");
console.log("Parameters:");
console.log(" -h, --help Output this help text and exit");
console.log(" -r, --refresh Auto-add new strings into lang/*.json");
console.log(' --deepl KEY Enable DEEPL as auto-translation engine and');
console.log(' use KEY as its API key. You also need to provide --turl');
console.log(' --turl URL In combination with --deepl, use URL as the API base URL');
process.exit(0);
default:
die("Unknown parameter: "+param+", use --help for options");
}
}
if((hadTURL !== false || hadDEEPL !== false) && hadTURL !== hadDEEPL)
{
die("Use of deepl requires both a --deepl API key and --turl URL");
}
}
handleCliParameters();
let translate = false;
if (process.env.DEEPL) {
// Requires translate
// npm i translate
translate = require("translate");
translate.engine = "deepl"; // Or "yandex", "libre", "deepl"
translate.key = process.env.DEEPL; // Requires API key (which are free)
translate.url = process.env.TURL;
}
var IGNORE_STRINGS = [
"5x5","6x8","6x8:2","4x6","12x20","6x15","5x9Numeric7Seg", "Vector", // fonts
"---","...","*","##","00","GPS","ram",
"12hour","rising","falling","title",
"sortorder","tl","tr",
"function","object", // typeof===
"txt", // layout styles
"play","stop","pause", "volumeup", "volumedown", // music state
"${hours}:${minutes}:${seconds}", "${hours}:${minutes}",
"BANGLEJS",
"fgH", "bgH", "m/s",
"undefined", "kbmedia", "NONE",
];
var IGNORE_FUNCTION_PARAMS = [
"read",
"readJSON",
"require",
"setFont","setUI","setLCDMode",
"on",
"RegExp","sendCommand",
"print","log"
];
var IGNORE_ARRAY_ACCESS = [
"WIDGETS"
];
var BASEDIR = __dirname+"/../";
Espruino = require(BASEDIR+"core/lib/espruinotools.js");
var fs = require("fs");
var APPSDIR = BASEDIR+"apps/";
function ERROR(s) {
console.error("ERROR: "+s);
process.exit(1);
}
function WARN(s) {
console.log("Warning: "+s);
}
function log(s) {
console.log(s);
}
var apploader = require("./lib/apploader.js");
apploader.init({
DEVICEID : "BANGLEJS2"
});
var apps = apploader.apps;
// Given a string value, work out if it's obviously not a text string
function isNotString(s, wasFnCall, wasArrayAccess) {
if (s=="") return true;
// wasFnCall is set to the function name if 's' is the first argument to a function
if (wasFnCall && IGNORE_FUNCTION_PARAMS.includes(wasFnCall)) return true;
if (wasArrayAccess && IGNORE_ARRAY_ACCESS.includes(wasArrayAccess)) return true;
if (s=="Storage") console.log("isNotString",s,wasFnCall);
if (s.length<3) return true; // too short
if (s.length>40) return true; // too long
if (s[0]=="#") return true; // a color
if (s.endsWith('.log') || s.endsWith('.js') || s.endsWith(".info") || s.endsWith(".csv") || s.endsWith(".json") || s.endsWith(".img") || s.endsWith(".txt")) return true; // a filename
if (s.endsWith("=")) return true; // probably base64
if (s.startsWith("BTN")) return true; // button name
if (IGNORE_STRINGS.includes(s)) return true; // one we know to ignore
if (!isNaN(parseFloat(s)) && isFinite(s)) return true; //is number
if (s.match(/^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$/)) return true; //roman number
if (!s.match(/.*[A-Z].*/i)) return true; // No letters
if (s.match(/.*[0-9].*/i)) return true; // No letters
if (s.match(/.*\(.*\).*/)) return true; // is function
if (s.match(/[A-Za-z]+[A-Z]([A-Z]|[a-z])*/)) return true; // is camel case
if (s.includes('_')) return true;
return false;
}
function getTextFromString(s) {
return s.replace(/^([.<>\-\n ]*)([^<>\!\?]*?)([.<>\!\?\-\n ]*)$/,"$2");
}
// A string that *could* be translated?
var untranslatedStrings = [];
// Strings that are marked with 'LANG'
var translatedStrings = [];
function addString(list, str, file) {
str = getTextFromString(str);
var entry = list.find(e => e.str==str);
if (!entry) {
entry = { str:str, uses:0, files : [] };
list.push(entry);
}
entry.uses++;
if (!entry.files.includes(file))
entry.files.push(file)
}
console.log("Scanning apps...");
//apps = apps.filter(a=>a.id=="wid_edit");
apps.forEach((app,appIdx) => {
var appDir = APPSDIR+app.id+"/";
app.storage.forEach((file) => {
if (!file.url || !file.name.endsWith(".js")) return;
var filePath = appDir+file.url;
var shortFilePath = "apps/"+app.id+"/"+file.url;
var fileContents = fs.readFileSync(filePath).toString();
var lex = Espruino.Core.Utils.getLexer(fileContents);
var lastIdx = 0;
var wasFnCall = undefined; // set to 'setFont' if we're at something like setFont(".."
var wasArrayAccess = undefined; // set to 'WIDGETS' if we're at something like WIDGETS[".."
var tok = lex.next();
while (tok!==undefined) {
var previousString = fileContents.substring(lastIdx, tok.startIdx);
if (tok.type=="STRING") {
if (previousString.includes("/*LANG*/")) { // translated!
addString(translatedStrings, tok.value, shortFilePath);
} else { // untranslated - potential to translate?
// filter out numbers
if (!isNotString(tok.value, wasFnCall, wasArrayAccess)) {
addString(untranslatedStrings, tok.value, shortFilePath);
}
}
} else {
if (tok.value!="(") wasFnCall=undefined;
if (tok.value!="[") wasArrayAccess=undefined;
}
//console.log(wasFnCall,tok.type,tok.value);
if (tok.type=="ID") {
wasFnCall = tok.value;
wasArrayAccess = tok.value;
}
lastIdx = tok.endIdx;
tok = lex.next();
}
});
});
untranslatedStrings.sort((a,b)=>a.uses - b.uses);
translatedStrings.sort((a,b)=>a.uses - b.uses);
/*
* @description Add lang to start of string
* @param str string to add LANG to
* @param file file that string is found
* @returns void
*/
//TODO fix settings bug
function applyLANG(str, file) {
fs.readFile(file, 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
const regex = new RegExp(`(.*)((?<!\/\*LANG\*\/)["|']${str}[^A-Z].*)`, 'gi');
const result = data.replace(regex, '$1/*LANG*/$2');
console.log(str, file);
fs.writeFile(file, result, 'utf8', function (err) {
if (err) return console.log(err);
});
});
}
var report = "";
log("Translated Strings that are not tagged with LANG");
log("=================================================================");
log("");
log("Maybe we should add /*LANG*/ to these automatically?");
log("");
const wordsToAdd = untranslatedStrings.filter(e => translatedStrings.find(t=>t.str==e.str));
// Uncomment to add LANG to all strings
// THIS IS EXPERIMENTAL
//wordsToAdd.forEach(e => e.files.forEach(a => applyLANG(e.str, a)));
log(wordsToAdd.map(e=>`${JSON.stringify(e.str)} (${e.uses} uses)`).join("\n"));
log("");
//process.exit(1);
log("Possible English Strings that could be translated");
log("=================================================================");
log("");
log("Add these to IGNORE_STRINGS if they don't make sense...");
log("");
// ignore ones only used once or twice
log(untranslatedStrings.filter(e => e.uses>2).filter(e => !translatedStrings.find(t=>t.str==e.str)).map(e=>`${JSON.stringify(e.str)} (${e.uses} uses)`).join("\n"));
log("");
//process.exit(1);
let languages = JSON.parse(fs.readFileSync(`${BASEDIR}/lang/index.json`).toString());
for (let language of languages) {
if (language.code == "en_GB") {
console.log(`Ignoring ${language.code}`);
continue;
}
console.log(`Scanning ${language.code}`);
log(language.code);
log("==========");
let translations = JSON.parse(fs.readFileSync(`${BASEDIR}/lang/${language.url}`).toString());
let translationPromises = [];
translatedStrings.forEach(translationItem => {
if (!translations.GLOBAL[translationItem.str]) {
console.log(`Missing GLOBAL translation for ${JSON.stringify(translationItem)}`);
translationItem.files.forEach(file => {
let m = file.match(/\/([a-zA-Z0-9_-]*)\//g);
if (m && m[0]) {
let appName = m[0].replaceAll("/", "");
if (translations[appName] && translations[appName][translationItem.str]) {
console.log(` but LOCAL translation found in \"${appName}\"`);
} else if (translate && language.code !== "tr_TR") { // Auto Translate
translationPromises.push(new Promise(async (resolve) => {
const translation = await translate(translationItem.str, language.code.split("_")[0]);
console.log("Translating:", translationItem.str, translation);
translations.GLOBAL[translationItem.str] = translation;
resolve()
}))
} else if(refresh && !translate) {
translationPromises.push(new Promise(async (resolve) => {
translations.GLOBAL[translationItem.str] = translationItem.str;
resolve()
}))
}
}
});
}
});
Promise.all(translationPromises).then(() => {
fs.writeFileSync(`${BASEDIR}/lang/${language.url}`, JSON.stringify(translations, null, 4))
});
log("");
}
console.log("Done.");

115
bin/lib/apploader.js
View File

@ -1,115 +0,0 @@
/* Node.js library with utilities to handle using the app loader from node.js */
var DEVICEID = "BANGLEJS2";
var MINIFY = true; // minify JSON?
var BASE_DIR = __dirname + "/../..";
var APPSDIR = BASE_DIR+"/apps/";
//eval(require("fs").readFileSync(__dirname+"../core/js/utils.js"));
var Espruino = require(__dirname + "/../../core/lib/espruinotools.js");
//eval(require("fs").readFileSync(__dirname + "/../../core/lib/espruinotools.js").toString());
//eval(require("fs").readFileSync(__dirname + "/../../core/js/utils.js").toString());
var AppInfo = require(__dirname+"/../../core/js/appinfo.js");
var SETTINGS = {
pretokenise : true
};
global.Const = {
/* Are we only putting a single app on a device? If so
apps should all be saved as .bootcde and we write info
about the current app into app.info */
SINGLE_APP_ONLY : false,
};
var apps = [];
var device = { id : DEVICEID, appsInstalled : [] };
// call with {DEVICEID:"BANGLEJS/BANGLEJS2"}
exports.init = function(options) {
if (options.DEVICEID) {
DEVICEID = options.DEVICEID;
device.id = options.DEVICEID;
}
// Try loading from apps.json
apps.length=0;
try {
var appsStr = require("fs").readFileSync(BASE_DIR+"/apps.json");
var appList = JSON.parse(appsStr);
appList.forEach(a => apps.push(a));
} catch (e) {
console.log("Couldn't load apps.json", e.toString());
}
// Load app metadata from each app
if (!apps.length) {
console.log("Loading apps/.../metadata.json");
var dirs = require("fs").readdirSync(APPSDIR, {withFileTypes: true});
dirs.forEach(dir => {
var appsFile;
if (dir.name.startsWith("_example") || !dir.isDirectory())
return;
try {
appsFile = require("fs").readFileSync(APPSDIR+dir.name+"/metadata.json").toString();
} catch (e) {
console.error(dir.name+"/metadata.json does not exist");
return;
}
apps.push(JSON.parse(appsFile));
});
}
};
exports.AppInfo = AppInfo;
exports.apps = apps;
// used by getAppFiles
function fileGetter(url) {
url = BASE_DIR+"/"+url;
console.log("Loading "+url)
var data;
if (MINIFY && url.endsWith(".json")) {
var f = url.slice(0,-5);
console.log("MINIFYING JSON "+f);
var j = eval("("+require("fs").readFileSync(url).toString("binary")+")");
data = JSON.stringify(j);
} else {
var blob = require("fs").readFileSync(url);
if (url.endsWith(".js") || url.endsWith(".json"))
data = blob.toString(); // allow JS/etc to be written in UTF-8
else
data = blob.toString("binary")
}
return Promise.resolve(data);
}
exports.getAppFiles = function(app) {
var allFiles = [];
var uploadOptions = {
apps : apps,
needsApp : app => {
if (app.provides_modules) {
if (!app.files) app.files="";
app.files = app.files.split(",").concat(app.provides_modules).join(",");
}
return AppInfo.getFiles(app, {
fileGetter:fileGetter,
settings : SETTINGS,
device : { id : DEVICEID }
}).then(files => { allFiles = allFiles.concat(files); return app; });
}
};
return AppInfo.checkDependencies(app, device, uploadOptions).then(() => AppInfo.getFiles(app, {
fileGetter:fileGetter,
settings : SETTINGS,
device : device
})).then(files => {
allFiles = allFiles.concat(files);
return allFiles;
});
};
// Get all the files for this app as a string of Storage.write commands
exports.getAppFilesString = function(app) {
return exports.getAppFiles(app).then(files => {
return files.map(f=>f.cmd).join("\n")+"\n"
})
};

115
bin/lib/emulator.js
View File

@ -1,115 +0,0 @@
/* Node.js library with utilities to handle using the emulator from node.js */
var EMULATOR = "banglejs2";
var DEVICEID = "BANGLEJS2";
var BASE_DIR = __dirname + "/../..";
var DIR_IDE = BASE_DIR + "/../EspruinoWebIDE";
/* we factory reset ONCE, get this, then we can use it to reset
state quickly for each new app */
var factoryFlashMemory;
// Log of messages from app
var appLog = "";
var lastOutputLine = "";
function onConsoleOutput(txt) {
appLog += txt + "\n";
lastOutputLine = txt;
}
exports.init = function(options) {
if (options.EMULATOR)
EMULATOR = options.EMULATOR;
if (options.DEVICEID)
DEVICEID = options.DEVICEID;
eval(require("fs").readFileSync(DIR_IDE + "/emu/emulator_"+EMULATOR+".js").toString());
eval(require("fs").readFileSync(DIR_IDE + "/emu/emu_"+EMULATOR+".js").toString());
eval(require("fs").readFileSync(DIR_IDE + "/emu/common.js").toString()/*.replace('console.log("EMSCRIPTEN:"', '//console.log("EMSCRIPTEN:"')*/);
jsRXCallback = function() {};
jsUpdateGfx = function() {};
factoryFlashMemory = new Uint8Array(FLASH_SIZE);
factoryFlashMemory.fill(255);
exports.flashMemory = flashMemory;
exports.GFX_WIDTH = GFX_WIDTH;
exports.GFX_HEIGHT = GFX_HEIGHT;
exports.tx = jsTransmitString;
exports.idle = jsIdle;
exports.stopIdle = jsStopIdle;
exports.getGfxContents = jsGetGfxContents;
return new Promise(resolve => {
setTimeout(function() {
console.log("Emulator Loaded...");
jsInit();
jsIdle();
console.log("Emulator Factory reset");
exports.tx("Bangle.factoryReset()\n");
factoryFlashMemory.set(flashMemory);
console.log("Emulator Ready!");
resolve();
},0);
});
};
// Factory reset
exports.factoryReset = function() {
exports.flashMemory.set(factoryFlashMemory);
exports.tx("reset()\n");
appLog="";
};
// Transmit a string
exports.tx = function() {}; // placeholder
exports.idle = function() {}; // placeholder
exports.stopIdle = function() {}; // placeholder
exports.getGfxContents = function() {}; // placeholder
exports.flashMemory = undefined; // placeholder
exports.GFX_WIDTH = undefined; // placeholder
exports.GFX_HEIGHT = undefined; // placeholder
// Get last line sent to console
exports.getLastLine = function() {
return lastOutputLine;
};
// Gets the screenshot as RGBA Uint32Array
exports.getScreenshot = function() {
var rgba = new Uint8Array(exports.GFX_WIDTH*exports.GFX_HEIGHT*4);
exports.getGfxContents(rgba);
var rgba32 = new Uint32Array(rgba.buffer);
return rgba32;
}
// Write the screenshot to a file options={errorIfBlank}
exports.writeScreenshot = function(imageFn, options) {
options = options||{};
return new Promise((resolve,reject) => {
var rgba32 = exports.getScreenshot();
if (options.errorIfBlank) {
var firstPixel = rgba32[0];
var blankImage = rgba32.every(col=>col==firstPixel);
if (blankImage) reject("Image is blank");
}
var Jimp = require("jimp");
let image = new Jimp(exports.GFX_WIDTH, exports.GFX_HEIGHT, function (err, image) {
if (err) throw err;
let buffer = image.bitmap.data;
buffer.set(new Uint8Array(rgba32.buffer));
image.write(imageFn, (err) => {
if (err) return reject(err);
console.log("Image written as "+imageFn);
resolve();
});
});
});
}

4
bin/runapptests.js
View File

@ -62,11 +62,11 @@ if (!require("fs").existsSync(DIR_IDE)) {
process.exit(1);
}
var apploader = require(BASE_DIR+"/bin/lib/apploader.js");
var apploader = require(BASE_DIR+"/core/lib/apploader.js");
apploader.init({
DEVICEID : DEVICEID
});
var emu = require(BASE_DIR+"/bin/lib/emulator.js");
var emu = require(BASE_DIR+"/core/lib/emulator.js");
// Last set of text received
var lastTxt;

4
bin/thumbnailer.js
View File

@ -8,8 +8,8 @@ var EMULATOR = "banglejs1";
var DEVICEID = "BANGLEJS";
var SCREENSHOT_DIR = __dirname+"/../screenshots/";
var emu = require("./lib/emulator.js");
var apploader = require("./lib/apploader.js");
var emu = require("../core/lib/emulator.js");
var apploader = require("../core/lib/apploader.js");
var singleAppId;

BIN
bin/unifont-15.0.01.ttf
View File

Binary file not shown.

2
core

@ -1 +1 @@
Subproject commit e521afd722c46689007e62fdb5e370991f249823
Subproject commit 8b92079209028d81be2f837f9fcf9649e292189c