Improve sanity check for locales

pull/3573/head
Anton 2024-09-14 22:01:43 +02:00
parent e814704b62
commit 0e53f127a1
5 changed files with 373 additions and 43 deletions

View File

@ -112,6 +112,7 @@ module.exports = {
"getSerial": "readonly",
"getTime": "readonly",
"global": "readonly",
"globalThis": "readonly",
"HIGH": "readonly",
"I2C1": "readonly",
"Infinity": "readonly",

View File

@ -21,15 +21,17 @@
<label><input id="customize" type="checkbox" /> Advanced: Customize the date and time formats.</label>
</div>
<p>
<span id="customize-warning"></span>
<table id="examples-short-long"></table>
<table id="examples"></table>
</p>
<p id="customize-warning"></p>
<p>Then click <button id="upload" class="btn btn-primary">Upload</button></p>
<script src="../../core/lib/customize.js"></script>
<script src="../../core/js/utils.js"></script>
<script src="sanitycheck.js"></script>
<script src="locales.js"></script>
<script>
@ -103,32 +105,6 @@ exports = { name : "system", currencySym:"£",
return '\\x'+(n+256).toString(16).slice(-2);
}
// do some sanity checks
Object.keys(locales).forEach(function(localeName) {
var locale = locales[localeName];
if (locale.trans && !locale.trans.on) console.error(localeName+": If translations are provided, 'on' *must* be included");
if (distanceUnits[locale.distance[0]]===undefined) console.error(localeName+": Unknown distance unit "+locale.distance[0]);
if (distanceUnits[locale.distance[1]]===undefined) console.error(localeName+": Unknown distance unit "+locale.distance[1]);
if (speedUnits[locale.speed]===undefined) console.error(localeName+": Unknown speed unit "+locale.speed);
if (locale.temperature!='°C' && locale.temperature!='°F')
console.error(localeName+": Unknown temperature unit "+locale.temperature);
// Now check that codepage is ok and all chars in translation are in that codepage
const codePageName = "ISO8859-1";
if (locale.codePage) codePageName = locale.codePage;
const codePage = codePages[codePageName];
if (codePage===undefined) console.error(localeName+": Unknown codePage "+codePageName);
function checkChars(v,path) {
if ("object"==typeof v)
Object.keys(v).forEach(k=>checkChars(v[k], path+"."+k));
else if ("string"==typeof v)
for (var i=0;i<v.length;i++)
if (codePageLookup(localeName, codePage, v[i])===undefined)
console.error(` ... in ${path}[${i}]`);
}
checkChars(locale,localeName);
});
function createLocaleModule() {
console.log(`Language ${lang}`);
@ -269,8 +245,6 @@ exports = {
}
var date = new Date();
// TODO: This warning should have a link to an article explaining how the formats work, and how long they are allowed to be
document.getElementById("customize-warning").innerText = customizeLocale ? "⚠️ If you make the formats too long, some apps will not work!" : "";
document.getElementById("examples-short-long").innerHTML = `
<tr><td class="table_t"></td><td style="font-weight:bold">Short</td><td style="font-weight:bold">Long</td></tr>
<tr><td class="table_t">Day</td><td>${exports.dow(date,1)}</td><td>${exports.dow(date,0)}</td></tr>
@ -332,27 +306,63 @@ ${customizeLocale ? `<tr><td class="table_t">Meridian names</td>
document.querySelector("input#short-date-pattern").addEventListener("input", event => {
locale.datePattern["1"] = event.target.value;
document.querySelector("td#short-date-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#long-date-pattern").addEventListener("input", event => {
locale.datePattern["0"] = event.target.value;
document.querySelector("td#long-date-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#short-time-pattern").addEventListener("input", event => {
locale.timePattern["1"] = event.target.value;
document.querySelector("td#short-time-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#long-time-pattern").addEventListener("input", event => {
locale.timePattern["0"] = event.target.value;
document.querySelector("td#long-time-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#meridian-am").addEventListener("input", event => {
locale.ampm["0"] = event.target.value;
document.querySelector("span#meridian-am-output").innerText = event.target.value;
checkCustomLocale();
});
document.querySelector("input#meridian-pm").addEventListener("input", event => {
locale.ampm["1"] = event.target.value;
document.querySelector("span#meridian-pm-output").innerText = event.target.value;
checkCustomLocale();
});
let isCheckingLocale = false;
// Polyfill for WebKit:
const requestIdleCallback = globalThis.requestIdleCallback || ((func) => {func()});
// Check that a custom locale follows some basic standards
function checkCustomLocale(){
if(isCheckingLocale) return;
isCheckingLocale = true;
setTimeout(() => {
requestIdleCallback(() => {
isCheckingLocale = false;
const result = globalThis.checkLocale(locale, {speedUnits, distanceUnits, codePages, CODEPAGE_CONVERSIONS});
let text = "";
for(const w of [...result.errors, ...result.warnings]){
text += `⚠️ ${w.name} ${w.error}.\n`;
}
const element = document.getElementById("customize-warning");
if(text.length > 0){
text += "\nIf you upload this locale, some apps might no longer work.\nPlease try to resolve the issues before uploading."
element.classList.add("toast");
element.classList.add("toast-error");
}else{
element.classList.remove("toast");
element.classList.remove("toast-error");
}
element.innerText = text;
}, {timeout: 2000})
}, 500);
}
}
return getLocaleModule(false);
}
@ -416,6 +426,10 @@ ${customizeLocale ? `<tr><td class="table_t">Meridian names</td>
}else{
createLocaleModule();
}
const warningsElement = document.getElementById("customize-warning")
warningsElement.innerText = "";
warningsElement.classList.remove("toast");
warningsElement.classList.remove("toast-error");
}
customizeSelector.addEventListener('change', handleCustomizeChange);
function handleCustomizeChange(){

View File

@ -536,7 +536,7 @@ var locales = {
temperature: '°C',
ampm: { 0: "öö", 1: "ös" },
timePattern: { 0: "%HH:%MM:%SS", 1: "%HH:%MM" },
datePattern: { 0: "%d %w %Y %A", 1: "%d/%m/%Y" }, // 1 Mart 2020 Pazar // "01/03/2020"
datePattern: { 0: "%d %B %Y %A", 1: "%d/%m/%Y" }, // 1 Mart 2020 Pazar // "01/03/2020"
abmonth: "Oca,Sub,Mar,Nis,May,Haz,Tem,Agu,Eyl,Eki,Kas,Ara",
month: "Ocak,Subat,Mart,Nisan,Mayis,Haziran,Temmuz,Agustos,Eylul,Ekim,Kasim,Aralik",
abday: "Paz,Pzt,Sal,Car,Per,Cum,Cmt",

233
apps/locale/sanitycheck.js Normal file
View File

@ -0,0 +1,233 @@
/**
* Maps the Espruino datetime format to min and max character lengths.
* Used when determining if a format can produce outputs that are too short or long.
*/
const datetime_length_map = {
// %A, %a, %B, %b vary depending on the locale, so they are calculated later
"%Y": [4, 4],
"%y": [2, 2],
"%m": [2, 2],
"%-m": [1, 2],
"%d": [2, 2],
"%-d": [1, 2],
"%HH": [2, 2],
"%MM": [2, 2],
"%SS": [2, 2],
};
/**
* Takes an Espruino datetime format string and returns the minumum and maximum possible length of characters that the format could use.
*
* @param {string} datetimeEspruino - The datetime Espruino format
* @returns first the minimum possible length, second the maximum possible length.
*/
function getLengthOfDatetimeFormat(name, datetimeEspruino, locale, errors) {
// Generate the length_map based on the actual names in the locale
const length_map = {...datetime_length_map};
for(const [symbol, values] of [
["%A", locale.day],
["%a", locale.abday],
["%B", locale.month],
["%b", locale.abmonth],
]){
const length = [Infinity, 0];
for(const value of values.split(",")){
if(length[0] > value.length) length[0] = value.length;
if(length[1] < value.length) length[1] = value.length;
}
length_map[symbol] = length;
}
// Find the length of the output
let formatLength = [0, 0];
let i = 0;
while (i < datetimeEspruino.length) {
if (datetimeEspruino[i] === "%") {
let match;
for(const symbolLength of [2, 3]){
const length = length_map[datetimeEspruino.substring(i, i+symbolLength)];
if(length){
match = {
length,
symbolLength,
}
}
}
if(match){
formatLength[0] += match.length[0];
formatLength[1] += match.length[1];
i += match.symbolLength;
}else{
errors.push({name, value: datetimeEspruino, lang: locale.lang, error: `uses an unsupported format symbol: ${datetimeEspruino.substring(i, i+3)}`});
formatLength[0]++;
formatLength[1]++;
i++;
}
} else {
formatLength[0]++;
formatLength[1]++;
i++;
}
}
return formatLength;
}
/**
* Checks that a locale conforms to some basic standards.
*
* @param {object} locale - The locale to test.
* @param {object} meta - Meta information that is needed to check if locales are supported.
* @param {object} meta.speedUnits - The table of speed units.
* @param {object} meta.distanceUnits - The table of distance units.
* @param {object} meta.codePages - Custom codepoint mappings.
* @param {object} meta.CODEPAGE_CONVERSIONS - The table of custom codepoint conversions.
* @returns an object with an array of errors and warnings.
*/
function checkLocale(locale, {speedUnits, distanceUnits, codePages, CODEPAGE_CONVERSIONS}){
const errors = [];
const warnings = [];
const speeds = Object.keys(speedUnits);
const distances = Object.keys(distanceUnits);
checkLength("lang", locale.lang, 5, undefined);
checkLength("decimal point", locale.decimal_point, 1, 1);
checkLength("thousands separator", locale.thousands_sep, 1, 1);
checkLength("speed", locale.speed, 2, 4);
checkIsIn("speed", locale.speed, "speedUnits", speeds);
checkLength("distance", locale.distance["0"], 1, 3);
checkLength("distance", locale.distance["1"], 1, 3);
checkIsIn("distance", locale.distance["0"], "distanceUnits", distances);
checkIsIn("distance", locale.distance["1"], "distanceUnits", distances);
checkLength("temperature", locale.temperature, 1, 2);
checkLength("meridian", locale.ampm["0"], 1, 3);
checkLength("meridian", locale.ampm["1"], 1, 3);
warnIfNot("long time format", locale.timePattern["0"], "%HH:%MM:%SS");
warnIfNot("short time format", locale.timePattern["1"], "%HH:%MM");
checkFormatLength("long time", locale.timePattern["0"], 8, 8);
checkFormatLength("short time", locale.timePattern["1"], 5, 5);
checkFormatLength("long date", locale.datePattern["0"], 6, 14);
checkFormatLength("short date", locale.datePattern["1"], 6, 11);
checkArrayLength("short months", locale.abmonth.split(","), 12, 12);
checkArrayLength("long months", locale.month.split(","), 12, 12);
checkArrayLength("short days", locale.abday.split(","), 7, 7);
checkArrayLength("long days", locale.day.split(","), 7, 7);
for (const abmonth of locale.abmonth.split(",")) {
checkLength("short month", abmonth, 2, 4);
}
for (const month of locale.month.split(",")) {
checkLength("month", month, 3, 11);
}
for (const abday of locale.abday.split(",")) {
checkLength("short day", abday, 2, 4);
}
for (const day of locale.day.split(",")) {
checkLength("day", day, 3, 13);
}
checkEncoding(locale);
function checkLength(name, value, min, max) {
if(typeof value !== "string"){
errors.push({name, value, lang: locale.lang, error: `must be defined and must be a string`});
return;
}
if (min && value.length < min) {
errors.push({name, value, lang: locale.lang, error: `must be longer than ${min-1} characters`});
}
if (max && value.length > max) {
errors.push({name, value, lang: locale.lang, error: `must be shorter than ${max+1} characters`});
}
}
function checkArrayLength(name, value, min, max){
if(!Array.isArray(value)){
errors.push({name, value, lang: locale.lang, error: `must be defined and must be an array`});
return;
}
if (min && value.length < min) {
errors.push({name, value, lang: locale.lang, error: `array must be longer than ${min-1} entries`});
}
if (max && value.length > max) {
errors.push({name, value, lang: locale.lang, error: `array must be shorter than ${max+1} entries`});
}
}
function checkFormatLength(name, value, min, max) {
const length = getLengthOfDatetimeFormat(name, value, locale, errors);
if (min && length[0] < min) {
errors.push({name, value, lang: locale.lang, error: `output must be longer than ${min-1} characters`});
}
if (max && length[1] > max) {
errors.push({name, value, lang: locale.lang, error: `output must be shorter than ${max+1} characters`});
}
}
function checkIsIn(name, value, listName, list) {
if (!list.includes(value)) {
errors.push({name, value, lang: locale.lang, error: `must be included in the ${listName} map`});
}
}
function warnIfNot(name, value, expected) {
if (value !== expected) {
warnings.push({name, value, lang: locale.lang, error: `might not work in some apps if it is not "${expected}"`});
}
}
function checkEncoding(object) {
if(!object){
return;
}else if(typeof object === "string"){
for(const char of object){
const charCode = char.charCodeAt();
if (charCode >= 32 && charCode < 128) {
// ASCII - fully supported
continue;
} else if (codePages["ISO8859-1"].map.indexOf(char) >= 0) {
// At upload time, the char can be converted to a custom codepage
continue;
} else if (CODEPAGE_CONVERSIONS[char]) {
// At upload time, the char can be converted to a similar supported char
continue;
}
errors.push({name: `character ${char}`, value: char, lang: locale.lang, error: `is not supported by BangleJS`});
}
}else{
for(const [key, value] of Object.entries(object)){
if(key === "icon") continue;
checkEncoding(value);
}
}
}
return {errors, warnings};
}
/**
* Checks that an array of locales conform to some basic standards.
*
* @param {object[]} locales - The locales to test.
* @param {object} meta.speedUnits - The table of speed units.
* @param {object} meta.distanceUnits - The table of distance units.
* @param {object} meta.codePages - Custom codepoint mappings.
* @param {object} meta.CODEPAGE_CONVERSIONS - The table of custom codepoint conversions.
* @returns an object with an array of errors and warnings.
*/
function checkLocales(locales, meta){
let errors = [];
let warnings = [];
for(const locale of Object.values(locales)){
const result = checkLocale(locale, meta);
errors = [...errors, ...result.errors];
warnings = [...warnings, ...result.warnings];
}
return {errors, warnings};
}
if(typeof module !== "undefined"){
module.exports = {
checkLocale,
checkLocales,
};
}else{
globalThis.checkLocale = checkLocale;
globalThis.checkLocales = checkLocales;
}

View File

@ -3,6 +3,7 @@
*/
var fs = require("fs");
var vm = require("vm");
var heatshrink = require("../webtools/heatshrink");
var acorn;
try {
@ -20,13 +21,19 @@ var BASEDIR = __dirname+"/../";
var APPSDIR_RELATIVE = "apps/";
var APPSDIR = BASEDIR + APPSDIR_RELATIVE;
var knownWarningCount = 0;
var knownErrorCount = 0;
var warningCount = 0;
var errorCount = 0;
function ERROR(msg, opt) {
// file=app.js,line=1,col=5,endColumn=7
opt = opt||{};
console.log(`::error${Object.keys(opt).length?" ":""}${Object.keys(opt).map(k=>k+"="+opt[k]).join(",")}::${msg}`);
errorCount++;
if (KNOWN_ERRORS.includes(msg)) {
console.log(`Known error : ${msg}`);
knownErrorCount++;
} else {
console.log(`::error${Object.keys(opt).length?" ":""}${Object.keys(opt).map(k=>k+"="+opt[k]).join(",")}::${msg}`);
errorCount++;
}
}
function WARN(msg, opt) {
// file=app.js,line=1,col=5,endColumn=7
@ -39,6 +46,71 @@ function WARN(msg, opt) {
warningCount++;
}
}
/* These are errors that we temporarily allow */
var KNOWN_ERRORS = [
"In locale en_CA, long date output must be shorter than 15 characters",
"In locale fr_FR, long date output must be shorter than 15 characters",
"In locale en_SE, long date output must be shorter than 15 characters",
"In locale en_NZ, long date output must be shorter than 15 characters",
"In locale en_AU, long date output must be shorter than 15 characters",
"In locale de_AT, long date output must be shorter than 15 characters",
"In locale en_IL, long date output must be shorter than 15 characters",
"In locale es_ES, long date output must be shorter than 15 characters",
"In locale fr_BE, long date output must be shorter than 15 characters",
"In locale fi_FI, long date output must be shorter than 15 characters",
"In locale de_CH, long date output must be shorter than 15 characters",
"In locale fr_CH, long date output must be shorter than 15 characters",
"In locale wae_CH, long date output must be shorter than 15 characters",
"In locale tr_TR, long date output must be shorter than 15 characters",
"In locale hu_HU, long date output must be shorter than 15 characters",
"In locale oc_FR, long date output must be shorter than 15 characters",
"In locale ca_ES, long date output must be shorter than 15 characters",
"In locale fr_BE, short month must be shorter than 5 characters",
"In locale fi_FI, short month must be shorter than 5 characters",
"In locale fr_CH, short month must be shorter than 5 characters",
"In locale oc_FR, short month must be shorter than 5 characters",
"In locale hr_HR, short month must be shorter than 5 characters",
"In locale ca_ES, short month must be shorter than 5 characters",
"In locale de_DE, meridian must be longer than 0 characters",
"In locale en_JP, meridian must be longer than 0 characters",
"In locale nl_NL, meridian must be longer than 0 characters",
"In locale fr_FR, meridian must be longer than 0 characters",
"In locale se_SE, meridian must be longer than 0 characters",
"In locale en_SE, meridian must be longer than 0 characters",
"In locale da_DK, meridian must be longer than 0 characters",
"In locale en_DK, meridian must be longer than 0 characters",
"In locale de_AT, meridian must be longer than 0 characters",
"In locale es_ES, meridian must be longer than 0 characters",
"In locale fr_BE, meridian must be longer than 0 characters",
"In locale it_CH, meridian must be longer than 0 characters",
"In locale it_IT, meridian must be longer than 0 characters",
"In locale wae_CH, meridian must be longer than 0 characters",
"In locale oc_FR, meridian must be longer than 0 characters",
"In locale pl_PL, meridian must be longer than 0 characters",
"In locale lv_LV, meridian must be longer than 0 characters",
"In locale nn_NO, meridian must be longer than 0 characters",
"In locale nb_NO, meridian must be longer than 0 characters",
"In locale ca_ES, meridian must be longer than 0 characters",
"In locale de_CH, meridian must be shorter than 4 characters",
"In locale hr_HR, meridian must be shorter than 4 characters",
"In locale sl_SI, meridian must be shorter than 4 characters",
"In locale fr_FR, short month must be shorter than 5 characters",
"In locale sv_SE, speed must be shorter than 5 characters",
];
/* These are warnings we know about but don't want in our output */
var KNOWN_WARNINGS = [
"App gpsrec data file wildcard .gpsrc? does not include app ID",
"App owmweather data file weather.json is also listed as data file for app weather",
"App messagegui storage file messagegui is also listed as storage file for app messagelist",
"App carcrazy has a setting file but no corresponding data entry (add `\"data\":[{\"name\":\"carcrazy.settings.json\"}]`)",
"App loadingscreen has a setting file but no corresponding data entry (add `\"data\":[{\"name\":\"loadingscreen.settings.json\"}]`)",
"App trex has a setting file but no corresponding data entry (add `\"data\":[{\"name\":\"trex.settings.json\"}]`)",
"widhwt isn't an app (widget) but has an app.js file (widhwtapp.js)",
`In locale it_CH, long time format might not work in some apps if it is not "%HH:%MM:%SS"`,
`In locale it_IT, long time format might not work in some apps if it is not "%HH:%MM:%SS"`,
`In locale wae_CH, long time format might not work in some apps if it is not "%HH:%MM:%SS"`,
`In locale wae_CH, short time format might not work in some apps if it is not "%HH:%MM"`,
];
var apps = [];
var dirs = fs.readdirSync(APPSDIR, {withFileTypes: true});
@ -91,16 +163,6 @@ const INTERNAL_FILES_IN_APP_TYPE = { // list of app types and files they SHOULD
'textinput' : ['textinput'],
// notify?
};
/* These are warnings we know about but don't want in our output */
var KNOWN_WARNINGS = [
"App gpsrec data file wildcard .gpsrc? does not include app ID",
"App owmweather data file weather.json is also listed as data file for app weather",
"App messagegui storage file messagegui is also listed as storage file for app messagelist",
"App carcrazy has a setting file but no corresponding data entry (add `\"data\":[{\"name\":\"carcrazy.settings.json\"}]`)",
"App loadingscreen has a setting file but no corresponding data entry (add `\"data\":[{\"name\":\"loadingscreen.settings.json\"}]`)",
"App trex has a setting file but no corresponding data entry (add `\"data\":[{\"name\":\"trex.settings.json\"}]`)",
"widhwt isn't an app (widget) but has an app.js file (widhwtapp.js)",
];
function globToRegex(pattern) {
const ESCAPE = '.*+-?^${}()|[]\\';
@ -397,8 +459,28 @@ while(fileA=allFiles.pop()) {
})
}
// Check each locale in the `locale` app.
sanityCheckLocales();
function sanityCheckLocales(){
const { CODEPAGE_CONVERSIONS } = require("../core/js/utils");
const { checkLocales } = require("../apps/locale/sanitycheck");
const localesCode = fs.readFileSync(__dirname+'/../apps/locale/locales.js', 'utf-8');
vm.runInThisContext(localesCode);
/* global locales, speedUnits, distanceUnits, codePages */
const {errors, warnings} = checkLocales(locales, {speedUnits, distanceUnits, codePages, CODEPAGE_CONVERSIONS});
const file = "locale/locales.js";
for(const w of warnings){
WARN(`In locale ${w.lang}, ${w.name} ${w.error}`, {file, value: w.value});
}
for(const e of errors){
ERROR(`In locale ${e.lang}, ${e.name} ${e.error}`, {file, value: e.value});
}
}
console.log("==================================");
console.log(`${errorCount} errors, ${warningCount} warnings (and ${knownWarningCount} known warnings)`);
console.log(`${errorCount} errors, ${warningCount} warnings (and ${knownErrorCount} known errors, ${knownWarningCount} known warnings)`);
console.log("==================================");
if (errorCount) {
process.exit(1);