Support for international fonts, and rendering UTF8 characters that come from iOS direct in the messages app

README.md

@@ -554,6 +554,30 @@ You can use `g.setColor(r,g,b)` OR `g.setColor(16bitnumber)` - some common 16 bi
 | GreenYellow | 0xAFE5 |
 | Pink | 0xF81F |
 
+## Fonts
+
+A recent addition to Bangle.js is the ability to use extra fonts with support for more characters.
+For example [all regions](https://banglejs.com/apps/?id=fontall) or [extended](https://banglejs.com/apps/?id=fontext) fonts.
+
+Once installed, these apps cause a new font, `Intl` to be added to `Graphics`, which can be used with just `g.setFont("Intl")`.
+
+There is also a `font` library - this is not implemented yet, but more information about the planned implementation
+is available at https://github.com/espruino/BangleApps/issues/3109
+
+For now, to make your app work with the international font, you can check if `Graphics.prototype.setFontIntl` exists,
+and if so you can change the font you plan on using:
+
+```JS
+myFont = "6x8:2";
+if (Graphics.prototype.setFontIntl)
+  myFont = "Intl";
+```
+
+Any new Font library must contain the metadata `"icon": "app.png", "tags": "font", "type": "module", "provides_modules" : ["fonts"],`
+and should provide a `font` library, as well as a `boot.js` that adds `Graphics.prototype.setFontIntl`. If you plan on
+making a new library it's best to just copy one of the existing ones for now.
+
+
 ## API Reference
 
 [Reference](http://www.espruino.com/Reference#software)

apps/.eslintrc.js

@@ -164,7 +164,9 @@ module.exports = {
         "D28": "readonly",
         "D29": "readonly",
         "D30": "readonly",
-        "D31": "readonly"
+        "D31": "readonly",
+
+        "bleServiceOptions": "writable", // available in boot.js code that's called ad part of bootupdate
     },
     "parserOptions": {
         "ecmaVersion": 11

apps/fontall/ChangeLog

@@ -0,0 +1 @@
+0.01: New App!

apps/fontall/README.md

@@ -0,0 +1,22 @@
+# Fonts (all languages)
+
+This library provides an international font that can be used to display messages.
+
+The font is the 16px high [GNU Unifont](https://unifoundry.com/unifont/index.html).
+All characters from Unicode codepoint 32 up until codepoint 65535 (U+FFFF) are included here,
+which should be enough for most languages.
+
+**The font is 2MB and takes a while to upload** - if you don't require all the languages
+it provides, consider installing another Font library like [extended fonts](https://banglejs.com/apps/?id=fontsext)
+that contains just the characters you need instead.
+
+## Usage
+
+See [the BangleApps README file](https://github.com/espruino/BangleApps/blob/master/README.md#api-reference)
+for more information on fonts.
+
+
+## Recreating font.pbf
+
+* Go to `bin` directory
+* Run `./font_creator.js "All" ../apps/fontall/font.pbf`

apps/fontall/boot.js

@@ -0,0 +1 @@
+Graphics.prototype.setFontIntl = function() { return this.setFontPBF(require("Storage").read("fontall.pbf")); };

apps/fontall/lib.js

@@ -0,0 +1,3 @@
+exports.getFont = (options) => {
+  return "Intl"; // placeholder for now - see https://github.com/espruino/BangleApps/issues/3109
+};

apps/fontall/metadata.json

@@ -0,0 +1,16 @@
+{ "id": "fontall",
+  "name": "Fonts (all languages)",
+  "version":"0.01",
+  "description": "Installs a font containing over 50,000 Unifont characters for Chinese, Japanese, Korean, Russian, and more. **Requires 2MB storage**",
+  "icon": "app.png",
+  "tags": "font",
+  "type": "module",
+  "provides_modules" : ["font"],
+  "supports" : ["BANGLEJS2"],
+  "readme": "README.md",
+  "storage": [
+    {"name":"font","url":"lib.js"},
+    {"name":"fontall.boot.js","url":"boot.js"},
+    {"name":"fontall.pbf","url":"font.pbf"}
+  ]
+}

apps/fontext/ChangeLog

@@ -0,0 +1 @@
+0.01: New App!

apps/fontext/README.md

@@ -0,0 +1,25 @@
+# Fonts (extended)
+
+This library provides an international font that can be used to display messages.
+
+The font is the 16px high [GNU Unifont](https://unifoundry.com/unifont/index.html).
+All characters from Unicode codepoint 32 up until codepoint 1103 (U+044F) are included here,
+which should be enough for [around 90% of languages](https://arxiv.org/pdf/1801.07779.pdf#page=5)
+but **not** Chinese/Japanese/Korean.
+
+The font is 20kb so is far more sensible than the [2MB all regions](https://banglejs.com/apps/?id=fontsall) font
+if you don't require non-latin languages.
+
+
+https://arxiv.org/pdf/1801.07779.pdf#page=5
+
+## Usage
+
+See [the BangleApps README file](https://github.com/espruino/BangleApps/blob/master/README.md#api-reference)
+for more information on fonts.
+
+
+## Recreating font.pbf
+
+* Go to `bin` directory
+* Run `./font_creator.js "Extended" ../apps/fontext/font.pbf`

apps/fontext/boot.js

@@ -0,0 +1 @@
+Graphics.prototype.setFontIntl = function() { return this.setFontPBF(require("Storage").read("fontext.pbf")); };

apps/fontext/lib.js

@@ -0,0 +1,3 @@
+exports.getFont = (options) => {
+  return "Intl"; // placeholder for now - see https://github.com/espruino/BangleApps/issues/3109
+};

apps/fontext/metadata.json

@@ -0,0 +1,16 @@
+{ "id": "fontext",
+  "name": "Fonts (150+ languages)",
+  "version":"0.01",
+  "description": "Installs a font containing 1000 Unifont characters, which should handle the majority of non-Chinese/Japanese/Korean languages (only 20kb)",
+  "icon": "app.png",
+  "tags": "font",
+  "type": "module",
+  "provides_modules" : ["font"],
+  "supports" : ["BANGLEJS2"],
+  "readme": "README.md",
+  "storage": [
+    {"name":"font","url":"lib.js"},
+    {"name":"fontext.boot.js","url":"boot.js"},
+    {"name":"fontext.pbf","url":"font.pbf"}
+  ]
+}

apps/ios/ChangeLog

@@ -15,3 +15,4 @@
       Allow negative/positive actions to pass through to message GUI
 0.15: Enable calendar and weather updates via custom notifications (via shortcuts app)
 0.16: Always request Current Time service from iOS
+0.17: Default to passing full UTF8 strings into messages app (which can now process them with an international font)

apps/ios/README.md

@@ -9,9 +9,15 @@ prompted for immediatly after you connect the Bangle to the iPhone.
 
 ### Setting
 
-Under `Settings -> Apps -> iOS Integration` there is
+Under `Settings -> Apps -> iOS Integration` there are some settings:
-a `Time Sync` setting. This will enable syncing between the
+
-watch and iOS.
+* `Time Sync` - This will enable syncing between the watch and iOS.
+* `Disable UTF8` - As of version 0.17 of this app, text strings from iOS
+are treated as UTF8. If you install a font library like https://banglejs.com/apps/?id=fontsall
+then the messages app will be able to use that to render characters from iOS. Without fonts
+installed, non-european (ISO8859-1) characters won't be displayed. If `Disable UTF8`
+is true *or no fonts library is installed*, text from iOS is converted to ISO8859-1, and known characters with equivalents
+within that range are converted (so text will display without a font library).
 
 ### Connecting your Bangle.js to your iPhone
 

apps/ios/boot.js

@@ -124,44 +124,9 @@ E.on('notify',msg=>{
     "tv.twitch": "Twitch",
     // could also use NRF.ancsGetAppInfo(msg.appId) here
   };
-  var unicodeRemap = {
+
-    '2019':"'",
-    '260':"A",
-    '261':"a",
-    '262':"C",
-    '263':"c",
-    '268':"C",
-    '269':"c",
-    '270':"D",
-    '271':"d",
-    '280':"E",
-    '281':"e",
-    '282':"E",
-    '283':"e",
-    '321':"L",
-    '322':"l",
-    '323':"N",
-    '324':"n",
-    '327':"N",
-    '328':"n",
-    '344':"R",
-    '345':"r",
-    '346':"S",
-    '347':"s",
-    '352':"S",
-    '353':"s",
-    '356':"T",
-    '357':"t",
-    '377':"Z",
-    '378':"z",
-    '379':"Z",
-    '380':"z",
-    '381':"Z",
-    '382':"z",
-  };
-  var replacer = ""; //(n)=>print('Unknown unicode '+n.toString(16));
   //if (appNames[msg.appId]) msg.a
-  if (msg.title&&E.decodeUTF8(msg.title, unicodeRemap, replacer) === "BangleDumpCalendar") {
+  if (msg.title === "BangleDumpCalendar") {
     // parse the message body into json:
     const d = JSON.parse(msg.message);
     /* Example:
@@ -178,7 +143,7 @@ E.on('notify',msg=>{
     {t:"calendar", id:int, type:int, timestamp:seconds, durationInSeconds, title:string, description:string,location:string,calName:string.color:int,allDay:bool
     for gadgetbridge
      */
-    calEvent = {
+    let calEvent = {
       t: "calendar",
       id: parseInt(d.id),
       type: 0,
@@ -205,14 +170,14 @@ E.on('notify',msg=>{
     NRF.ancsAction(msg.uid, false);
     return;
   }
-  if (msg.title&&E.decodeUTF8(msg.title, unicodeRemap, replacer) === "BangleDumpWeather") {
+  if (msg.title === "BangleDumpWeather") {
     const d = JSON.parse(msg.message);
     /* Example:
     {"temp":"291.07","hi":"293.02","lo":"288.18","hum":"49","rain":"0","uv":"0","wind":"1.54","code":"01d","txt":"Mostly Sunny","wdir":"303","loc":"Berlin"}
     what we want:
     t:"weather", temp,hi,lo,hum,rain,uv,code,txt,wind,wdir,loc
      */
-    weatherEvent = {
+    let weatherEvent = {
         t: "weather",
         temp: d.temp,
         hi: d.hi,
@@ -236,9 +201,9 @@ E.on('notify',msg=>{
     id : msg.uid,
     src : appNames[msg.appId] || msg.appId,
     new : msg.new,
-    title : msg.title&&E.decodeUTF8(msg.title, unicodeRemap, replacer),
+    title : msg.title&&Bangle.ancsConvertUTF8(msg.title),
-    subject : msg.subtitle&&E.decodeUTF8(msg.subtitle, unicodeRemap, replacer),
+    subject : msg.subtitle&&Bangle.ancsConvertUTF8(msg.subtitle),
-    body : msg.message&&E.decodeUTF8(msg.message, unicodeRemap, replacer) || "Cannot display",
+    body : msg.message&&Bangle.ancsConvertUTF8(msg.message) || "Cannot display",
     positive : msg.positive,
     negative : msg.negative
   });
@@ -333,4 +298,46 @@ E.emit("ANCS", {
       NRF.ctsGetTime().then(ctsUpdate, function(){ /* */ })
     E.on('CTS',ctsUpdate);
   }
+  if (settings.no_utf8 || !require("Storage").read("fonts")) {
+    // if UTF8 disabled or there is no fonts lib, convert UTF8 to ISO8859-1
+    let unicodeRemap = {
+      '2019':"'",
+      '260':"A",
+      '261':"a",
+      '262':"C",
+      '263':"c",
+      '268':"C",
+      '269':"c",
+      '270':"D",
+      '271':"d",
+      '280':"E",
+      '281':"e",
+      '282':"E",
+      '283':"e",
+      '321':"L",
+      '322':"l",
+      '323':"N",
+      '324':"n",
+      '327':"N",
+      '328':"n",
+      '344':"R",
+      '345':"r",
+      '346':"S",
+      '347':"s",
+      '352':"S",
+      '353':"s",
+      '356':"T",
+      '357':"t",
+      '377':"Z",
+      '378':"z",
+      '379':"Z",
+      '380':"z",
+      '381':"Z",
+      '382':"z",
+    };
+    let replacer = ""; //(n)=>print('Unknown unicode '+n.toString(16));
+    Bangle.ancsConvertUTF8 = text => E.decodeUTF8(text, unicodeRemap, replacer);
+  } else {
+    Bangle.ancsConvertUTF8 = E.asUTF8;
+  }
 }

apps/ios/metadata.json

@@ -1,7 +1,7 @@
 {
   "id": "ios",
   "name": "iOS Integration",
-  "version": "0.16",
+  "version": "0.17",
   "description": "Display notifications/music/etc from iOS devices",
   "icon": "app.png",
   "tags": "tool,system,ios,apple,messages,notifications",

apps/ios/settings.js

@@ -12,6 +12,13 @@
         settings.timeSync = v;
         updateSettings();
       }
+    },
+    /*LANG*/"Disable UTF8" : {
+      value : !!settings.no_utf8,
+      onchange: v => {
+        settings.no_utf8 = v;
+        updateSettings();
+      }
     }
   };
   E.showMenu(mainmenu);

apps/lint_exemptions.js

@@ -1010,12 +1010,6 @@ module.exports = {
       "no-undef"
     ]
   },
-  "messagegui/app.js": {
-    "hash": "6ffb405ae2f1e62f5d1ff19888cbfd71e40850752ea8c49a1cc2e358fca7de80",
-    "rules": [
-      "no-undef"
-    ]
-  },
   "marioclock/marioclock-app.js": {
     "hash": "d46990c757fd217593c6966c82f421bcd51a4d073109dea2cbc398a0f6064602",
     "rules": [
@@ -1076,12 +1070,6 @@ module.exports = {
       "no-undef"
     ]
   },
-  "ios/boot.js": {
-    "hash": "875f34ea333f9e1c28dd0cf0c0a73ec7003bd8f03e2ed6632b7978ce7b5f5c7e",
-    "rules": [
-      "no-undef"
-    ]
-  },
   "infoclk/settings.js": {
     "hash": "56adc3eff3cbc04dd08238ed7e559416ebbc7736c872070c757d70bf5f31b440",
     "rules": [

apps/messagegui/ChangeLog

@@ -103,4 +103,4 @@
 0.74: Add option for driving on left (affects roundabout icons in navigation)
 0.75: Handle text with images in messages list by just displaying the first line
 0.76: Swipe up/down on a shown message to show the next newer/older message.
-
+0.77: Messages can now use international fonts if they are installed

apps/messagegui/app.js

@@ -22,12 +22,27 @@ GB({t:"nav",src:"maps",title:"Navigation",instr:"Main St / I-29 ALT / Centerpoin
 require("messages").pushMessage({"t":"add","id":"call","src":"Phone","title":"Bob","body":"12421312",positive:true,negative:true})
 */
 var Layout = require("Layout");
+var layout; // global var containing the layout for the currently displayed message
 var settings = require('Storage').readJSON("messages.settings.json", true) || {};
 var fontSmall = "6x8";
 var fontMedium = g.getFonts().includes("6x15")?"6x15":"6x8:2";
 var fontBig = g.getFonts().includes("12x20")?"12x20":"6x8:2";
 var fontLarge = g.getFonts().includes("6x15")?"6x15:2":"6x8:4";
 var fontVLarge = g.getFonts().includes("6x15")?"12x20:2":"6x8:5";
+
+// If a font library is installed, just switch to using that for everything in messages
+if (Graphics.prototype.setFontIntl) {
+  fontSmall = "Intl";
+  fontMedium = "Intl";
+  fontBig = "Intl";
+  /* 2v21 and before have a bug where the scale factor for PBF fonts wasn't
+  taken into account in metrics, so we can't have big fonts on those firmwares.
+  Having 'PBF' listed as a font was a bug fixed at the same time so we check for that. */
+  let noScale = g.getFonts().includes("PBF");
+  fontLarge = noScale?"Intl":"Intl:2";
+  fontVLarge = noScale?"Intl":"Intl:3";
+}
+
 var active; // active screen (undefined/"list"/"music"/"map"/"message"/"scroller"/"settings")
 var openMusic = false; // go back to music screen after we handle something else?
 // hack for 2v10 firmware's lack of ':size' font handling
@@ -475,9 +490,9 @@ function checkMessages(options) {
       if (title) g.setFontAlign(-1,-1).setFont(fontBig).drawString(title, x,r.y+2);
       var longBody = false;
       if (body) {
-        g.setFontAlign(-1,-1).setFont("6x8");
+        g.setFontAlign(-1,-1).setFont(fontSmall);
         // if the body includes an image, it probably won't be small enough to allow>1 line
-        let maxLines = 3, pady = 0;
+        let maxLines = Math.floor(34/g.getFontHeight()), pady = 0;
         if (body.includes("\0")) { maxLines=1; pady=4; }
         var l = g.wrapString(body, r.w-(x+14));
         if (l.length>maxLines) {

apps/messagegui/metadata.json

@@ -2,7 +2,7 @@
   "id": "messagegui",
   "name": "Message UI",
   "shortName": "Messages",
-  "version": "0.76",
+  "version": "0.77",
   "description": "Default app to display notifications from iOS and Gadgetbridge/Android",
   "icon": "app.png",
   "type": "app",

bin/README.md

@@ -6,6 +6,7 @@ BangleApps Utilities
 * `find_banglejs1_only_apps.sh` - show apps that only work on Bangle.js 1 (and not 2)
 * `firmwaremaker_c.js` - create the binary blob needed for the Bangle.js firmware (containing default apps)
 * `pre-publish.sh` - this is run before we publish to https://banglejs.com/apps/ - it works out how recently all the apps were updated and writes it to `appdates.csv`
+* `font_creator.js` - creates PBF-format fonts for font libraries like `apps/fontsall`
 
 **You should also check out https://github.com/espruino/EspruinoAppLoaderCore/tree/master/tools** (available in `core/tools` in this repo) - this contains tools for handling languages, as well as a command-line based app loader
 

bin/font_creator.js

@@ -0,0 +1,50 @@
+#!/usr/bin/node
+// Used for apps/fontsall/etc
+// Needs 'npm install pngjs'
+
+var FONTFILE = "unifont-15.1.05.png";
+
+console.log("Espruino Font Creator");
+console.log("---------------------");
+console.log("");
+
+let fontconverter = require("../webtools/fontconverter.js");
+let charCodeRanges = fontconverter.getRanges();
+console.log("Available char code ranges\n  - "+Object.keys(charCodeRanges).join("\n  - "));
+
+
+if (process.argv.length!=4) {
+  console.log(process.argv,"");
+  console.log("USAGE:");
+  console.log("  font_creator 'CharCodeRange' outputfile.pbf");
+  process.exit(1);
+}
+let charCodeRange = process.argv[2];
+let outputFile = process.argv[3];
+
+if (!(charCodeRange in charCodeRanges)) {
+  console.log("Char code range "+charCodeRange+" not found");
+  process.exit(1);
+}
+
+if (!require("fs").existsSync(FONTFILE)) {
+  console.log("Unifont file "+FONTFILE+" not found!")
+  console.log("Download from https://unifoundry.com/unifont/index.html and convert to png")
+  process.exit(1);
+}
+
+// load a unifont PNG file
+let font = fontconverter.load({
+  fn : FONTFILE,
+  mapWidth : 256, mapHeight : 256,
+  mapOffsetX : 32, mapOffsetY : 64,
+  height : 16, // actual used height of font map
+  range : charCodeRanges[charCodeRange].range
+});
+font.removeUnifontPlaceholders();
+// quick hack as space looks too long
+font.glyphs[32].width -= 3;
+font.glyphs[32].xEnd -= 3;
+font.glyphs[32].advance -= 3;
+//font.debugChars();
+require("fs").writeFileSync(outputFile, Buffer.from(font.getPBF()));

webtools

@@ -1 +1 @@
-Subproject commit 8d671ad0dfb1d5a36f4ee9952390f4d79019e61d
+Subproject commit 7e4283948f68b1dd14a59a73e544c537a26c800a