Merge pull request #3002 from pavelmachek/m_10_spacew

Prepare version 0.02 of Space Weaver

apps/spacew/README.md

@@ -38,6 +38,8 @@ Directories at the end of .mtar should be hashed, not linear searched.
 
 Geojson is not really suitable as it takes a lot of storage.
 
-It would be nice to support polygons.
-
 Web-based tool for preparing maps would be nice.
+
+Storing 12bit coordinates, but only using 8bits.
+
+Polygons should go first to get proper z-order. 

apps/spacew/app.js

@@ -386,7 +386,7 @@ function emptyMap() {
       m.scale = 2;
       g.reset().clearRect(R);
       redraw(18);
-      print("Benchmark done (31 sec)");
+      print("Benchmark done");
     }
     };
     if (fix.fix) menu[/*LANG*/"Center GPS"]=() =>{
@@ -400,36 +400,52 @@ function emptyMap() {
 
 var gjson = null;
 
+function stringFromArray(data) {
+  var count = data.length;
+  var str = "";
+
+  for(var index = 0; index < count; index += 1)
+    str += String.fromCharCode(data[index]);
+
+  return str;
+}
+
+const st = require('Storage');
+const hs = require('heatshrink');
+
 function readTarFile(tar, f) {
-  const st = require('Storage');
+  let json_off = st.read(tar, 0, 16) * 1;
-  json_off = st.read(tar, 0, 16) * 1;
   if (isNaN(json_off)) {
     print("Don't have archive", tar);
     return undefined;
   }
   while (1) {
-    json_len = st.read(tar, json_off, 6) * 1;
+    let json_len = st.read(tar, json_off, 6) * 1;
     if (json_len == -1)
       break;
     json_off += 6;
-    json = st.read(tar, json_off, json_len);
+    let json = st.read(tar, json_off, json_len);
     //print("Have directory, ", json.length, "bytes");
-  //print(json);
+    let files = JSON.parse(json);
-    files = JSON.parse(json);
+    let rec = files[f];
-  //print(files);
+    if (rec) {
-    rec = files[f];
+      let cs = st.read(tar, rec.st, rec.si);
-    if (rec)
+      if (rec.comp == "hs") {
-      return st.read(tar, rec.st, rec.si);
+        let d = stringFromArray(hs.decompress(cs));
+        //print("Decompressed", d);
+        return d;
+      }
+      return cs;
+    }
     json_off += json_len;
   }
   return undefined;
 }
-
 function loadVector(name) {
   var t1 = getTime();
   print(".. Read", name);
     //s = require("Storage").read(name);
-  var s = readTarFile("delme.mtar", name);
+  var s = readTarFile("world.mtar", name);
   if (s == undefined) {
     print("Don't have file", name);
     return null;
@@ -438,8 +454,7 @@ function loadVector(name) {
   print(".... Read and parse took ", getTime()-t1);
   return r;
 }
-
+function drawPoint(a) {   /* FIXME: let... */
-function drawPoint(a) {  
   lon = a.geometry.coordinates[0];
   lat = a.geometry.coordinates[1];
   
@@ -459,7 +474,6 @@ function drawPoint(a) {
   g.drawString(a.properties.name, p.x, p.y);
   points ++;
 }
-
 function drawLine(a, qual) {
   lon = a.geometry.coordinates[0][0];
   lat = a.geometry.coordinates[0][1];
@@ -485,38 +499,196 @@ function drawLine(a, qual) {
       i = len-1;
     points ++;
     p1 = p2;
-    g.flip();
   }
 }
+function drawPolygon(a, qual) {
+  lon = a.geometry.coordinates[0][0];
+  lat = a.geometry.coordinates[0][1];
+  i = 1;
+  step = 1;
+  len = a.geometry.coordinates.length;
+  if (len > 62) {
+    step = log2(len) - 5;
+    step = 1<<step;
+  }
+  step = step * qual;
+  var p1 = m.latLonToXY(lat, lon);
+  let pol = [p1.x, p1.y];
+  while (i < len) {  
+    lon = a.geometry.coordinates[i][0];
+    lat = a.geometry.coordinates[i][1];
+    var p2 = m.latLonToXY(lat, lon);
 
-function drawVector(gjson, qual) {
+    pol.push(p2.x, p2.y);
+    if (i == len-1)
+      break;
+    i = i + step;
+    if (i>len)
+      i = len-1;
+    points ++;
+  }
+  if (a.properties.fill) {
+    g.setColor(a.properties.fill);
+  } else {
+    g.setColor(.75, .75, 1);
+  }
+  g.fillPoly(pol, true);
+  if (a.properties.stroke) {
+    g.setColor(a.properties.stroke);
+  } else {
+    g.setColor(0,0,0)
+  }
+  g.drawPoly(pol, true);
+}
+
+function toScreen(tile, xy) {
+//  w, s, e, n, (x,y in 0..4096 range)
+  let x = xy[0];
+  let y = xy[1];
+  let r = {};
+  r.x = ((x/4096) * (tile[2]-tile[0])) + tile[0];
+  r.y = ((1-(y/4096)) * (tile[3]-tile[1])) + tile[1];
+  return r;
+}
+var d_off = 1;
+function getBin(bin, i, prev) {
+  let x = bin[i*3 + d_off  ]<<4;
+  let y = bin[i*3 + d_off+1]<<4;
+  //print("Point", x, y, bin);
+  return [x, y];
+}
+function getBinLength(bin) {
+  return (bin.length-d_off) / 3;
+}
+function newPoint(tile, a, rec, bin) {  
+  var p = toScreen(tile, getBin(bin, 0, null));
+  var sz = 2;
+  if (a.properties) {
+    if (a.properties["marker-color"]) {
+      g.setColor(a.properties["marker-color"]);
+    }
+    if (a.properties.marker_size == "small")
+      sz = 1;
+    if (a.properties.marker_size == "large")
+      sz = 4;
+  }
+  
+  g.fillRect(p.x-sz, p.y-sz, p.x+sz, p.y+sz);
+  if (rec.tags) {
+    g.setColor(0,0,0);
+    g.setFont("Vector", 18).setFontAlign(-1,-1);
+    g.drawString(rec.tags.name, p.x, p.y);
+  }
+  points ++;
+}
+function newLine(tile, a, bin) {
+  let xy = getBin(bin, 0, null);
+  let i = 1;
+  let step = 1;
+  let len = getBinLength(bin);
+  let p1 = toScreen(tile, xy);
+  if (a.properties && a.properties.stroke) {
+    g.setColor(a.properties.stroke);
+  }
+  while (i < len) {  
+    xy = getBin(bin, i, xy);
+    var p2 = toScreen(tile, xy);
+
+    //print(p1.x, p1.y, p2.x, p2.y);
+    g.drawLine(p1.x, p1.y, p2.x, p2.y);
+    if (i == len-1)
+      break;
+    i = i + step;
+    if (i>len)
+      i = len-1;
+    points ++;
+    p1 = p2;
+  }
+}
+function newPolygon(tile, a, bin) {
+  let xy = getBin(bin, 0, null);
+  i = 1;
+  step = 1;
+  len = getBinLength(bin);
+  if (len > 62) {
+    step = log2(len) - 5;
+    step = 1<<step;
+  }
+  var p1 = toScreen(tile, xy);
+  let pol = [p1.x, p1.y];
+  while (i < len) {  
+    xy = getBin(bin, i, xy); // FIXME... when skipping
+    var p2 = toScreen(tile, xy);
+
+    pol.push(p2.x, p2.y);
+    if (i == len-1)
+      break;
+    i = i + step;
+    if (i>len)
+      i = len-1;
+    points ++;
+  }
+  if (a.properties && a.properties.fill) {
+    g.setColor(a.properties.fill);
+  } else {
+    g.setColor(.75, .75, 1);
+  }
+  g.fillPoly(pol, true);
+  if (a.properties && a.properties.stroke) {
+    g.setColor(a.properties.stroke);
+  } else {
+    g.setColor(0,0,0)
+  }
+  g.drawPoly(pol, true);
+}
+function newVector(tile, rec) {
+  let bin = E.toUint8Array(atob(rec.b));
+  a = meta.attrs[bin[0]];  
+  if (a.type == 1) {
+    newPoint(tile, a, rec, bin);
+  } else if (a.type == 2) {
+    newLine(tile, a, bin);
+  } else if (a.type == 3) {
+    newPolygon(tile, a, bin);
+  } else print("Unknown record", a);
+  g.flip();
+}
+function drawVector(gjson, tile, qual) {
   var d = gjson;
   points = 0;
   var t1 = getTime();
   
-  for (var a of d.features) {
+  let xy1 = m.latLonToXY(tile[1], tile[0]);
-    if (a.type != "Feature")
+  let xy2 = m.latLonToXY(tile[3], tile[2]);
-        print("Expecting feature");
+  let t2 = [ xy1.x, xy1.y, xy2.x, xy2.y ];
+  print(t2);
+  
+  for (var a of d) { // d.features for geojson
     g.setColor(0,0,0);
+    if (a.type != "Feature") {
+      newVector(t2, a);
+      continue;
+    }
     // marker-size, marker-color, stroke
     if (qual < 32 && a.geometry.type == "Point")
       drawPoint(a);
     if (qual < 8 && a.geometry.type == "LineString")
       drawLine(a, qual);
+    if (qual < 8 && a.geometry.type == "Polygon")
+      drawPolygon(a, qual);    
   }
   print("....", points, "painted in", getTime()-t1, "sec");
 }
-
 function fname(lon, lat, zoom) {
     var bbox = [lon, lat, lon, lat];
     var r = xyz(bbox, 13, false, "WGS84");
     //console.log('fname', r);
     return 'z'+zoom+'-'+r.minX+'-'+r.minY+'.json';
 }
-
 function fnames(zoom) {
     var bb = [m.lon, m.lat, m.lon, m.lat];
     var r = xyz(bb, zoom, false, "WGS84");
+    let maxt = 16;
     while (1) {
       var bb2 = bbox(r.minX, r.minY, zoom, false, "WGS84");
       var os = m.latLonToXY(bb2[3], bb2[0]);
@@ -525,6 +697,9 @@ function fnames(zoom) {
       else if (os.y >= 0)
         r.minY -= 1;
       else break;
+      if (!maxt)
+        break;
+      maxt--;
     }
     while (1) {
       var bb2 = bbox(r.maxX, r.maxY, zoom, false, "WGS84");
@@ -534,13 +709,16 @@ function fnames(zoom) {
       else if (os.y <= g.getHeight())
         r.maxY += 1;
       else break;
+      if (!maxt)
+        break;
+      maxt--;
     }
+  if (!maxt)
+    print("!!! Too many tiles, not painting some");
   print(".. paint range", r);
   return r;
 }
-
 function log2(x) { return Math.log(x) / Math.log(2); }
-
 function getZoom(qual) {
   var z = 16-Math.round(log2(m.scale));
   z += qual;
@@ -551,7 +729,6 @@ function getZoom(qual) {
     return meta.max_zoom;
   return z;
 }
-
 function drawDebug(text, perc) {
   g.setClipRect(0,0,R.x2,R.y);
   g.reset();
@@ -564,7 +741,6 @@ function drawDebug(text, perc) {
   g.setClipRect(R.x,R.y,R.x2,R.y2);
   g.flip();
 }
-
 function drawAll(qual) {
   var zoom = getZoom(qual);
   var t1 = getTime();
@@ -583,7 +759,7 @@ function drawAll(qual) {
       var n ='z'+zoom+'-'+x+'-'+y+'-'+cnt+'.json';
       var gjson = loadVector(n);
       if (!gjson) break;
-      drawVector(gjson, 1);
+      drawVector(gjson, bbox(x, y, zoom, false, "WGS84"), 1);
     }
     num++;
     drawDebug("Zoom "+zoom+" tiles "+num+"/"+tiles, num/tiles);
@@ -611,7 +787,7 @@ function introScreen() {
 }
 
 
-m.scale = 76;
+m.scale = 76000;
 m.lat = 50.001;
 m.lon = 14.759;
 

apps/spacew/bench.js

@@ -0,0 +1,126 @@
+R = Bangle.appRect;
+
+function introScreen() {
+  g.reset().clearRect(R);
+  g.setColor(0,0,0).setFont("Vector",25);
+  g.setFontAlign(0,0);
+  g.drawString("Benchmark", 85,35);
+  g.setColor(0,0,0).setFont("Vector",18);
+  g.drawString("Press button", 85,55);
+}
+function lineBench() {
+  /* 500 lines a second on hardware, 125 lines with flip */
+  for (let i=0; i<1000; i++) {
+    let x1 = Math.random() * 160;
+    let y1 = Math.random() * 160;
+    let x2 = Math.random() * 160;
+    let y2 = Math.random() * 160;
+    
+    g.drawLine(x1, y1, x2, y2);
+    //g.flip();
+  }
+}
+function polyBench() {
+  /* 275 hollow polygons a second on hardware, 99 with flip */
+  /* 261 filled polygons a second on hardware, 99 with flip */
+  for (let i=0; i<1000; i++) {
+    let x1 = Math.random() * 160;
+    let y1 = Math.random() * 160;
+    let x2 = Math.random() * 160;
+    let y2 = Math.random() * 160;
+    let c = Math.random();
+    
+    g.setColor(c, c, c);
+    g.fillPoly([80, x1, y1, 80, 80, x2, y2, 80], true);
+    //g.flip();
+  }
+}
+function checksum(d) {
+  let sum = 0;
+  for (i=0; i<d.length; i++) {
+    sum += (d[i]*1);
+  }
+  return sum;
+}
+function linearRead() {
+  /* 10000b block -> 8.3MB/sec, 781..877 IOPS
+      1000b block -> 920K/sec, 909 IOPS, 0.55 sec
+       100b block -> 100K/sec
+        10b block -> 10K/sec, 1020 IOPS, 914 IOPS with ops counting
+        
+      1000b block backwards -- 0.59 sec.
+       100b block -- 5.93.
+                  backwards -- 6.27
+                  random -- 7.13
+     checksum 5.97 -> 351 seconds with checksum. 1400bytes/second
+   */
+     
+  let size = 500000;
+  let block = 100;
+  let i = 0;
+  let ops = 0;
+  let sum = 0;
+  while (i < size) {
+    //let pos = Math.random() * size;
+    let pos = i;
+    //let pos = size-i;
+    let d = require("Storage").read("delme.mtar", pos, block);
+    //sum += checksum(E.toUint8Array(d));
+    i += block;
+    ops ++;
+  }
+  print(ops, "ops", sum);
+}
+function drawBench(name) {
+  g.setColor(0,0,0).setFont("Vector",25);
+  g.setFontAlign(0,0);
+  g.drawString(name, 85,35);
+  g.setColor(0,0,0).setFont("Vector",18);
+  g.drawString("Running", 85,55);
+  g.flip();
+}
+function runBench(b, name) {
+  drawBench(name);
+  g.reset().clearRect(R);
+
+  let t1 = getTime();
+  print("--------------------------------------------------");
+  print("Running",name);
+  b();
+  let m = (getTime()-t1) + " sec";
+  print("..done in", m);
+  drawBench(name);
+  g.setColor(0,0,0).setFont("Vector",18);
+  g.drawString(m, 85,85);
+
+}
+function redraw() {
+  //runBench(lineBench, "Lines");
+  runBench(polyBench, "Polygons");
+  //runBench(linearRead, "Linear read");
+}
+function showMap() {
+  g.reset().clearRect(R);
+  redraw();
+  emptyMap();
+}
+function emptyMap() {
+  Bangle.setUI({mode:"custom",drag:e=>{
+      g.reset().clearRect(R);
+      redraw();    
+  }, btn: btn=>{
+    mapVisible = false;
+    var menu = {"":{title:"Benchmark"},
+    "< Back": ()=> showMap(),
+    /*LANG*/"Run": () =>{
+      showMap();
+    }};
+    E.showMenu(menu);
+  }});
+}
+
+const st = require('Storage');
+const hs = require('heatshrink');
+
+introScreen();
+emptyMap();

apps/spacew/metadata.json

@@ -1,6 +1,6 @@
 { "id": "spacew",
   "name": "Space Weaver",
-  "version":"0.01",
+  "version":"0.02",
   "description": "Application for displaying vector maps",
   "icon": "app.png",
   "readme": "README.md",
@@ -8,6 +8,7 @@
   "tags": "outdoors,gps,osm",
   "storage": [
       {"name":"spacew.app.js","url":"app.js"},
-      {"name":"spacew.img","url":"app-icon.js","evaluate":true}
+      {"name":"spacew.img","url":"app-icon.js","evaluate":true},
+      {"name":"world.mtar","url":"world.mtar"}
   ]
 }

apps/spacew/prep/minitar.js

@@ -1,7 +1,8 @@
 #!/usr/bin/nodejs
 
+// https://stackoverflow.com/questions/49129643/how-do-i-merge-an-array-of-uint8arrays
+
 var pc = 1;
-var hack = 0;
 const hs = require('./heatshrink.js');
 
 if (pc) {
@@ -23,19 +24,21 @@ function writeTar(tar, dir) {
     var h_len = 16;
     var cur = h_len;
     files = fs.readdirSync(dir);
-    data = '';
+    let data = [];
     var directory = '';
     var json = {};
     for (f of files) {
+        let f_rec = {};
         d = fs.readFileSync(dir+f);
+        if (0) {
+            cs = hs.compress(d);
+            f_rec.comp = "hs";
+        } else
             cs = d;
-        //cs = String.fromCharCode.apply(null, hs.compress(d))
         print("Processing", f, cur, d.length, cs.length);
-        //if (d.length == 42) continue;
+        data.push(cs);
-        data = data + cs;
-        var f_rec = {};
         f_rec.st = cur;
-        var len = d.length;
+        var len = cs.length;
         f_rec.si = len;
         cur = cur + len;
         json[f] = f_rec;
@@ -53,10 +56,10 @@ function writeTar(tar, dir) {
     while (header.length < h_len) {
         header = header+' ';
     }
-    if (!hack)
+    fs.writeFileSync(tar, header);
-        fs.writeFileSync(tar, header+data+directory);
+    for (d of data)
-    else
+        fs.appendFileSync(tar, Buffer.from(d));
-        fs.writeFileSync(tar, directory);
+    fs.appendFileSync(tar, directory);
 }
 
 function readTarFile(tar, f) {
@@ -70,7 +73,7 @@ function readTarFile(tar, f) {
 }
 
 if (pc)
-  writeTar("delme.mtaz", "delme/");
+  writeTar("delme.mtar", "delme/");
 else {
   print(readTarFile("delme.mtar", "ahoj"));
   print(readTarFile("delme.mtar", "nazdar"));

apps/spacew/prep/prepare.json

@@ -14,5 +14,5 @@
     "linear_tags":  true,
     "area_tags":    false,
     "exclude_tags": [],
-    "include_tags": [ "place", "name", "landuse", "highway" ]
+    "include_tags": [ "place", "name", "landuse", "highway", "natural" ]
 }

apps/spacew/prep/prepare.sh

@@ -1,17 +1,49 @@
 #!/bin/bash
-if [ ".$1" == "-f" ]; then
+
+# http://bboxfinder.com/#0.000000,0.000000,0.000000,0.000000
+Z=
+# Czech republic -- hitting internal limit in nodejs
+#BBOX=10,60,20,30
+#Z="--maxz 9"
+# No Moravia -- ascii conversion takes 43min, "Error: Cannot create a string longer than 0x3fffffe7 characters"
+#BBOX=10,60,17.75,30
+#Z="--maxz 9"
+# Just Moravia -- 266MB delme.pbf -- FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed
+#BBOX=16.13,60,35,30
+#Z="--maxz 9"
+# Roudnice az Kutna hora -- band 1.1 deg -- 145MB delme.pbf, 5m cstocs, 40+m split.
+#                        -- band 0.1 deg -- 13MB delme.pbf, 13s split, 21k result.
+#BBOX=14.20,50.45,15.32,49.20
+#Z="--maxz 9"
+# Roudnice az... -- band 0.5 deg --  91MB delme.pbf, 200k result.
+# Roudnice az... -- band 0.8 deg -- 120MB delme.pbf, 2.5GB while splitting, 260k result.
+#BBOX=14.20,50.45,15.0,49.20
+#Z="--maxz 9"
+# Prague; 1.2MB map, not really useful
+#BBOX=14.25,50.17,14.61,49.97
+#Z="--maxz 14"
+# Zernovka small -- 3.5 delme.pbf, ~850K result.
+BBOX=14.7,49.9,14.8,50.1
+# Zernovka big
+#BBOX=14.6,49.7,14.9,50.1
+
+if [ ".$1" == ".-f" ]; then
     I=/data/gis/osm/dumps/czech_republic-2023-07-24.osm.pbf
     #I=/data/gis/osm/dumps/zernovka.osm.bz2
     O=cr.geojson
     rm delme.pbf $O
-    time osmium extract $I --bbox 14.7,49.9,14.8,50.1 -f pbf -o delme.pbf
+    ls -alh $I
+    time osmium extract $I --bbox $BBOX -f pbf -o delme.pbf
+    ls -alh delme.pbf
     time osmium export delme.pbf -c prepare.json -o $O
+    ls -alh $O
+    # ~.5G in 15min
     echo "Converting to ascii"
     time cstocs utf8 ascii cr.geojson  > cr_ascii.geojson
     mv -f cr_ascii.geojson delme.json
 fi
 rm -r delme/; mkdir delme
-./split.js
+time ./split.js $Z
 ./minitar.js
 ls -lS delme/*.json  | head -20
 cat delme/* | wc -c

apps/spacew/prep/split.js

@@ -6,11 +6,11 @@
 
 const fs = require('fs');
 const sphm = require('./sphericalmercator.js');
-var split = require('geojson-vt')
+var split = require('geojson-vt');
+const process = require('process');
+
 
 // delme.json needs to be real file, symlink to geojson will not work
-console.log("Loading json");
-var gjs = require("./delme.json");
 
 function tileToLatLon(x, y, z, x_, y_) {
     var [ w, s, e, n ] = merc.bbox(x, y, z);
@@ -30,9 +30,39 @@ function convGeom(tile, geom) {
     return g;
 }
 
+function clamp(i) {
+    if (i<0)
+        return 0;
+    if (i>4095)
+        return 4095;
+    return i;
+}
+
+function binGeom(tile, geom) {
+    let off = 1;
+    let r = new Uint8Array(geom.length * 3 + off);
+    let j = off;
+    for (i = 0; i< geom.length; i++) {
+        let x = geom[i][0];
+        let y = geom[i][1];
+        x = clamp(x);
+        y = clamp(y);
+        r[j++] = x >> 4;
+        r[j++] = y >> 4;
+        r[j++] = (x & 0x0f) + ((y & 0x0f) << 4);
+    }
+
+    return r;
+}
+
 function zoomPoint(tags) {
     var z = 99;
 
+    if (tags.featurecla == "Admin-0 scale ranksscalerank") z = 2;
+    if (tags.featurecla == "Admin-0 capital") z = 3;
+    if (tags.featurecla == "Admin-1 capital") z = 4;
+    if (tags.featurecla == "Populated place") z = 5;
+
     if (tags.place == "city") z = 4;
     if (tags.place == "town") z = 8;
     if (tags.place == "village") z = 10;
@@ -40,10 +70,59 @@ function zoomPoint(tags) {
     return z;
 }
 
+var meta = {};
+var ac = -1;
+meta.attrs = [];
+var a_town = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 1;
+var a_village = ++ac
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 1;
+meta.attrs[ac].properties = {};
+meta.attrs[ac].properties["marker-color"] = "#800000";
+var a_way = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 2;
+var a_secondary = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 2;
+meta.attrs[ac].properties = {};
+meta.attrs[ac].properties.stroke = "#000040";
+var a_tertiary = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 2;
+meta.attrs[ac].properties = {};
+meta.attrs[ac].properties.stroke = "#000080";
+var a_track = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 2;
+meta.attrs[ac].properties = {};
+meta.attrs[ac].properties.stroke = "#404040";
+var a_path = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 2;
+meta.attrs[ac].properties = {};
+meta.attrs[ac].properties.stroke = "#408040";
+var a_polygon = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].type = 3;
+var a_forest = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].properties = {};
+meta.attrs[ac].properties.fill = "#c0ffc0";
+meta.attrs[ac].type = 3;
+var a_water = ++ac;
+meta.attrs[ac] = {};
+meta.attrs[ac].properties = {};
+meta.attrs[ac].properties.fill = "#c0c0ff";
+meta.attrs[ac].type = 3;
+
 function paintPoint(tags) {
     var p = {};
 
-    if (tags.place == "village") p["marker-color"] = "#ff0000";
+    if (tags.place == "city" || tags.place == "town") { p.attr = a_town; }
+    if (tags.place == "village") { p.attr = a_village; p["marker-color"] = "#ff0000"; }
 
     return p;
 }
@@ -51,15 +130,17 @@ function paintPoint(tags) {
 function zoomWay(tags) {
     var z = 99;
 
+    if (tags.scalerank == 0) z = 0;
+
     if (tags.highway == "motorway") z = 7;
     if (tags.highway == "primary") z = 9;
     if (tags.highway == "secondary") z = 13;
     if (tags.highway == "tertiary") z = 14;
-    if (tags.highway == "unclassified") z = 16;
+    if (tags.highway == "unclassified") z = 15;
-    if (tags.highway == "residential") z = 17;
+    if (tags.highway == "residential") z = 15;
-    if (tags.highway == "track") z = 17;
+    if (tags.highway == "track") z = 15;
-    if (tags.highway == "path") z = 17;
+    if (tags.highway == "path") z = 16;
-    if (tags.highway == "footway") z = 17;
+    if (tags.highway == "footway") z = 16;
 
     return z;
 }
@@ -67,52 +148,120 @@ function zoomWay(tags) {
 function paintWay(tags) {
     var p = {};
 
+    p.attr = a_way;
     if (tags.highway == "motorway" || tags.highway == "primary") /* ok */;
-    if (tags.highway == "secondary" || tags.highway == "tertiary") p.stroke = "#0000ff";
+    if (tags.highway == "secondary" || tags.highway == "tertiary") { p.stroke = "#0000ff"; p.attr = a_secondary; }
-    if (tags.highway == "tertiary" || tags.highway == "unclassified" || tags.highway == "residential") p.stroke = "#00ff00";
+    if (tags.highway == "tertiary" || tags.highway == "unclassified" || tags.highway == "residential") { p.stroke = "#00ff00"; p.attr = a_tertiary; }
-    if (tags.highway == "track") p.stroke = "#ff0000";
+    if (tags.highway == "track") { p.stroke = "#ff0000"; p.attr = a_track; }
-    if (tags.highway == "path" || tags.highway == "footway") p.stroke = "#800000";
+    if (tags.highway == "path" || tags.highway == "footway") { p.stroke = "#800000"; p.attr = a_path; }
+
+    return p;
+}
+
+function zoomPolygon(tags) {
+    var z = 99;
+
+    if (tags.scalerank == 0) z = 0;
+
+    if (tags.landuse == "forest") z = 16;
+    if (tags.natural == "water") z = 16;
+
+    return z;
+}
+
+function paintPolygon(tags) {
+    var p = {};
+
+    p.attr = a_polygon;
+    if (tags.landuse == "forest") { p.fill = "#c0ffc0"; p.attr = a_forest; }
+    if (tags.natural == "water") { p.fill = "#c0c0ff"; p.attr = a_water; }
+
+    if (tags.featurecla == "Admin-0 sovereignty") p.attr = a_way;
 
     return p;
 }
 
 function writeFeatures(name, feat)
 {
+    if (0) {
         var n = {};
         n.type = "FeatureCollection";
         n.features = feat;
     
         fs.writeFile(name+'.json', JSON.stringify(n), on_error);
+    } else {
+        if (feat.length > 0)
+            fs.writeFile(name+'.json', JSON.stringify(feat), on_error);
+    }
 }
 
+function btoa(s) {
+    return Buffer.from(s).toString('base64');
+}
+
+// E.toString()
+
 function toGjson(name, d, tile) {
     var cnt = 0;
     var feat = [];
     for (var a of d) {
-        var f = {};
+        let f = {};        // geojson output
+        let b = {};     // moving towards binary output 
         var zoom = 99;
         var p = {};
+        var bin = [];
+        if (!a.tags)
+            a.tags = a.properties;
         f.properties = a.tags;
         f.type = "Feature";
         f.geometry = {};
         if (a.type == 1) {
             f.geometry.type = "Point";
             f.geometry.coordinates = convGeom(tile, a.geometry)[0];
+            bin = binGeom(tile, a.geometry);
             zoom = zoomPoint(a.tags);
             p = paintPoint(a.tags);
         } else if (a.type == 2) {
             f.geometry.type = "LineString";
             f.geometry.coordinates = convGeom(tile, a.geometry[0]);
+            bin = binGeom(tile, a.geometry[0]);
             zoom = zoomWay(a.tags);
             p = paintWay(a.tags);            
+            if (zoom == 99) {
+                f.geometry.type = "Polygon";
+                zoom = zoomPolygon(a.tags);
+                p = paintPolygon(a.tags);
+            }
+        } else if (a.type == 3) {
+            f.geometry.type = "Polygon";
+            f.geometry.coordinates = convGeom(tile, a.geometry[0]);
+            bin = binGeom(tile, a.geometry[0]);
+            zoom = zoomPolygon(a.tags);
+            p = paintPolygon(a.tags);
         } else {
-            //console.log("Unknown type", a.type);
+            console.log("Unknown type", a.type);
         }
         //zoom -= 4; // Produces way nicer map, at expense of space.
         if (tile.z < zoom)
             continue;
         f.properties = Object.assign({}, f.properties, p);
-        feat.push(f);
+        //feat.push(f); FIXME
+        bin[0] = p.attr;
+        b.b = btoa(bin);
+        b.tags = {};
+        if (a.tags.name)
+            b.tags.name = a.tags.name;
+        if (a.tags.nameascii)
+            b.tags.name = a.tags.nameascii;
+        if (a.tags.sr_subunit)
+            b.tags.name = a.tags.sr_subunit;
+        
+        //delete(a.tags.highway);
+        //delete(a.tags.landuse);
+        //delete(a.tags.natural);
+        //delete(a.tags.place);
+        //        b.properties = p
+        feat.push(b);
         var s = JSON.stringify(feat);
         if (s.length > 6000) {
             console.log("tile too big, splitting", cnt);
@@ -135,16 +284,41 @@ var merc = new sphm({
 });
 
 console.log("Splitting data");
-var meta = {}
 meta.min_zoom = 0;
-meta.max_zoom = 17; // HERE
+meta.max_zoom = 16; // HERE
                  // = 16 ... split3 takes > 30 minutes
                  // = 13 ... 2 minutes
+if (process.argv[2] == "-h") {
+    console.log("help here");
+    process.exit(0);
+}
+if (process.argv[2] == "--maxz") {
+    meta.max_zoom = 1*process.argv[3];
+    console.log("... max zoom", meta.max_zoom);
+}
+if (process.argv[2] == "--world") {
+    console.log("Loading world");
+    meta.max_zoom = 4;
+    var g_sovereign = require("./ne_10m_admin_0_sovereignty.json");
+    var g_labels = require("./ne_10m_admin_0_label_points.json");
+    var g_places = require("./ne_10m_populated_places_simple.json");
+
+    gjs = {}
+    gjs.type = "FeatureCollection";
+    //gjs.features = g_sovereign.features + g_labels.features + g_places.features;
+    gjs.features = g_sovereign.features.concat(g_labels.features).concat(g_places.features);
+    console.log(gjs);
+} else {
+    console.log("Loading json");
+    gjs = require("./delme.json");
+}
+
 var index = split(gjs, Object.assign({
     maxZoom: meta.max_zoom,
     indexMaxZoom: meta.max_zoom,
     indexMaxPoints: 0,
-    tolerance: 30,
+    tolerance: 10,
+    buffer: 0,
 }), {});
 console.log("Producing output");
 

apps/spacew/prep/stats.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 zoom() {
-    echo "Zoom $1"
+    VAL=`cat delme/z$1-* | wc -c`
-    cat delme/z$1-* | wc -c
+    echo "Zoom $1 -- " $[$VAL/1024]
-    echo "M..k..."
 }
 
 echo "Total data"
@@ -16,7 +15,17 @@ zoom 14
 zoom 13
 zoom 12
 zoom 11
-zoom 10
+zoom 9
+zoom 8
+zoom 7
+zoom 6
+zoom 5
+zoom 4
+zoom 3
+zoom 2
+zoom 1
+zoom 0
+
 echo "Zoom 1..9"
 cat delme/z?-* | wc -c
 echo "M..k..."

apps/spacew/prep/world.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+if [ ".$1" == ".-f" ]; then
+    wget https://raw.githubusercontent.com/martynafford/natural-earth-geojson/master/10m/cultural/ne_10m_admin_0_sovereignty.json
+    wget https://raw.githubusercontent.com/martynafford/natural-earth-geojson/master/10m/cultural/ne_10m_admin_0_label_points.json
+    wget https://raw.githubusercontent.com/martynafford/natural-earth-geojson/master/10m/cultural/ne_10m_populated_places_simple.json
+fi
+rm delme.json
+rm -r delme/; mkdir delme
+./split.js --world
+./minitar.js
+ls -lS delme/*.json  | head -20
+cat delme/* | wc -c
+ls -l delme.mtar
+