xtermjs · jerch · May 30, 2018 · May 30, 2018 · Jun 3, 2018 · Jun 3, 2018
diff --git a/bin/create-graphemedata.js b/bin/create-graphemedata.js
@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+'use strict';
+
+const URL = 'https://www.unicode.org/Public/10.0.0/ucd/auxiliary/GraphemeBreakProperty.txt';
+const PATH = __dirname + '/../src/GraphemeData.ts';
+
+const GRAPHEME_REX = /^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([A-Za-z_]+)/gm;
+
+const TYPES  = {
+    Other: 0,
+    L: 1,
+    V: 2,
+    T: 3,
+    LV: 4,
+    LVT: 5,
+    CR: 6,
+    LF: 7,
+    ZWJ: 8,
+    Prepend: 9,
+    Control: 10,
+    Extend: 11,
+    SpacingMark: 12,
+    E_Base: 13,
+    Glue_After_Zwj: 14,
+    E_Modifier: 15,
+    E_Base_GAZ: 16,
+    Regional_Indicator: 17
+};
+
+function parseDefinitions(data) {
+    let codepoints = Object.create(null);
+    let match = null;
+    while (match = GRAPHEME_REX.exec(data)) {
+        let start = parseInt(match[1], 16);
+        let end = parseInt(match[2], 16) || start;
+        for (let i = start; i < end + 1; ++i)
+            codepoints[i] = match[3];
+    }
+    return codepoints;
+}
+
+
+function createPackedBMP(codepoints, start, end) {
+    let type = -1;
+    let count = 0;
+    let lengths = [];
+    let types = [];
+    for (let i = start; i < end; ++i) {
+        let t = parseInt(TYPES[codepoints[i] || 'Other']);
+        if (t !== type) {
+            lengths.push(count);
+            types.push(type);
+            type = t;
+            count = 0;
+        }
+        if (count === 255) {
+            lengths.push(count);
+            types.push(type);
+            count = 0;
+        }
+        count++;
+    }
+    lengths.push(count);
+    types.push(type);
+
+    // remove start entries
+    lengths.shift();
+    types.shift();
+
+    if (types.length & 1)
+        types.push(0);
+
+    let accu = 0;
+    let finalTypes = [];
+    for (let i = 0; i < types.length; ++i) {
+        accu <<= 4;
+        accu |= types[i];
+        if (i & 1) {
+            finalTypes.push(accu);
+            accu = 0;
+        }
+    }
+
+    // null terminate length values
+    lengths.push(0);
+    return new Buffer(lengths.concat(finalTypes)).toString('base64');
+}
+
+function createPackedHIGH(codepoints, plane, start, end) {
+    start = start + 65536 * plane;
+    end = end + 65536 * plane;
+    let length = 0;
+    let type = -1;
+    const segments = [];
+    let segmentStart = -1;
+    for (let i = start; i < end; ++i) {
+        let t = parseInt(TYPES[codepoints[i] || 'Other']);
+        if (t !== type) {
+            // end of segment reached
+            // only push non Other segments
+            if (type) segments.push([segmentStart, length, type]);
+            segmentStart = i;
+            length = 0;
+            type = t;
+        }
+        if (length === 255) {
+            if (type) {
+                segments.push([segmentStart, length, type]);
+                segmentStart = i;
+                length = 0;
+            }
+        }
+        length++;
+    }
+    if (type) segments.push([segmentStart, length, type]);
+    segments.shift();
+    console.log(segments);
+
+    // write to byte typed
+    let final = [];
+    for (let i = 0; i < segments.length; ++i) {
+        final.push(segments[i][0] >> 8);
+        final.push(segments[i][0] & 255);
+        final.push(segments[i][1]);
+        final.push(segments[i][2]);
+    }
+    return new Buffer(final).toString('base64');
+}
+
+
+function createGraphemeDataFile(url, path) {
+    require('https').get(url, (resp) => {
+        let data = '';
+        resp.on('data', (chunk) => {
+            data += chunk;
+        });
+        resp.on('end', () => {
+            const codepoints = parseDefinitions(data);
+            let highest = 0;
+            for (let el in codepoints)
+                highest = Math.max(highest, parseInt(el));
+
+            // codepoint < 12443
+            const first = createPackedBMP(codepoints, 0, 12443);
+            // 42606 <= codepoint < 65536
+            const second = createPackedBMP(codepoints, 42606, 65536);
+            // Supplementary Multilingual Plane (1): 0 <= codepoint < 63966
+            const third = createPackedHIGH(codepoints, 1, 0, 63966);
+            // Supplementary Special-purpose Plane (14): 0 <= codepoint < highest + 1
+            const fourth = createPackedHIGH(codepoints, 14, 0, highest + 1);
+
+            // write to ts file
+            let final = '';
+            final += `// FIRST: 0 <= codepoint < 12443\n`;
+            final += `export const FIRST: string = '${first}';\n`;
+            final += `// SECOND: 42606 <= codepoint < 65536\n`;
+            final += `export const SECOND: string = '${second}';\n`;
+            final += `// THIRD: Supplementary Multilingual Plane (1) 0 <= codepoint < 63966\n`;
+            final += `export const THIRD: string = '${third}';\n`;
+            final += `// FOURTH: Supplementary Special-purpose Plane (14) 0 <= codepoint <= highest\n`;
+            final += `export const FOURTH: string = '${fourth}';\n`;
+            require('fs').writeFileSync(path, final);
+        });
+    }).on('error', (err) => {
+        console.log('error', err.message);
+    });
+}
+
+createGraphemeDataFile(URL, PATH);
diff --git a/package.json b/package.json
@@ -55,6 +55,7 @@
     "prepublish": "npm run build",
     "coveralls": "gulp coveralls",
     "webpack": "gulp webpack",
-    "watch": "gulp watch"
+    "watch": "gulp watch",
+    "graphemedata": "node bin/create-graphemedata"
   }
 }
diff --git a/src/Grapheme.test.ts b/src/Grapheme.test.ts
@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) 2018 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import { graphemeType, canBreak, BreakState, GraphemeTypes } from './Grapheme';
+import * as chai from 'chai';
+
+const _TYPES  = {
+  Other: 0,
+  L: 1,
+  V: 2,
+  T: 3,
+  LV: 4,
+  LVT: 5,
+  CR: 6,
+  LF: 7,
+  ZWJ: 8,
+  Prepend: 9,
+  Control: 10,
+  Extend: 11,
+  SpacingMark: 12,
+  E_Base: 13,
+  Glue_After_Zwj: 14,
+  E_Modifier: 15,
+  E_Base_GAZ: 16,
+  Regional_Indicator: 17
+};
+
+const URL = 'https://www.unicode.org/Public/10.0.0/ucd/auxiliary/GraphemeBreakProperty.txt';
+const GRAPHEME_REX = /^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([A-Za-z_]+)/gm;
+
+let CODEPOINTS = null;
+
+function parseDefinitions(data: string): {[key: number]: number} {
+  const codepoints = Object.create(null);
+  let match = null;
+  while (match = GRAPHEME_REX.exec(data)) {
+    const start = parseInt(match[1], 16);
+    const end = parseInt(match[2], 16) || start;
+    for (let i = start; i < end + 1; ++i) codepoints[i] = match[3];
+  }
+  return codepoints;
+}
+
+function loadUnicodeData(done: Function): void {
+  require('https').get(URL, (resp): any => {
+    let data = '';
+    resp.on('data', (chunk): void => {
+      data += chunk;
+    });
+    resp.on('end', () => {
+      CODEPOINTS = parseDefinitions(data);
+      done();
+    });
+  }).on('error', (err) => {
+    throw Error('error fetching unicode data');
+  });
+}
+
+describe('grapheme cluster', function (): void {
+  before(function(done: Function): void {
+    this.timeout(5000);
+    loadUnicodeData(done);
+  });
+  describe('graphemeType', function(): void {
+    it('BMP (0)', function(): void {
+      if (!CODEPOINTS) return;
+      for (let cp = 0; cp < 65536; ++cp) {
+        chai.expect(graphemeType(cp)).equals(_TYPES[CODEPOINTS[cp]] || 0);
+      }
+    });
+    it('SMP (1)', function(): void {
+      if (!CODEPOINTS) return;
+      for (let cp = 65536; cp < 2 * 65536; ++cp) {
+        chai.expect(graphemeType(cp)).equals(_TYPES[CODEPOINTS[cp]] || 0);
+      }
+    });
+    it('SSP (14)', function(): void {
+      if (!CODEPOINTS) return;
+      for (let cp = 14 * 65536; cp < 15 * 65536; ++cp) {
+        chai.expect(graphemeType(cp)).equals(_TYPES[CODEPOINTS[cp]] || 0);
+      }
+    });
+  });
+  describe('break rules', function(): void {
+    it('GB 3', function(): void {
+      chai.expect(canBreak(GraphemeTypes.LF, GraphemeTypes.CR)).equals(BreakState.FALSE);
+    });
+    it('GB 4', function(): void { // TODO: test all states
+      const types = [GraphemeTypes.CONTROL, GraphemeTypes.CR, GraphemeTypes.LF];
+      for (let pos in types) {
+        chai.expect(canBreak(GraphemeTypes.OTHER, types[pos])).equals(BreakState.TRUE);
+      }
+    });
+    it('GB 5', function(): void { // TODO: test all states
+      const types = [GraphemeTypes.CONTROL, GraphemeTypes.CR, GraphemeTypes.LF];
+      for (let pos in types) {
+        chai.expect(canBreak(types[pos], GraphemeTypes.OTHER)).equals(BreakState.TRUE);
+      }
+    });
+    it('GB 6', function(): void {
+      const types = [GraphemeTypes.L, GraphemeTypes.V, GraphemeTypes.LV, GraphemeTypes.LVT];
+      for (let pos in types) {
+        chai.expect(canBreak(types[pos], GraphemeTypes.L)).equals(BreakState.FALSE);
+      }
+    });
+    it('GB 7', function(): void {
+      chai.expect(canBreak(GraphemeTypes.V, GraphemeTypes.LV)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.T, GraphemeTypes.LV)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.V, GraphemeTypes.V)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.T, GraphemeTypes.V)).equals(BreakState.FALSE);
+    });
+    it('GB 8', function(): void {
+      chai.expect(canBreak(GraphemeTypes.T, GraphemeTypes.LVT)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.T, GraphemeTypes.T)).equals(BreakState.FALSE);
+    });
+    it('GB 9', function(): void {
+      chai.expect(canBreak(GraphemeTypes.EXTEND, GraphemeTypes.OTHER)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.ZWJ, GraphemeTypes.OTHER)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.EXTEND, GraphemeTypes.E_BASE)).equals(BreakState.EMOJI_EXTEND);
+      chai.expect(canBreak(GraphemeTypes.ZWJ, GraphemeTypes.E_BASE)).equals(BreakState.EMOJI_EXTEND);  // wrong here?
+      chai.expect(canBreak(GraphemeTypes.EXTEND, GraphemeTypes.E_BASE_GAZ)).equals(BreakState.EMOJI_EXTEND);
+      chai.expect(canBreak(GraphemeTypes.ZWJ, GraphemeTypes.E_BASE_GAZ)).equals(BreakState.EMOJI_EXTEND);  // wrong here?
+    });
+    it('GB 9a', function(): void {
+      chai.expect(canBreak(GraphemeTypes.SPACINGMARK, GraphemeTypes.OTHER)).equals(BreakState.FALSE);
+    });
+    it('GB 9b', function(): void {
+      chai.expect(canBreak(GraphemeTypes.OTHER, GraphemeTypes.PREPEND)).equals(BreakState.FALSE);
+    });
+    it('GB 10', function(): void {
+      chai.expect(canBreak(GraphemeTypes.E_MODIFIER, GraphemeTypes.E_BASE)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.E_MODIFIER, GraphemeTypes.E_BASE_GAZ)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.E_MODIFIER, BreakState.EMOJI_EXTEND)).equals(BreakState.FALSE);
+    });
+    it('GB 11', function(): void {
+      chai.expect(canBreak(GraphemeTypes.GLUE_AFTER_ZWJ, GraphemeTypes.ZWJ)).equals(BreakState.FALSE);
+      chai.expect(canBreak(GraphemeTypes.E_BASE_GAZ, GraphemeTypes.ZWJ)).equals(BreakState.FALSE);
+    });
+    it('GB 12 & 13', function(): void {
+      chai.expect(canBreak(GraphemeTypes.REGIONAL_INDICATOR, GraphemeTypes.REGIONAL_INDICATOR)).equals(BreakState.REGIONAL_SECOND);
+      chai.expect(canBreak(GraphemeTypes.REGIONAL_INDICATOR, BreakState.REGIONAL_SECOND)).equals(BreakState.TRUE);
+    });
+  });
+});