aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRobert Sipka <rsipka.uszeged@partner.samsung.com>2017-02-15 15:57:55 +0100
committerLászló Langó <llango.u-szeged@partner.samsung.com>2017-02-15 15:57:55 +0100
commit1b5f839db92051a35a04b265c1fcdedd25edfb64 (patch)
tree1c98e15ac4c12aab16ae4a1726025fd3c0531dcb
parent025a99ccbba2902b4b67e737903cc73445ca62d5 (diff)
Improve `toLowerCase` and `toUpperCase` functions. (#1575)
Language-sensitive mappings are not processed now. Fixes #323 JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com
-rw-r--r--jerry-core/lit/lit-char-helpers.c257
-rw-r--r--jerry-core/lit/lit-unicode-conversions.inc.h162
-rw-r--r--jerry-core/profiles/minimal.profile1
-rw-r--r--tests/jerry/string-upper-lower-case-conversion.js67
-rwxr-xr-xtools/unicode_case_conversion.py681
5 files changed, 1163 insertions, 5 deletions
diff --git a/jerry-core/lit/lit-char-helpers.c b/jerry-core/lit/lit-char-helpers.c
index 70dbd249..1cc15fa5 100644
--- a/jerry-core/lit/lit-char-helpers.c
+++ b/jerry-core/lit/lit-char-helpers.c
@@ -17,6 +17,10 @@
#include "lit/lit-unicode-ranges.inc.h"
#include "lit-strings.h"
+#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
+#include "lit-unicode-conversions.inc.h"
+#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
+
#define NUM_OF_ELEMENTS(array) (sizeof (array) / sizeof ((array)[0]))
/**
@@ -458,6 +462,184 @@ lit_char_is_word_char (ecma_char_t c) /**< code unit */
|| c == LIT_CHAR_UNDERSCORE);
} /* lit_char_is_word_char */
+#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
+
+/**
+ * Check if the specified character is in one of those tables which contain bidirectional conversions.
+ *
+ * @return the mapped character sequence of an ecma character, if it's in the table.
+ * 0 - otherwise.
+ */
+static ecma_length_t
+search_in_bidirectional_conversion_tables (ecma_char_t character, /**< code unit */
+ ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */
+ bool is_lowercase) /**< is lowercase conversion */
+{
+ /* 1, Check if the specified character is part of the jerry_character_case_ranges table. */
+ int number_of_case_ranges = NUM_OF_ELEMENTS (jerry_character_case_ranges);
+ int conv_counter = 0;
+
+ for (int i = 0; i < number_of_case_ranges; i++)
+ {
+ if (i % 2 == 0 && i > 0)
+ {
+ conv_counter++;
+ }
+
+ int range_length = jerry_character_case_range_lengths[conv_counter];
+ ecma_char_t start_point = jerry_character_case_ranges[i];
+
+ if (start_point > character || character >= start_point + range_length)
+ {
+ continue;
+ }
+
+ int char_dist = character - start_point;
+
+ if (i % 2 == 0)
+ {
+ output_buffer_p[0] = is_lowercase ? (ecma_char_t) (jerry_character_case_ranges[i + 1] + char_dist) : character;
+ }
+ else
+ {
+ output_buffer_p[0] = is_lowercase ? character : (ecma_char_t) (jerry_character_case_ranges[i - 1] + char_dist);
+ }
+
+ return 1;
+ }
+
+ /* 2, Check if the specified character is part of the character_pair_ranges table. */
+ int bottom = 0;
+ int top = NUM_OF_ELEMENTS (jerry_character_pair_ranges) - 1;
+
+ while (bottom <= top)
+ {
+ int middle = (bottom + top) / 2;
+ ecma_char_t current_sp = jerry_character_pair_ranges[middle];
+
+ if (current_sp <= character && character < current_sp + jerry_character_pair_range_lengths[middle])
+ {
+ int char_dist = character - current_sp;
+
+ if ((character - current_sp) % 2 == 0)
+ {
+ output_buffer_p[0] = is_lowercase ? (ecma_char_t) (current_sp + char_dist + 1) : character;
+ }
+ else
+ {
+ output_buffer_p[0] = is_lowercase ? character : (ecma_char_t) (current_sp + char_dist - 1);
+ }
+
+ return 1;
+ }
+
+ if (character > current_sp)
+ {
+ bottom = middle + 1;
+ }
+ else
+ {
+ top = middle - 1;
+ }
+ }
+
+ /* 3, Check if the specified character is part of the character_pairs table. */
+ int number_of_character_pairs = NUM_OF_ELEMENTS (jerry_character_pairs);
+
+ for (int i = 0; i < number_of_character_pairs; i++)
+ {
+ if (character != jerry_character_pairs[i])
+ {
+ continue;
+ }
+
+ if (i % 2 == 0)
+ {
+ output_buffer_p[0] = is_lowercase ? jerry_character_pairs[i + 1] : character;
+ }
+ else
+ {
+ output_buffer_p[0] = is_lowercase ? character : jerry_character_pairs[i - 1];
+ }
+
+ return 1;
+ }
+
+ return 0;
+} /* search_in_bidirectional_conversion_tables */
+
+/**
+ * Check if the specified character is in the given conversion table.
+ *
+ * @return the mapped character sequence of an ecma character, if it's in the table.
+ * 0 - otherwise.
+ */
+static ecma_length_t
+search_in_conversion_table (ecma_char_t character, /**< code unit */
+ ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */
+ const ecma_char_t *array, /**< array */
+ const uint8_t *counters) /**< case_values counter */
+{
+ int end_point = 0;
+
+ for (int i = 0; i < 3; i++)
+ {
+ int start_point = end_point;
+ int size_of_case_value = i + 1;
+ end_point += counters[i] * (size_of_case_value + 1);
+
+ int bottom = start_point;
+ int top = end_point - size_of_case_value;
+
+ while (bottom <= top)
+ {
+ int middle = (bottom + top) / 2;
+
+ middle -= ((middle - bottom) % (size_of_case_value + 1));
+
+ ecma_char_t current = array[middle];
+
+ if (current == character)
+ {
+ ecma_length_t char_sequence = 1;
+
+ switch (size_of_case_value)
+ {
+ case 3:
+ {
+ output_buffer_p[2] = array[middle + 3];
+ char_sequence++;
+ /* FALLTHRU */
+ }
+ case 2:
+ {
+ output_buffer_p[1] = array[middle + 2];
+ char_sequence++;
+ /* FALLTHRU */
+ }
+ default:
+ {
+ output_buffer_p[0] = array[middle + 1];
+ return char_sequence;
+ }
+ }
+ }
+
+ if (character < current)
+ {
+ top = middle - (size_of_case_value + 1);
+ }
+ else
+ {
+ bottom = middle + (size_of_case_value + 1);
+ }
+ }
+ }
+
+ return 0;
+} /* search_in_conversion_table */
+#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
+
/**
* Returns the lowercase character sequence of an ecma character.
*
@@ -471,8 +653,6 @@ lit_char_to_lower_case (ecma_char_t character, /**< input character value */
ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */
ecma_length_t buffer_size) /**< buffer size */
{
- /* TODO: Needs a proper lower case implementation. See issue #323. */
-
JERRY_ASSERT (buffer_size >= LIT_MAXIMUM_OTHER_CASE_LENGTH);
if (character >= LIT_CHAR_UPPERCASE_A && character <= LIT_CHAR_UPPERCASE_Z)
@@ -481,6 +661,41 @@ lit_char_to_lower_case (ecma_char_t character, /**< input character value */
return 1;
}
+#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
+
+ ecma_length_t lowercase_sequence = search_in_bidirectional_conversion_tables (character, output_buffer_p, true);
+
+ if (lowercase_sequence != 0)
+ {
+ return lowercase_sequence;
+ }
+
+ int num_of_lowercase_ranges = NUM_OF_ELEMENTS (jerry_lower_case_ranges);
+
+ for (int i = 0, j = 0; i < num_of_lowercase_ranges; i += 2, j++)
+ {
+ int range_length = jerry_lower_case_range_lengths[j] - 1;
+ ecma_char_t start_point = jerry_lower_case_ranges[i];
+
+ if (start_point <= character && character <= start_point + range_length)
+ {
+ output_buffer_p[0] = (ecma_char_t) (jerry_lower_case_ranges[i + 1] + (character - start_point));
+ return 1;
+ }
+ }
+
+ lowercase_sequence = search_in_conversion_table (character,
+ output_buffer_p,
+ jerry_lower_case_conversions,
+ jerry_lower_case_conversion_counters);
+
+ if (lowercase_sequence != 0)
+ {
+ return lowercase_sequence;
+ }
+
+#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
+
output_buffer_p[0] = character;
return 1;
} /* lit_char_to_lower_case */
@@ -498,8 +713,6 @@ lit_char_to_upper_case (ecma_char_t character, /**< input character value */
ecma_char_t *output_buffer_p, /**< buffer for the result characters */
ecma_length_t buffer_size) /**< buffer size */
{
- /* TODO: Needs a proper upper case implementation. See issue #323. */
-
JERRY_ASSERT (buffer_size >= LIT_MAXIMUM_OTHER_CASE_LENGTH);
if (character >= LIT_CHAR_LOWERCASE_A && character <= LIT_CHAR_LOWERCASE_Z)
@@ -508,6 +721,42 @@ lit_char_to_upper_case (ecma_char_t character, /**< input character value */
return 1;
}
+#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION
+
+ ecma_length_t uppercase_sequence = search_in_bidirectional_conversion_tables (character, output_buffer_p, false);
+
+ if (uppercase_sequence != 0)
+ {
+ return uppercase_sequence;
+ }
+
+ int num_of_upper_case_special_ranges = NUM_OF_ELEMENTS (jerry_upper_case_special_ranges);
+
+ for (int i = 0, j = 0; i < num_of_upper_case_special_ranges; i += 3, j++)
+ {
+ int range_length = jerry_upper_case_special_range_lengths[j];
+ ecma_char_t start_point = jerry_upper_case_special_ranges[i];
+
+ if (start_point <= character && character <= start_point + range_length)
+ {
+ output_buffer_p[0] = (ecma_char_t) (jerry_upper_case_special_ranges[i + 1] + (character - start_point));
+ output_buffer_p[1] = (ecma_char_t) (jerry_upper_case_special_ranges[i + 2]);
+ return 2;
+ }
+ }
+
+ uppercase_sequence = search_in_conversion_table (character,
+ output_buffer_p,
+ jerry_upper_case_conversions,
+ jerry_upper_case_conversion_counters);
+
+ if (uppercase_sequence != 0)
+ {
+ return uppercase_sequence;
+ }
+
+#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */
+
output_buffer_p[0] = character;
return 1;
} /* lit_char_to_upper_case */
diff --git a/jerry-core/lit/lit-unicode-conversions.inc.h b/jerry-core/lit/lit-unicode-conversions.inc.h
new file mode 100644
index 00000000..0e772573
--- /dev/null
+++ b/jerry-core/lit/lit-unicode-conversions.inc.h
@@ -0,0 +1,162 @@
+/* Copyright JS Foundation and other contributors, http://js.foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This file is automatically generated by the unicode_case_conversion.py script. Do not edit!
+ */
+
+/* Contains start points of character case ranges (these are bidirectional conversions). */
+static const uint16_t jerry_character_case_ranges[] JERRY_CONST_DATA =
+{
+ 0x00c0, 0x00e0, 0x00d8, 0x00f8, 0x0189, 0x0256, 0x01b1, 0x028a, 0x0388, 0x03ad,
+ 0x038e, 0x03cd, 0x0391, 0x03b1, 0x03a3, 0x03c3, 0x03fd, 0x037b, 0x0400, 0x0450,
+ 0x0410, 0x0430, 0x0531, 0x0561, 0x10a0, 0x2d00, 0x13a0, 0xab70, 0x13f0, 0x13f8,
+ 0x1f08, 0x1f00, 0x1f18, 0x1f10, 0x1f28, 0x1f20, 0x1f38, 0x1f30, 0x1f48, 0x1f40,
+ 0x1f68, 0x1f60, 0x1fb8, 0x1fb0, 0x1fba, 0x1f70, 0x1fc8, 0x1f72, 0x1fd8, 0x1fd0,
+ 0x1fda, 0x1f76, 0x1fe8, 0x1fe0, 0x1fea, 0x1f7a, 0x1ff8, 0x1f78, 0x1ffa, 0x1f7c,
+ 0x2160, 0x2170, 0x24b6, 0x24d0, 0x2c00, 0x2c30, 0x2c7e, 0x023f, 0xff21, 0xff41
+};
+
+/* Interval lengths of start points in `character_case_ranges` table. */
+static const uint8_t jerry_character_case_range_lengths[] JERRY_CONST_DATA =
+{
+ 0x0017, 0x0007, 0x0002, 0x0002, 0x0003, 0x0002, 0x0011, 0x0009, 0x0003, 0x0010,
+ 0x0020, 0x0026, 0x0026, 0x0050, 0x0006, 0x0008, 0x0006, 0x0008, 0x0008, 0x0006,
+ 0x0008, 0x0002, 0x0002, 0x0004, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002,
+ 0x0010, 0x001a, 0x002f, 0x0002, 0x001a
+};
+
+/* Contains the start points of bidirectional conversion ranges. */
+static const uint16_t jerry_character_pair_ranges[] JERRY_CONST_DATA =
+{
+ 0x0100, 0x0132, 0x0139, 0x014a, 0x0179, 0x0182, 0x0187, 0x018b, 0x0191, 0x0198,
+ 0x01a0, 0x01a7, 0x01ac, 0x01af, 0x01b3, 0x01b8, 0x01bc, 0x01cd, 0x01de, 0x01f4,
+ 0x01f8, 0x0222, 0x023b, 0x0241, 0x0246, 0x0370, 0x0376, 0x03d8, 0x03f7, 0x03fa,
+ 0x0460, 0x048a, 0x04c1, 0x04d0, 0x1e00, 0x1ea0, 0x2183, 0x2c60, 0x2c67, 0x2c72,
+ 0x2c75, 0x2c80, 0x2ceb, 0x2cf2, 0xa640, 0xa680, 0xa722, 0xa732, 0xa779, 0xa77e,
+ 0xa78b, 0xa790, 0xa796, 0xa7b4
+};
+
+/* Interval lengths of start points in `character_pair_ranges` table. */
+static const uint8_t jerry_character_pair_range_lengths[] JERRY_CONST_DATA =
+{
+ 0x0030, 0x0006, 0x0010, 0x002e, 0x0006, 0x0004, 0x0002, 0x0002, 0x0002, 0x0002,
+ 0x0006, 0x0002, 0x0002, 0x0002, 0x0004, 0x0002, 0x0002, 0x0010, 0x0012, 0x0002,
+ 0x0028, 0x0012, 0x0002, 0x0002, 0x000a, 0x0004, 0x0002, 0x0018, 0x0002, 0x0002,
+ 0x0022, 0x0036, 0x000e, 0x0060, 0x0096, 0x0060, 0x0002, 0x0002, 0x0006, 0x0002,
+ 0x0002, 0x0064, 0x0004, 0x0002, 0x002e, 0x001c, 0x000e, 0x003e, 0x0004, 0x000a,
+ 0x0002, 0x0004, 0x0014, 0x0004
+};
+
+/* Contains lower/upper case bidirectional conversion pairs. */
+static const uint16_t jerry_character_pairs[] JERRY_CONST_DATA =
+{
+ 0x0178, 0x00ff, 0x0181, 0x0253, 0x0186, 0x0254, 0x018e, 0x01dd, 0x018f, 0x0259,
+ 0x0190, 0x025b, 0x0193, 0x0260, 0x0194, 0x0263, 0x0196, 0x0269, 0x0197, 0x0268,
+ 0x019c, 0x026f, 0x019d, 0x0272, 0x019f, 0x0275, 0x01a6, 0x0280, 0x01a9, 0x0283,
+ 0x01ae, 0x0288, 0x01b7, 0x0292, 0x01c4, 0x01c6, 0x01c7, 0x01c9, 0x01ca, 0x01cc,
+ 0x01f1, 0x01f3, 0x01f6, 0x0195, 0x01f7, 0x01bf, 0x0220, 0x019e, 0x023a, 0x2c65,
+ 0x023d, 0x019a, 0x023e, 0x2c66, 0x0243, 0x0180, 0x0244, 0x0289, 0x0245, 0x028c,
+ 0x037f, 0x03f3, 0x0386, 0x03ac, 0x038c, 0x03cc, 0x03cf, 0x03d7, 0x03f9, 0x03f2,
+ 0x04c0, 0x04cf, 0x10c7, 0x2d27, 0x10cd, 0x2d2d, 0x1f59, 0x1f51, 0x1f5b, 0x1f53,
+ 0x1f5d, 0x1f55, 0x1f5f, 0x1f57, 0x1fec, 0x1fe5, 0x2132, 0x214e, 0x2c62, 0x026b,
+ 0x2c63, 0x1d7d, 0x2c64, 0x027d, 0x2c6d, 0x0251, 0x2c6e, 0x0271, 0x2c6f, 0x0250,
+ 0x2c70, 0x0252, 0xa77d, 0x1d79, 0xa78d, 0x0265, 0xa7aa, 0x0266, 0xa7ab, 0x025c,
+ 0xa7ac, 0x0261, 0xa7ad, 0x026c, 0xa7ae, 0x026a, 0xa7b0, 0x029e, 0xa7b1, 0x0287,
+ 0xa7b2, 0x029d, 0xa7b3, 0xab53
+};
+
+/* Contains start points of one-to-two uppercase ranges where the second character
+ * is always the same.
+ */
+static const uint16_t jerry_upper_case_special_ranges[] JERRY_CONST_DATA =
+{
+ 0x1f80, 0x1f08, 0x0399, 0x1f88, 0x1f08, 0x0399, 0x1f90, 0x1f28, 0x0399, 0x1f98,
+ 0x1f28, 0x0399, 0x1fa0, 0x1f68, 0x0399, 0x1fa8, 0x1f68, 0x0399
+};
+
+/* Interval lengths for start points in `upper_case_special_ranges` table. */
+static const uint8_t jerry_upper_case_special_range_lengths[] JERRY_CONST_DATA =
+{
+ 0x0007, 0x0007, 0x0007, 0x0007, 0x0007, 0x0007
+};
+
+/* Contains start points of lowercase ranges. */
+static const uint16_t jerry_lower_case_ranges[] JERRY_CONST_DATA =
+{
+ 0x1e96, 0x1e96, 0x1f80, 0x1f80, 0x1f88, 0x1f80, 0x1f90, 0x1f90, 0x1f98, 0x1f90,
+ 0x1fa0, 0x1fa0, 0x1fa8, 0x1fa0, 0x1fb2, 0x1fb2, 0x1fb6, 0x1fb6, 0x1fc2, 0x1fc2,
+ 0x1fc6, 0x1fc6, 0x1fd2, 0x1fd2, 0x1fd6, 0x1fd6, 0x1fe2, 0x1fe2, 0x1fe6, 0x1fe6,
+ 0x1ff2, 0x1ff2, 0x1ff6, 0x1ff6, 0xfb00, 0xfb00, 0xfb13, 0xfb13
+};
+
+/* Interval lengths for start points in `lower_case_ranges` table. */
+static const uint8_t jerry_lower_case_range_lengths[] JERRY_CONST_DATA =
+{
+ 0x0005, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0003, 0x0002, 0x0003,
+ 0x0002, 0x0002, 0x0002, 0x0003, 0x0002, 0x0003, 0x0002, 0x0007, 0x0005
+};
+
+/* The remaining lowercase conversions. The lowercase variant can be one-to-three character long. */
+static const uint16_t jerry_lower_case_conversions[] JERRY_CONST_DATA =
+{
+ 0x00df, 0x00df, 0x0149, 0x0149, 0x01c5, 0x01c6, 0x01c8, 0x01c9, 0x01cb, 0x01cc,
+ 0x01f0, 0x01f0, 0x01f2, 0x01f3, 0x0390, 0x0390, 0x03b0, 0x03b0, 0x03f4, 0x03b8,
+ 0x0587, 0x0587, 0x1e9e, 0x00df, 0x1f50, 0x1f50, 0x1f52, 0x1f52, 0x1f54, 0x1f54,
+ 0x1f56, 0x1f56, 0x1fbc, 0x1fb3, 0x1fcc, 0x1fc3, 0x1ffc, 0x1ff3, 0x2126, 0x03c9,
+ 0x212a, 0x006b, 0x212b, 0x00e5, 0x0130, 0x0069, 0x0307
+};
+
+/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */
+static const uint8_t jerry_lower_case_conversion_counters[] JERRY_CONST_DATA =
+{
+ 0x0016, 0x0001, 0x0000
+};
+
+/* The remaining uppercase conversions. The uppercase variant can be one-to-three character long. */
+static const uint16_t jerry_upper_case_conversions[] JERRY_CONST_DATA =
+{
+ 0x00b5, 0x039c, 0x0130, 0x0130, 0x0131, 0x0049, 0x017f, 0x0053, 0x01c5, 0x01c4,
+ 0x01c8, 0x01c7, 0x01cb, 0x01ca, 0x01f2, 0x01f1, 0x0345, 0x0399, 0x03c2, 0x03a3,
+ 0x03d0, 0x0392, 0x03d1, 0x0398, 0x03d5, 0x03a6, 0x03d6, 0x03a0, 0x03f0, 0x039a,
+ 0x03f1, 0x03a1, 0x03f5, 0x0395, 0x1c80, 0x0412, 0x1c81, 0x0414, 0x1c82, 0x041e,
+ 0x1c83, 0x0421, 0x1c84, 0x0422, 0x1c85, 0x0422, 0x1c86, 0x042a, 0x1c87, 0x0462,
+ 0x1c88, 0xa64a, 0x1e9b, 0x1e60, 0x1fbe, 0x0399, 0x00df, 0x0053, 0x0053, 0x0149,
+ 0x02bc, 0x004e, 0x01f0, 0x004a, 0x030c, 0x0587, 0x0535, 0x0552, 0x1e96, 0x0048,
+ 0x0331, 0x1e97, 0x0054, 0x0308, 0x1e98, 0x0057, 0x030a, 0x1e99, 0x0059, 0x030a,
+ 0x1e9a, 0x0041, 0x02be, 0x1f50, 0x03a5, 0x0313, 0x1f87, 0x1f0f, 0x0399, 0x1f8f,
+ 0x1f0f, 0x0399, 0x1f97, 0x1f2f, 0x0399, 0x1f9f, 0x1f2f, 0x0399, 0x1fa7, 0x1f6f,
+ 0x0399, 0x1faf, 0x1f6f, 0x0399, 0x1fb2, 0x1fba, 0x0399, 0x1fb3, 0x0391, 0x0399,
+ 0x1fb4, 0x0386, 0x0399, 0x1fb6, 0x0391, 0x0342, 0x1fbc, 0x0391, 0x0399, 0x1fc2,
+ 0x1fca, 0x0399, 0x1fc3, 0x0397, 0x0399, 0x1fc4, 0x0389, 0x0399, 0x1fc6, 0x0397,
+ 0x0342, 0x1fcc, 0x0397, 0x0399, 0x1fd6, 0x0399, 0x0342, 0x1fe4, 0x03a1, 0x0313,
+ 0x1fe6, 0x03a5, 0x0342, 0x1ff2, 0x1ffa, 0x0399, 0x1ff3, 0x03a9, 0x0399, 0x1ff4,
+ 0x038f, 0x0399, 0x1ff6, 0x03a9, 0x0342, 0x1ffc, 0x03a9, 0x0399, 0xfb00, 0x0046,
+ 0x0046, 0xfb01, 0x0046, 0x0049, 0xfb02, 0x0046, 0x004c, 0xfb05, 0x0053, 0x0054,
+ 0xfb06, 0x0053, 0x0054, 0xfb13, 0x0544, 0x0546, 0xfb14, 0x0544, 0x0535, 0xfb15,
+ 0x0544, 0x053b, 0xfb16, 0x054e, 0x0546, 0xfb17, 0x0544, 0x053d, 0x0390, 0x0399,
+ 0x0308, 0x0301, 0x03b0, 0x03a5, 0x0308, 0x0301, 0x1f52, 0x03a5, 0x0313, 0x0300,
+ 0x1f54, 0x03a5, 0x0313, 0x0301, 0x1f56, 0x03a5, 0x0313, 0x0342, 0x1fb7, 0x0391,
+ 0x0342, 0x0399, 0x1fc7, 0x0397, 0x0342, 0x0399, 0x1fd2, 0x0399, 0x0308, 0x0300,
+ 0x1fd3, 0x0399, 0x0308, 0x0301, 0x1fd7, 0x0399, 0x0308, 0x0342, 0x1fe2, 0x03a5,
+ 0x0308, 0x0300, 0x1fe3, 0x03a5, 0x0308, 0x0301, 0x1fe7, 0x03a5, 0x0308, 0x0342,
+ 0x1ff7, 0x03a9, 0x0342, 0x0399, 0xfb03, 0x0046, 0x0046, 0x0049, 0xfb04, 0x0046,
+ 0x0046, 0x004c
+};
+
+/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */
+static const uint8_t jerry_upper_case_conversion_counters[] JERRY_CONST_DATA =
+{
+ 0x001c, 0x002c, 0x0010
+};
+
diff --git a/jerry-core/profiles/minimal.profile b/jerry-core/profiles/minimal.profile
index b9a87794..d7504439 100644
--- a/jerry-core/profiles/minimal.profile
+++ b/jerry-core/profiles/minimal.profile
@@ -10,4 +10,5 @@ CONFIG_DISABLE_NUMBER_BUILTIN
CONFIG_DISABLE_REGEXP_BUILTIN
CONFIG_DISABLE_STRING_BUILTIN
CONFIG_DISABLE_TYPEDARRAY_BUILTIN
+CONFIG_DISABLE_UNICODE_CASE_CONVERSION
diff --git a/tests/jerry/string-upper-lower-case-conversion.js b/tests/jerry/string-upper-lower-case-conversion.js
index 3ac373f6..75a6dea9 100644
--- a/tests/jerry/string-upper-lower-case-conversion.js
+++ b/tests/jerry/string-upper-lower-case-conversion.js
@@ -12,7 +12,72 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Conversion
+// LATIN SMALL LIGATURES
+// LATIN SMALL LIGATURE FF
+assert ("\ufb00".toLowerCase() == "\ufb00");
+assert ("\ufb00".toUpperCase() == "\u0046\u0046");
+// LATIN SMALL LIGATURE FI
+assert ("\ufb01".toLowerCase() == "\ufb01");
+assert ("\ufb01".toUpperCase() == "\u0046\u0049");
+// LATIN SMALL LIGATURE FL
+assert ("\ufb02".toLowerCase() == "\ufb02");
+assert ("\ufb02".toUpperCase() == "\u0046\u004c");
+// LATIN SMALL LIGATURE FFI
+assert ("\ufb03".toLowerCase() == "\ufb03");
+assert ("\ufb03".toUpperCase() == "\u0046\u0046\u0049");
+// LATIN SMALL LIGATURE FFL
+assert ("\ufb04".toLowerCase() == "\ufb04");
+assert ("\ufb04".toUpperCase() == "\u0046\u0046\u004c");
+// LATIN SMALL LIGATURE LONG S T
+assert ("\ufb05".toLowerCase() == "\ufb05");
+assert ("\ufb05".toUpperCase() == "\u0053\u0054");
+// LATIN SMALL LIGATURE ST
+assert ("\ufb06".toLowerCase() == "\ufb06");
+assert ("\ufb06".toUpperCase() == "\u0053\u0054");
+
+// LATIN CAPITAL LETTER I WITH DOT ABOVE
+assert ("\u0130".toLowerCase() == "\u0069\u0307");
+assert ("\u0130".toUpperCase() == "\u0130");
+
+// LATIN SMALL LETTER SHARP S
+assert ("\u00df".toLowerCase() == "\u00df");
+assert ("\u00df".toUpperCase() == "\u0053\u0053");
+
+// LATIN CAPITAL LETTER I WITH BREVE
+assert ("\u012c".toLowerCase() == "\u012d");
+assert ("\u012c".toUpperCase() == "\u012c");
+// LATIN SMALL LETTER I WITH BREVE
+assert ("\u012d".toLowerCase() == "\u012d")
+assert ("\u012d".toUpperCase() == "\u012c");
+
+// Check randomly selected characters from conversion tables
+
+// lower-case conversions
+assert ("\u01c5\u01c8\u01cb\u212b".toLowerCase() == "\u01c6\u01c9\u01cc\u00e5");
+assert ("\u0130".toLowerCase() == "\u0069\u0307");
+
+// upper-case conversions
+assert ("\u00b5\u017f".toUpperCase() == "\u039c\u0053");
+assert ("\ufb17\u00df\u1fbc".toUpperCase() == "\u0544\u053D\u0053\u0053\u0391\u0399");
+assert ("\ufb03\ufb04".toUpperCase() == "\u0046\u0046\u0049\u0046\u0046\u004c");
+
+// character case ranges
+assert ("\u0100\u0101\u0139\u03fa\ua7b4".toLowerCase() == "\u0101\u0101\u013a\u03fb\ua7b5");
+assert ("\u0101\u0100\u013a\u03fb\ua7b5".toUpperCase() == "\u0100\u0100\u0139\u03fa\ua7b4");
+
+// character pairs
+assert ("\u0178\ua7b1\u0287\ua7b3".toLowerCase() == "\u00ff\u0287\u0287\uab53");
+assert ("\u00ff\u0287\ua7b1\uab53".toUpperCase() == "\u0178\ua7b1\ua7b1\ua7b3");
+
+// character case ranges
+assert ("\u00e0\u00c0\u00c1\u00c2\uff21".toLowerCase() == "\u00e0\u00e0\u00e1\u00e2\uff41");
+assert ("\u00e0\u00c0\u00e1\u00e2\uff41".toUpperCase() == "\u00c0\u00c0\u00c1\u00c2\uff21");
+
+// lower-case ranges
+assert ("\u1f88\u1f98\u1fa8\u1f8b\u1faf".toLowerCase() == "\u1f80\u1f90\u1fa0\u1f83\u1fa7");
+
+// upper-case special ranges
+assert ("\u1f80\u1f81\u1fa7".toUpperCase() == "\u1f08\u0399\u1f09\u0399\u1f6f\u0399");
assert ("0123456789abcdefghijklmnopqrstuvwxzyABCDEFGHIJKLMNOPQRSTUVWXYZ".toLowerCase()
== "0123456789abcdefghijklmnopqrstuvwxzyabcdefghijklmnopqrstuvwxyz");
diff --git a/tools/unicode_case_conversion.py b/tools/unicode_case_conversion.py
new file mode 100755
index 00000000..473953c5
--- /dev/null
+++ b/tools/unicode_case_conversion.py
@@ -0,0 +1,681 @@
+#!/usr/bin/env python
+
+# Copyright JS Foundation and other contributors, http://js.foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import csv
+import itertools
+import os
+import sys
+import warnings
+
+try:
+ unichr
+except NameError:
+ unichr = chr
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_DIR = os.path.normpath(os.path.join(TOOLS_DIR, '..'))
+C_SOURCE_FILE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h')
+
+
+def parse_unicode_sequence(raw_data):
+ """
+ Parse unicode sequence from raw data.
+
+ :param raw_data: Contains the unicode sequence which needs to parse.
+ :return: The parsed unicode sequence.
+ """
+
+ result = ''
+
+ for unicode_char in raw_data.split(' '):
+ if unicode_char == '':
+ continue
+
+ # Convert it to unicode code point (from hex value without 0x prefix)
+ result += unichr(int(unicode_char, 16))
+ return result
+
+
+def read_case_mappings(unicode_data_file, special_casing_file):
+ """
+ Read the corresponding unicode values of lower and upper case letters and store these in tables.
+
+ :param unicode_data_file: Contains the default case mappings (one-to-one mappings).
+ :param special_casing_file: Contains additional informative case mappings that are either not one-to-one
+ or which are context-sensitive.
+ :return: Upper and lower case mappings.
+ """
+
+ lower_case_mapping = CaseMapping()
+ upper_case_mapping = CaseMapping()
+
+ # Add one-to-one mappings
+ with open(unicode_data_file) as unicode_data:
+ unicode_data_reader = csv.reader(unicode_data, delimiter=';')
+
+ for line in unicode_data_reader:
+ letter_id = int(line[0], 16)
+
+ # Skip supplementary planes and ascii chars
+ if letter_id >= 0x10000 or letter_id < 128:
+ continue
+
+ capital_letter = line[12]
+ small_letter = line[13]
+
+ if capital_letter:
+ upper_case_mapping.add(letter_id, parse_unicode_sequence(capital_letter))
+
+ if small_letter:
+ lower_case_mapping.add(letter_id, parse_unicode_sequence(small_letter))
+
+ # Update the conversion tables with the special cases
+ with open(special_casing_file) as special_casing:
+ special_casing_reader = csv.reader(special_casing, delimiter=';')
+
+ for line in special_casing_reader:
+ # Skip comment sections and empty lines
+ if not line or line[0].startswith('#'):
+ continue
+
+ # Replace '#' character with empty string
+ for idx, i in enumerate(line):
+ if i.find('#') >= 0:
+ line[idx] = ''
+
+ letter_id = int(line[0], 16)
+ condition_list = line[4]
+
+ # Skip supplementary planes, ascii chars, and condition_list
+ if letter_id >= 0x10000 or letter_id < 128 or condition_list:
+ continue
+
+ small_letter = parse_unicode_sequence(line[1])
+ capital_letter = parse_unicode_sequence(line[3])
+
+ lower_case_mapping.add(letter_id, small_letter)
+ upper_case_mapping.add(letter_id, capital_letter)
+
+ return lower_case_mapping, upper_case_mapping
+
+
+class CaseMapping(dict):
+ """Class defines an informative, default mapping."""
+
+ def __init__(self):
+ """Initialize the case mapping table."""
+ self._conversion_table = {}
+
+ def add(self, letter_id, mapped_value):
+ """
+ Add mapped value of the unicode letter.
+
+ :param letter_id: An integer, representing the unicode code point of the character.
+ :param mapped_value: Corresponding character of the case type.
+ """
+ self._conversion_table[letter_id] = mapped_value
+
+ def remove(self, letter_id):
+ """
+ Remove mapping from the conversion table.
+
+ :param letter_id: An integer, representing the unicode code point of the character.
+ """
+ del self._conversion_table[letter_id]
+
+ def get_value(self, letter_id):
+ """
+ Get the mapped value of the given unicode character.
+
+ :param letter_id: An integer, representing the unicode code point of the character.
+ :return: The mapped value of the character.
+ """
+
+ if self.contains(letter_id):
+ return self._conversion_table[letter_id]
+
+ return None
+
+ def get_conversion_distance(self, letter_id):
+ """
+ Calculate the distance between the unicode character and its mapped value
+ (only needs and works with one-to-one mappings).
+
+ :param letter_id: An integer, representing the unicode code point of the character.
+ :return: The conversion distance.
+ """
+
+ mapped_value = self.get_value(letter_id)
+
+ if mapped_value and len(mapped_value) == 1:
+ return ord(mapped_value) - letter_id
+
+ return None
+
+ def is_bidirectional_conversion(self, letter_id, other_case_mapping):
+ """
+ Check that two unicode value are also a mapping value of each other.
+
+ :param letter_id: An integer, representing the unicode code point of the character.
+ :param other_case_mapping: Comparable case mapping table which possible contains
+ the return direction of the conversion.
+ :return: True, if it's a reverible conversion, false otherwise.
+ """
+
+ if not self.contains(letter_id):
+ return False
+
+ # Check one-to-one mapping
+ mapped_value = self.get_value(letter_id)
+ if len(mapped_value) > 1:
+ return False
+
+ # Check two way conversions
+ mapped_value_id = ord(mapped_value)
+ if other_case_mapping.get_value(mapped_value_id) != unichr(letter_id):
+ return False
+
+ return True
+
+ def contains(self, letter_id):
+ """
+ Check that a unicode character is in the conversion table.
+
+ :param letter_id: An integer, representing the unicode code point of the character.
+ :return: True, if it contains the character, false otherwise.
+ """
+ if letter_id in self._conversion_table:
+ return True
+
+ return False
+
+ def get_table(self):
+ return self._conversion_table
+
+ def extract_ranges(self, other_case_mapping=None):
+ """
+ Extract ranges from case mappings
+ (the second param is optional, if it's not empty, a range will contains bidirectional conversions only).
+
+ :param letter_id: An integer, representing the unicode code point of the character.
+ :param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion.
+ :return: A table with the start points and their mapped value, and another table with the lengths of the ranges.
+ """
+
+ in_range = False
+ range_position = -1
+ ranges = []
+ range_lengths = []
+
+ for letter_id in sorted(self._conversion_table.keys()):
+ prev_letter_id = letter_id - 1
+
+ # One-way conversions
+ if other_case_mapping is None:
+ if len(self.get_value(letter_id)) > 1:
+ in_range = False
+ continue
+
+ if not self.contains(prev_letter_id) or len(self.get_value(prev_letter_id)) > 1:
+ in_range = False
+ continue
+
+ # Two way conversions
+ else:
+ if not self.is_bidirectional_conversion(letter_id, other_case_mapping):
+ in_range = False
+ continue
+
+ if not self.is_bidirectional_conversion(prev_letter_id, other_case_mapping):
+ in_range = False
+ continue
+
+ conv_distance = self.get_conversion_distance(letter_id)
+ prev_conv_distance = self.get_conversion_distance(prev_letter_id)
+
+ if (conv_distance != prev_conv_distance):
+ in_range = False
+ continue
+
+ if in_range:
+ range_lengths[range_position] += 1
+ else:
+ in_range = True
+ range_position += 1
+
+ # Add the start point of the range and its mapped value
+ ranges.extend([prev_letter_id, ord(self.get_value(prev_letter_id))])
+ range_lengths.append(2)
+
+ # Remove all ranges from the case mapping table.
+ index = 0
+ while index != len(ranges):
+ range_length = range_lengths[index // 2]
+
+ for incr in range(range_length):
+ self.remove(ranges[index] + incr)
+ if other_case_mapping is not None:
+ other_case_mapping.remove(ranges[index + 1] + incr)
+
+ index += 2
+
+ return ranges, range_lengths
+
+ def extract_character_pair_ranges(self, other_case_mapping):
+ """
+ Extract two or more character pairs from the case mapping tables.
+
+ :param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion.
+ :return: A table with the start points, and another table with the lengths of the ranges.
+ """
+
+ start_points = []
+ lengths = []
+ in_range = False
+ element_counter = -1
+
+ for letter_id in sorted(self._conversion_table.keys()):
+ # Only extract character pairs
+ if not self.is_bidirectional_conversion(letter_id, other_case_mapping):
+ in_range = False
+ continue
+
+ if self.get_value(letter_id) == unichr(letter_id + 1):
+ prev_letter_id = letter_id - 2
+
+ if not self.is_bidirectional_conversion(prev_letter_id, other_case_mapping):
+ in_range = False
+
+ if in_range:
+ lengths[element_counter] += 2
+ else:
+ element_counter += 1
+ start_points.append(letter_id)
+ lengths.append(2)
+ in_range = True
+
+ else:
+ in_range = False
+
+ # Remove all founded case mapping from the conversion tables after the scanning method
+ idx = 0
+ while idx != len(start_points):
+ letter_id = start_points[idx]
+ conv_length = lengths[idx]
+
+ for incr in range(0, conv_length, 2):
+ self.remove(letter_id + incr)
+ other_case_mapping.remove(letter_id + 1 + incr)
+
+ idx += 1
+
+ return start_points, lengths
+
+ def extract_character_pairs(self, other_case_mapping):
+ """
+ Extract character pairs. Check that two unicode value are also a mapping value of each other.
+
+ :param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion.
+ :return: A table with character pairs.
+ """
+
+ character_pairs = []
+
+ for letter_id in sorted(self._conversion_table.keys()):
+ if self.is_bidirectional_conversion(letter_id, other_case_mapping):
+ mapped_value = self.get_value(letter_id)
+ character_pairs.extend([letter_id, ord(mapped_value)])
+
+ # Remove character pairs from case mapping tables
+ self.remove(letter_id)
+ other_case_mapping.remove(ord(mapped_value))
+
+ return character_pairs
+
+ def extract_special_ranges(self):
+ """
+ Extract special ranges. It contains that ranges of one-to-two mappings where the second character
+ of the mapped values are equals and the other characters are following each other.
+ eg.: \u1f80 and \u1f81 will be in one range becase their upper-case values are \u1f08\u0399 and \u1f09\u0399
+
+ :return: A table with the start points and their mapped values, and a table with the lengths of the ranges.
+ """
+
+ special_ranges = []
+ special_range_lengths = []
+
+ range_position = -1
+
+ for letter_id in sorted(self._conversion_table.keys()):
+ mapped_value = self.get_value(letter_id)
+
+ if len(mapped_value) != 2:
+ continue
+
+ prev_letter_id = letter_id - 1
+
+ if not self.contains(prev_letter_id):
+ in_range = False
+ continue
+
+ prev_mapped_value = self.get_value(prev_letter_id)
+
+ if len(prev_mapped_value) != 2:
+ continue
+
+ if prev_mapped_value[1] != mapped_value[1]:
+ continue
+
+ if (ord(prev_mapped_value[0]) - prev_letter_id) != (ord(mapped_value[0]) - letter_id):
+ in_range = False
+ continue
+
+ if in_range:
+ special_range_lengths[range_position] += 1
+ else:
+ range_position += 1
+ in_range = True
+
+ special_ranges.extend([prev_letter_id, ord(prev_mapped_value[0]), ord(prev_mapped_value[1])])
+ special_range_lengths.append(1)
+
+ # Remove special ranges from the conversion table
+ idx = 0
+ while idx != len(special_ranges):
+ range_length = special_range_lengths[idx // 3]
+ letter_id = special_ranges[idx]
+
+ for incr in range(range_length):
+ self.remove(special_ranges[idx] + incr)
+
+ idx += 3
+
+ return special_ranges, special_range_lengths
+
+ def extract_conversions(self):
+ """
+ Extract conversions. It provide the full (or remained) case mappings from the table.
+ The counter table contains the information of how much one-to-one, one-to-two or one-to-three mappings
+ exists successively in the conversion table.
+
+ :return: A table with conversions, and a table with counters.
+ """
+
+ unicodes = [[], [], []]
+ unicode_lengths = [0, 0, 0]
+
+ # 1 to 1 byte
+ for letter_id in sorted(self._conversion_table.keys()):
+ mapped_value = self.get_value(letter_id)
+
+ if len(mapped_value) != 1:
+ continue
+
+ unicodes[0].extend([letter_id, ord(mapped_value)])
+ self.remove(letter_id)
+
+ # 1 to 2 bytes
+ for letter_id in sorted(self._conversion_table.keys()):
+ mapped_value = self.get_value(letter_id)
+
+ if len(mapped_value) != 2:
+ continue
+
+ unicodes[1].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1])])
+ self.remove(letter_id)
+
+ # 1 to 3 bytes
+ for letter_id in sorted(self._conversion_table.keys()):
+ mapped_value = self.get_value(letter_id)
+
+ if len(mapped_value) != 3:
+ continue
+
+ unicodes[2].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1]), ord(mapped_value[2])])
+ self.remove(letter_id)
+
+ unicode_lengths = [int(len(unicodes[0]) / 2), int(len(unicodes[1]) / 3), int(len(unicodes[2]) / 4)]
+
+ return list(itertools.chain.from_iterable(unicodes)), unicode_lengths
+
+
+def regroup(l, n):
+ return [l[i:i+n] for i in range(0, len(l), n)]
+
+
+def hex_format(ch):
+ if isinstance(ch, str):
+ ch = ord(ch)
+
+ return "0x{:04x}".format(ch)
+
+
+def format_code(code, indent):
+ lines = []
+ # convert all characters to hex format
+ converted_code = map(hex_format, code)
+ # 10 hex number per line
+ for line in regroup(", ".join(converted_code), 10 * 8):
+ lines.append((' ' * indent) + line.strip())
+ return "\n".join(lines)
+
+
+def create_c_format_table(type_name, array_name, table, description=""):
+ return """{DESC}
+static const {TYPE} jerry_{NAME}[] JERRY_CONST_DATA =
+{{
+{TABLE}
+}};
+
+""".format(DESC=description, TYPE=type_name, NAME=array_name, TABLE=format_code(table, 1))
+
+
+def copy_tables_to_c_source(gen_tables, c_source):
+ data = []
+
+ header = """/* Copyright JS Foundation and other contributors, http://js.foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This file is automatically generated by the {SCRIPT} script. Do not edit!
+ */
+
+""".format(SCRIPT=os.path.basename(__file__))
+
+ data.append(header)
+
+ character_case_ranges = gen_tables.get_character_case_ranges()
+ character_pair_ranges = gen_tables.get_character_pair_ranges()
+ character_pairs = gen_tables.get_character_pairs()
+ upper_case_special_ranges = gen_tables.get_upper_case_special_ranges()
+ lower_case_ranges = gen_tables.get_lower_case_ranges()
+ lower_case_conversions = gen_tables.get_lower_case_conversions()
+ upper_case_conversions = gen_tables.get_upper_case_conversions()
+
+ description = "/* Contains start points of character case ranges (these are bidirectional conversions). */"
+ data.append(create_c_format_table('uint16_t', 'character_case_ranges',
+ character_case_ranges[0],
+ description))
+
+ description = "/* Interval lengths of start points in `character_case_ranges` table. */"
+ data.append(create_c_format_table('uint8_t',
+ 'character_case_range_lengths',
+ character_case_ranges[1],
+ description))
+
+ description = "/* Contains the start points of bidirectional conversion ranges. */"
+ data.append(create_c_format_table('uint16_t',
+ 'character_pair_ranges',
+ character_pair_ranges[0],
+ description))
+
+ description = "/* Interval lengths of start points in `character_pair_ranges` table. */"
+ data.append(create_c_format_table('uint8_t',
+ 'character_pair_range_lengths',
+ character_pair_ranges[1],
+ description))
+
+ description = "/* Contains lower/upper case bidirectional conversion pairs. */"
+ data.append(create_c_format_table('uint16_t',
+ 'character_pairs',
+ character_pairs,
+ description))
+
+ description = """/* Contains start points of one-to-two uppercase ranges where the second character
+ * is always the same.
+ */"""
+ data.append(create_c_format_table('uint16_t',
+ 'upper_case_special_ranges',
+ upper_case_special_ranges[0],
+ description))
+
+ description = "/* Interval lengths for start points in `upper_case_special_ranges` table. */"
+ data.append(create_c_format_table('uint8_t',
+ 'upper_case_special_range_lengths',
+ upper_case_special_ranges[1],
+ description))
+
+ description = "/* Contains start points of lowercase ranges. */"
+ data.append(create_c_format_table('uint16_t', 'lower_case_ranges', lower_case_ranges[0], description))
+
+ description = "/* Interval lengths for start points in `lower_case_ranges` table. */"
+ data.append(create_c_format_table('uint8_t', 'lower_case_range_lengths', lower_case_ranges[1], description))
+
+ description = "/* The remaining lowercase conversions. The lowercase variant can be one-to-three character long. */"
+ data.append(create_c_format_table('uint16_t',
+ 'lower_case_conversions',
+ lower_case_conversions[0],
+ description))
+
+ description = "/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */"
+
+ data.append(create_c_format_table('uint8_t',
+ 'lower_case_conversion_counters',
+ lower_case_conversions[1],
+ description))
+
+ description = "/* The remaining uppercase conversions. The uppercase variant can be one-to-three character long. */"
+ data.append(create_c_format_table('uint16_t',
+ 'upper_case_conversions',
+ upper_case_conversions[0],
+ description))
+
+ description = "/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */"
+ data.append(create_c_format_table('uint8_t',
+ 'upper_case_conversion_counters',
+ upper_case_conversions[1],
+ description))
+
+ with open(c_source, 'w') as genereted_source:
+ genereted_source.write(''.join(data))
+
+
+class GenTables(object):
+ """Class defines an informative, default generated tables."""
+
+ def __init__(self, lower_case_table, upper_case_table):
+ """
+ Generate the extracted tables from the given case mapping tables.
+
+ :param lower_case_table: Lower-case mappings.
+ :param upper_case_table: Upper-case mappings.
+ """
+
+ self._character_case_ranges = lower_case_table.extract_ranges(upper_case_table)
+ self._character_pair_ranges = lower_case_table.extract_character_pair_ranges(upper_case_table)
+ self._character_pairs = lower_case_table.extract_character_pairs(upper_case_table)
+ self._upper_case_special_ranges = upper_case_table.extract_special_ranges()
+ self._lower_case_ranges = lower_case_table.extract_ranges()
+ self._lower_case_conversions = lower_case_table.extract_conversions()
+ self._upper_case_conversions = upper_case_table.extract_conversions()
+
+ if lower_case_table.get_table():
+ warnings.warn('Not all elements extracted from the lowercase conversion table!')
+ if upper_case_table.get_table():
+ warnings.warn('Not all elements extracted from the uppercase conversion table!')
+
+ def get_character_case_ranges(self):
+ return self._character_case_ranges
+
+ def get_character_pair_ranges(self):
+ return self._character_pair_ranges
+
+ def get_character_pairs(self):
+ return self._character_pairs
+
+ def get_upper_case_special_ranges(self):
+ return self._upper_case_special_ranges
+
+ def get_lower_case_ranges(self):
+ return self._lower_case_ranges
+
+ def get_lower_case_conversions(self):
+ return self._lower_case_conversions
+
+ def get_upper_case_conversions(self):
+ return self._upper_case_conversions
+
+
+def main():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument('--unicode-data',
+ metavar='FILE',
+ action='store',
+ required=True,
+ help='specify the unicode data file')
+
+ parser.add_argument('--special-casing',
+ metavar='FILE',
+ action='store',
+ required=True,
+ help='specify the special casing file')
+
+ parser.add_argument('--c-source',
+ metavar='FILE',
+ action='store',
+ default=C_SOURCE_FILE,
+ help='specify the output c source (default: %(default)s)')
+
+ script_args = parser.parse_args()
+
+ if not os.path.isfile(script_args.unicode_data) or not os.access(script_args.unicode_data, os.R_OK):
+ print('The %s file is missing or not readable!' % script_args.unicode_data)
+ sys.exit(1)
+
+ if not os.path.isfile(script_args.special_casing) or not os.access(script_args.special_casing, os.R_OK):
+ print('The %s file is missing or not readable!' % script_args.special_casing)
+ sys.exit(1)
+
+ lower_case_table, upper_case_table = read_case_mappings(script_args.unicode_data, script_args.special_casing)
+
+ gen_tables = GenTables(lower_case_table, upper_case_table)
+
+ copy_tables_to_c_source(gen_tables, script_args.c_source)
+
+if __name__ == "__main__":
+ main()