@@ -50069,7 +50069,9 @@ WIDTH
<U3099>...<U309A> 0
<U309B>...<U30FF> 2
<U3105>...<U312F> 2
-<U3131>...<U318E> 2
+<U3131>...<U3163> 2
+<U3164> 0
+<U3165>...<U318E> 2
<U3190>...<U31E3> 2
<U31F0>...<U321E> 2
<U3220>...<UA48C> 2
@@ -50124,6 +50126,7 @@ WIDTH
<UFE68>...<UFE6B> 2
<UFEFF> 0
<UFF01>...<UFF60> 2
+<UFFA0> 0
<UFFE0>...<UFFE6> 2
<UFFF9>...<UFFFB> 0
<U000101FD> 0
@@ -1,4 +1,5 @@
# Copyright (C) 2015-2023 Free Software Foundation, Inc.
+# Copyright (C) 2024 The GNU Toolchain Authors.
# This file is part of the GNU C Library.
# The GNU C Library is free software; you can redistribute it and/or
@@ -94,6 +95,7 @@ UTF-8: UnicodeData.txt EastAsianWidth.txt
UTF-8: utf8_gen.py
$(PYTHON3) utf8_gen.py -u UnicodeData.txt \
-e EastAsianWidth.txt -p PropList.txt \
+ -d DerivedCoreProperties.txt \
--unicode_version $(UNICODE_VERSION)
UTF-8-report: UTF-8 ../charmaps/UTF-8
@@ -1,6 +1,7 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2023 Free Software Foundation, Inc.
+# Copyright (C) 2024 The GNU Toolchain Authors.
# This file is part of the GNU C Library.
#
# The GNU C Library is free software; you can redistribute it and/or
@@ -217,11 +218,13 @@ def write_header_width(outfile, unicode_version):
# outfile.write("% \"grep '^[^;]*;ZERO WIDTH ' UnicodeData.txt\"\n")
outfile.write("WIDTH\n")
-def process_width(outfile, ulines, elines, plines):
+def process_width(outfile, ulines, elines, plines, dlines):
'''ulines are lines from UnicodeData.txt, elines are lines from
EastAsianWidth.txt containing characters with width “W” or “F”,
plines are lines from PropList.txt which contain characters
with the property “Prepended_Concatenation_Mark”.
+ dlines are lines from DerivedCoreProperties.txt which contain
+ characters with the property “Default_Ignorable_Code_Point”.
'''
width_dict = {}
@@ -252,6 +255,24 @@ def process_width(outfile, ulines, elines, plines):
int(code_points[1], 16)+1):
del width_dict[key] # default width is 1
+ for line in dlines:
+ # Characters with the property “Default_Ignorable_Code_Point”
+ # should have the width 0:
+ fields = line.split(";")
+ if not '..' in fields[0]:
+ code_points = (fields[0], fields[0])
+ else:
+ code_points = fields[0].split("..")
+ for key in range(int(code_points[0], 16),
+ int(code_points[1], 16)+1):
+ width_dict[key] = 0 # default width is 1
+
+ # special case: U+115F HANGUL CHOSEONG FILLER
+ # combines with other Hangul jamo to form a width-2
+ # syllable block, so treat it as width 2
+ # despite it being a `Default_Ignorable_Code_Point`
+ width_dict[0x115F] = 2
+
# handle special cases for compatibility
for key in list((0x00AD,)):
# https://www.cs.tut.fi/~jkorpela/shy.html
@@ -325,6 +346,13 @@ if __name__ == "__main__":
default='PropList.txt',
help=('The PropList.txt file to read, '
+ 'default: %(default)s'))
+ PARSER.add_argument(
+ '-d', '--derived_core_properties_file',
+ nargs='?',
+ type=str,
+ default='DerivedCoreProperties.txt',
+ help=('The DerivedCoreProperties.txt file to read, '
+ + 'default: %(default)s'))
PARSER.add_argument(
'--unicode_version',
nargs='?',
@@ -357,6 +385,13 @@ if __name__ == "__main__":
for LINE in PROP_LIST_FILE:
if re.match(r'^[^;]*;[\s]*Prepended_Concatenation_Mark', LINE):
PROP_LIST_LINES.append(LINE.strip())
+ with open(ARGS.derived_core_properties_file, mode='r') as DERIVED_CORE_PROPERTIES_FILE:
+ DERIVED_CORE_PROPERTIES_LINES = []
+ for LINE in DERIVED_CORE_PROPERTIES_FILE:
+ if re.match(r'.*<reserved-.+>', LINE):
+ continue
+ if re.match(r'^[^;]*;[\s]*Default_Ignorable_Code_Point', LINE):
+ DERIVED_CORE_PROPERTIES_LINES.append(LINE.strip())
with open('UTF-8', mode='w') as OUTFILE:
# Processing UnicodeData.txt and write CHARMAP to UTF-8 file
write_header_charmap(OUTFILE)
@@ -367,5 +402,6 @@ if __name__ == "__main__":
process_width(OUTFILE,
UNICODE_DATA_LINES,
EAST_ASIAN_WIDTH_LINES,
- PROP_LIST_LINES)
+ PROP_LIST_LINES,
+ DERIVED_CORE_PROPERTIES_LINES)
OUTFILE.write("END WIDTH\n")