#include <stdio.h>
#include <stdlib.h>
#include "unicode/utypes.h"
#include "unicode/uchar.h"
#include "unicode/ustring.h"
#include "cmemory.h"
#include "cstring.h"
#include "filestrm.h"
#include "unicode/udata.h"
#include "utrie.h"
#include "unicode/uset.h"
#include "toolutil.h"
#include "unewdata.h"
#include "unormimp.h"
#include "gennorm.h"
#ifdef WIN32
# pragma warning(disable: 4100)
#endif
#define DO_DEBUG_OUT 0
#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
#if UCONFIG_NO_NORMALIZATION
static UDataInfo dataInfo = {
sizeof(UDataInfo),
0,
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
0,
{ 0, 0, 0, 0 },
{ 0, 0, 0, 0 },
{ 0, 0, 0, 0 }
};
#else
static UDataInfo dataInfo={
sizeof(UDataInfo),
0,
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
0,
{ 0x4e, 0x6f, 0x72, 0x6d },
{ 2, 3, UTRIE_SHIFT, UTRIE_INDEX_SHIFT },
{ 3, 2, 0, 0 }
};
extern void
setUnicodeVersion(const char *v) {
UVersionInfo version;
u_versionFromString(version, v);
uprv_memcpy(dataInfo.dataVersion, version, 4);
}
static int32_t indexes[_NORM_INDEX_TOP]={ 0 };
typedef void EnumTrieFn(void *context, uint32_t code, Norm *norm);
static UNewTrie
*normTrie,
*norm32Trie,
*fcdTrie,
*auxTrie;
static UToolMemory *normMem, *utf32Mem, *extraMem, *combiningTriplesMem;
static Norm *norms;
static uint32_t haveSeenFlags[256];
static USet *nfdQCNoSet;
static uint32_t combiningCPs[2000];
static uint16_t combiningIndexes[2000];
static uint16_t combineFwdTop=0, combineBothTop=0, combineBackTop=0;
typedef struct CombiningTriple {
uint16_t leadIndex, trailIndex;
uint32_t lead, trail, combined;
} CombiningTriple;
static uint16_t combiningTable[0x8000];
static uint16_t combiningTableTop=0;
#define _NORM_MAX_SET_SEARCH_TABLE_LENGTH 0x4000
static uint16_t canonStartSets[_NORM_MAX_CANON_SETS+2*_NORM_MAX_SET_SEARCH_TABLE_LENGTH
+10000];
static int32_t canonStartSetsTop=_NORM_SET_INDEX_TOP;
static int32_t canonSetsCount=0;
extern void
init() {
uint16_t *p16;
normTrie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
uprv_memset(normTrie, 0, sizeof(UNewTrie));
norm32Trie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
uprv_memset(norm32Trie, 0, sizeof(UNewTrie));
fcdTrie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
uprv_memset(fcdTrie, 0, sizeof(UNewTrie));
auxTrie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
uprv_memset(auxTrie, 0, sizeof(UNewTrie));
if(NULL==utrie_open(normTrie, NULL, 30000, 0, 0, FALSE)) {
fprintf(stderr, "error: failed to initialize tries\n");
exit(U_MEMORY_ALLOCATION_ERROR);
}
normMem=utm_open("gennorm normalization structs", 20000, 20000, sizeof(Norm));
norms=utm_alloc(normMem);
utf32Mem=utm_open("gennorm UTF-32 strings", 30000, 30000, 4);
uprv_memset(haveSeenFlags, 0, sizeof(haveSeenFlags));
nfdQCNoSet=uset_open(1, 0);
extraMem=utm_open("gennorm extra 16-bit memory", _NORM_EXTRA_INDEX_TOP, _NORM_EXTRA_INDEX_TOP, 2);
p16=(uint16_t *)utm_alloc(extraMem);
*p16=1;
combiningTriplesMem=utm_open("gennorm combining triples", 0x4000, 0x4000, sizeof(CombiningTriple));
indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]=0xffff;
indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]=0xffff;
indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]=0xffff;
indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]=0xffff;
uprv_memset(canonStartSets, 0, _NORM_SET_INDEX_TOP*2);
}
static Norm *
createNorm(uint32_t code) {
Norm *p;
uint32_t i;
i=utrie_get32(normTrie, (UChar32)code, NULL);
if(i!=0) {
p=norms+i;
} else {
p=(Norm *)utm_alloc(normMem);
if(!utrie_set32(normTrie, (UChar32)code, (uint32_t)(p-norms))) {
fprintf(stderr, "error: too many normalization entries\n");
exit(U_BUFFER_OVERFLOW_ERROR);
}
}
return p;
}
static Norm *
getNorm(uint32_t code) {
uint32_t i;
i=utrie_get32(normTrie, (UChar32)code, NULL);
if(i==0) {
return NULL;
}
return norms+i;
}
static uint8_t
getCCFromCP(uint32_t code) {
Norm *norm=getNorm(code);
if(norm==NULL) {
return 0;
} else {
return norm->udataCC;
}
}
static uint32_t
enumTrie(EnumTrieFn *fn, void *context) {
uint32_t count, i;
UChar32 code;
UBool isInBlockZero;
count=0;
for(code=0; code<=0x10ffff;) {
i=utrie_get32(normTrie, code, &isInBlockZero);
if(isInBlockZero) {
code+=UTRIE_DATA_BLOCK_LENGTH;
} else {
if(i!=0) {
fn(context, (uint32_t)code, norms+i);
++count;
}
++code;
}
}
return count;
}
static void
setHaveSeenString(const uint32_t *s, int32_t length) {
uint32_t c;
while(length>0) {
c=*s++;
haveSeenFlags[(c>>5)&0xff]|=(1<<(c&0x1f));
--length;
}
}
#define HAVE_SEEN(c) (haveSeenFlags[((c)>>5)&0xff]&(1<<((c)&0x1f)))
static void
addCombiningCP(uint32_t code, uint8_t flags) {
uint32_t newEntry;
uint16_t i;
newEntry=code|((uint32_t)flags<<24);
for(i=0; i<combineBackTop; ++i) {
if(code==(combiningCPs[i]&0xffffff)) {
if(newEntry==combiningCPs[i]) {
return;
}
newEntry|=combiningCPs[i];
if(i!=--combineBackTop) {
uprv_memmove(combiningCPs+i, combiningCPs+i+1, (combineBackTop-i)*4);
}
if(i<combineBothTop) {
--combineBothTop;
}
if(i<combineFwdTop) {
--combineFwdTop;
}
break;
}
}
if(combineBackTop>=sizeof(combiningCPs)/4) {
fprintf(stderr, "error: gennorm combining code points - trying to use more than %ld units\n",
(long)(sizeof(combiningCPs)/4));
exit(U_MEMORY_ALLOCATION_ERROR);
}
flags=(uint8_t)(newEntry>>24);
if(flags==1) {
i=combineFwdTop++;
++combineBothTop;
} else if(flags==3) {
i=combineBothTop++;
} else {
i=combineBackTop;
}
if(i<combineBackTop) {
uprv_memmove(combiningCPs+i+1, combiningCPs+i, (combineBackTop-i)*4);
}
combiningCPs[i]=newEntry;
++combineBackTop;
}
static uint16_t
findCombiningCP(uint32_t code, UBool isLead) {
uint16_t i, limit;
if(isLead) {
i=0;
limit=combineBothTop;
} else {
i=combineFwdTop;
limit=combineBackTop;
}
for(; i<limit; ++i) {
if(code==(combiningCPs[i]&0xffffff)) {
return i;
}
}
return 0xffff;
}
static void
addCombiningTriple(uint32_t lead, uint32_t trail, uint32_t combined) {
CombiningTriple *triple;
createNorm(lead)->combiningFlags|=1;
createNorm(trail)->combiningFlags|=2;
addCombiningCP(lead, 1);
addCombiningCP(trail, 2);
triple=(CombiningTriple *)utm_alloc(combiningTriplesMem);
triple->lead=lead;
triple->trail=trail;
triple->combined=combined;
}
static int
compareTriples(const void *l, const void *r) {
int diff;
diff=(int)((CombiningTriple *)l)->leadIndex-
(int)((CombiningTriple *)r)->leadIndex;
if(diff==0) {
diff=(int)((CombiningTriple *)l)->trailIndex-
(int)((CombiningTriple *)r)->trailIndex;
}
return diff;
}
static void
processCombining() {
CombiningTriple *triples;
uint16_t *p;
uint32_t combined;
uint16_t i, j, count, tableTop, finalIndex, combinesFwd;
triples=utm_getStart(combiningTriplesMem);
count=(uint16_t)utm_countItems(combiningTriplesMem);
for(i=0; i<count; ++i) {
triples[i].leadIndex=findCombiningCP(triples[i].lead, TRUE);
triples[i].trailIndex=findCombiningCP(triples[i].trail, FALSE);
}
qsort(triples, count, sizeof(CombiningTriple), compareTriples);
tableTop=0;
j=0;
for(i=0; i<combineBothTop; ++i) {
createNorm(combiningCPs[i]&0xffffff)->combiningIndex=combiningIndexes[i]=tableTop;
while(j<count && i==triples[j].leadIndex) {
combined=triples[j++].combined;
if(combined<=0x1fff) {
tableTop+=2;
} else {
tableTop+=3;
}
}
}
finalIndex=tableTop;
for(; i<combineBackTop; ++i) {
createNorm(combiningCPs[i]&0xffffff)->combiningIndex=combiningIndexes[i]=finalIndex++;
}
if(finalIndex>0x8000) {
fprintf(stderr, "error: gennorm combining table - trying to use %u units, more than the %ld units available\n",
tableTop, (long)(sizeof(combiningTable)/4));
exit(U_MEMORY_ALLOCATION_ERROR);
}
combiningTableTop=tableTop;
p=combiningTable;
j=0;
for(i=0; i<combineBothTop; ++i) {
combined=0;
while(j<count && i==triples[j].leadIndex) {
finalIndex=combiningIndexes[triples[j].trailIndex];
combined=triples[j++].combined;
combinesFwd=(uint16_t)((getNorm(combined)->combiningFlags&1)<<13);
*p++=finalIndex;
if(combined<=0x1fff) {
*p++=(uint16_t)(combinesFwd|combined);
} else if(combined<=0xffff) {
*p++=(uint16_t)(0x8000|combinesFwd);
*p++=(uint16_t)combined;
} else {
*p++=(uint16_t)(0xc000|combinesFwd|((combined-0x10000)>>10));
*p++=(uint16_t)(0xdc00|(combined&0x3ff));
}
}
if(combined<=0x1fff) {
*(p-2)|=0x8000;
} else {
*(p-3)|=0x8000;
}
}
}
static void
getHangulDecomposition(uint32_t c, Norm *pHangulNorm, uint32_t hangulBuffer[3]) {
uint32_t c2;
uint8_t length;
uprv_memset(pHangulNorm, 0, sizeof(Norm));
c-=HANGUL_BASE;
c2=c%JAMO_T_COUNT;
c/=JAMO_T_COUNT;
if(c2>0) {
hangulBuffer[2]=JAMO_T_BASE+c2;
length=3;
} else {
hangulBuffer[2]=0;
length=2;
}
hangulBuffer[1]=JAMO_V_BASE+c%JAMO_V_COUNT;
hangulBuffer[0]=JAMO_L_BASE+c/JAMO_V_COUNT;
pHangulNorm->nfd=pHangulNorm->nfkd=hangulBuffer;
pHangulNorm->lenNFD=pHangulNorm->lenNFKD=length;
}
static void
decompStoreNewNF(uint32_t code, Norm *norm) {
uint32_t nfd[40], nfkd[40], hangulBuffer[3];
Norm hangulNorm;
uint32_t *s32;
Norm *p;
uint32_t c;
int32_t i, length;
uint8_t lenNFD=0, lenNFKD=0;
UBool changedNFD=FALSE, changedNFKD=FALSE;
if((length=norm->lenNFD)!=0) {
changedNFD=TRUE;
s32=norm->nfd;
} else if((length=norm->lenNFKD)!=0) {
changedNFKD=TRUE;
s32=norm->nfkd;
} else {
return;
}
for(i=0; i<length; ++i) {
c=s32[i];
p=getNorm(c);
if(p==NULL) {
if(HANGUL_BASE<=c && c<(HANGUL_BASE+HANGUL_COUNT)) {
getHangulDecomposition(c, &hangulNorm, hangulBuffer);
p=&hangulNorm;
} else {
nfd[lenNFD++]=c;
nfkd[lenNFKD++]=c;
continue;
}
}
if(changedNFD) {
if(p->lenNFD!=0) {
uprv_memcpy(nfd+lenNFD, p->nfd, p->lenNFD*4);
lenNFD+=p->lenNFD;
} else {
nfd[lenNFD++]=c;
}
}
if(p->lenNFKD!=0) {
uprv_memcpy(nfkd+lenNFKD, p->nfkd, p->lenNFKD*4);
lenNFKD+=p->lenNFKD;
changedNFKD=TRUE;
} else if(p->lenNFD!=0) {
uprv_memcpy(nfkd+lenNFKD, p->nfd, p->lenNFD*4);
lenNFKD+=p->lenNFD;
changedNFKD=TRUE;
} else {
nfkd[lenNFKD++]=c;
}
}
if(norm->lenNFD==2 && !(norm->combiningFlags&0x80)) {
addCombiningTriple(s32[0], s32[1], code);
}
if(changedNFD) {
if(lenNFD!=0) {
s32=utm_allocN(utf32Mem, lenNFD);
uprv_memcpy(s32, nfd, lenNFD*4);
} else {
s32=NULL;
}
norm->lenNFD=lenNFD;
norm->nfd=s32;
setHaveSeenString(nfd, lenNFD);
}
if(changedNFKD) {
if(lenNFKD!=0) {
s32=utm_allocN(utf32Mem, lenNFKD);
uprv_memcpy(s32, nfkd, lenNFKD*4);
} else {
s32=NULL;
}
norm->lenNFKD=lenNFKD;
norm->nfkd=s32;
setHaveSeenString(nfkd, lenNFKD);
}
}
typedef struct DecompSingle {
uint32_t c;
Norm *norm;
} DecompSingle;
static void
decompWithSingleFn(void *context, uint32_t code, Norm *norm) {
uint32_t nfd[40], nfkd[40];
uint32_t *s32;
DecompSingle *me=(DecompSingle *)context;
uint32_t c, myC;
int32_t i, length;
uint8_t lenNFD=0, lenNFKD=0, myLenNFD, myLenNFKD;
UBool changedNFD=FALSE, changedNFKD=FALSE;
myC=me->c;
myLenNFD=me->norm->lenNFD;
myLenNFKD=me->norm->lenNFKD;
if((length=norm->lenNFD)!=0 && myLenNFD!=0) {
s32=norm->nfd;
for(i=0; i<length; ++i) {
c=s32[i];
if(c==myC) {
uprv_memcpy(nfd+lenNFD, me->norm->nfd, myLenNFD*4);
lenNFD+=myLenNFD;
changedNFD=TRUE;
} else {
nfd[lenNFD++]=c;
}
}
}
if((length=norm->lenNFKD)!=0) {
s32=norm->nfkd;
for(i=0; i<length; ++i) {
c=s32[i];
if(c==myC) {
if(myLenNFKD!=0) {
uprv_memcpy(nfkd+lenNFKD, me->norm->nfkd, myLenNFKD*4);
lenNFKD+=myLenNFKD;
} else {
uprv_memcpy(nfkd+lenNFKD, me->norm->nfd, myLenNFD*4);
lenNFKD+=myLenNFD;
}
changedNFKD=TRUE;
} else {
nfkd[lenNFKD++]=c;
}
}
} else if((length=norm->lenNFD)!=0 && myLenNFKD!=0) {
s32=norm->nfd;
for(i=0; i<length; ++i) {
c=s32[i];
if(c==myC) {
uprv_memcpy(nfkd+lenNFKD, me->norm->nfkd, myLenNFKD*4);
lenNFKD+=myLenNFKD;
changedNFKD=TRUE;
} else {
nfkd[lenNFKD++]=c;
}
}
}
if(changedNFD) {
if(lenNFD!=0) {
if(lenNFD>norm->lenNFD) {
s32=utm_allocN(utf32Mem, lenNFD);
} else {
s32=norm->nfd;
}
uprv_memcpy(s32, nfd, lenNFD*4);
} else {
s32=NULL;
}
norm->lenNFD=lenNFD;
norm->nfd=s32;
}
if(changedNFKD) {
if(lenNFKD!=0) {
if(lenNFKD>norm->lenNFKD) {
s32=utm_allocN(utf32Mem, lenNFKD);
} else {
s32=norm->nfkd;
}
uprv_memcpy(s32, nfkd, lenNFKD*4);
} else {
s32=NULL;
}
norm->lenNFKD=lenNFKD;
norm->nfkd=s32;
}
}
extern void
storeNorm(uint32_t code, Norm *norm) {
DecompSingle decompSingle;
Norm *p;
p=createNorm(code);
norm->qcFlags=p->qcFlags;
norm->combiningFlags=p->combiningFlags;
norm->fncIndex=p->fncIndex;
if((norm->lenNFD|norm->lenNFKD)!=0) {
decompStoreNewNF(code, norm);
if(HAVE_SEEN(code)) {
decompSingle.c=code;
decompSingle.norm=norm;
enumTrie(decompWithSingleFn, &decompSingle);
}
}
uprv_memcpy(p, norm, sizeof(Norm));
}
extern void
setQCFlags(uint32_t code, uint8_t qcFlags) {
createNorm(code)->qcFlags|=qcFlags;
if(code<0xffff) {
if((qcFlags&_NORM_QC_NFC) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]) {
indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]=(uint16_t)code;
}
if((qcFlags&_NORM_QC_NFKC) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]) {
indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]=(uint16_t)code;
}
if((qcFlags&_NORM_QC_NFD) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]) {
indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]=(uint16_t)code;
}
if((qcFlags&_NORM_QC_NFKD) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]) {
indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]=(uint16_t)code;
}
}
if(qcFlags&_NORM_QC_NFD) {
uset_add(nfdQCNoSet, (UChar32)code);
}
}
extern void
setCompositionExclusion(uint32_t code) {
createNorm(code)->combiningFlags|=0x80;
}
static void
setHangulJamoSpecials() {
Norm *norm;
uint32_t c, hangul;
hangul=0xac00;
for(c=0x1100; c<=0x1112; ++c) {
norm=createNorm(c);
norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_JAMO_L;
norm->combiningFlags=1;
norm->canonStart=uset_open(hangul, hangul+21*28-1);
hangul+=21*28;
}
for(c=0x1161; c<=0x1175; ++c) {
norm=createNorm(c);
norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_JAMO_V;
norm->combiningFlags=2;
norm->unsafeStart=TRUE;
}
for(c=0x11a8; c<=0x11c2; ++c) {
norm=createNorm(c);
norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_JAMO_T;
norm->combiningFlags=2;
norm->unsafeStart=TRUE;
}
norm=(Norm *)utm_alloc(normMem);
norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_HANGUL;
norm->qcFlags=_NORM_QC_NFD|_NORM_QC_NFKD;
if(!utrie_setRange32(normTrie, 0xac00, 0xd7a4, (uint32_t)(norm-norms), TRUE)) {
fprintf(stderr, "error: too many normalization entries (setting Hangul)\n");
exit(U_BUFFER_OVERFLOW_ERROR);
}
}
U_CFUNC void
setFNC(uint32_t c, UChar *s) {
uint16_t *p;
int32_t length, i, count;
UChar first;
count=utm_countItems(extraMem);
length=s[0];
first=s[1];
if(length==1 && first<0xff00) {
p=utm_getStart(extraMem);
for(i=1; i<count; ++i) {
if(first==p[i]) {
break;
}
}
} else {
i=count;
}
if(i==count) {
if(count>_NORM_AUX_MAX_FNC) {
fprintf(stderr, "gennorm error: too many FNC strings\n");
exit(U_INDEX_OUTOFBOUNDS_ERROR);
}
s[0]=(uint16_t)(0xff00+length);
++length;
p=(uint16_t *)utm_allocN(extraMem, length);
uprv_memcpy(p, s, length*2);
count+=length;
((uint16_t *)utm_getStart(extraMem))[0]=(uint16_t)count;
}
createNorm(c)->fncIndex=i;
}
static uint16_t
reorderString(uint32_t *s, int32_t length) {
uint8_t ccs[40];
uint32_t c;
int32_t i, j;
uint8_t cc, prevCC;
if(length<=0) {
return 0;
}
for(i=0; i<length; ++i) {
c=s[i];
cc=getCCFromCP(c);
if(cc!=0 && i!=0) {
j=i;
do {
prevCC=ccs[j-1];
if(prevCC<=cc) {
break;
}
s[j]=s[j-1];
ccs[j]=prevCC;
} while(--j!=0);
s[j]=c;
ccs[j]=cc;
} else {
ccs[i]=cc;
}
}
return (uint16_t)(((uint16_t)ccs[0]<<8)|ccs[length-1]);
}
static UBool combineAndQC[64]={ 0 };
static void
postParseFn(void *context, uint32_t code, Norm *norm) {
int32_t length;
length=norm->lenNFD;
if(length>0) {
norm->canonBothCCs=reorderString(norm->nfd, length);
}
length=norm->lenNFKD;
if(length>0) {
norm->compatBothCCs=reorderString(norm->nfkd, length);
}
if((norm->lenNFD!=0) != ((norm->qcFlags&_NORM_QC_NFD)!=0)) {
fprintf(stderr, "gennorm warning: U+%04lx has NFD[%d] but quick check 0x%02x\n", (long)code, norm->lenNFD, norm->qcFlags);
}
if(((norm->lenNFD|norm->lenNFKD)!=0) != ((norm->qcFlags&(_NORM_QC_NFD|_NORM_QC_NFKD))!=0)) {
fprintf(stderr, "gennorm warning: U+%04lx has NFD[%d] NFKD[%d] but quick check 0x%02x\n", (long)code, norm->lenNFD, norm->lenNFKD, norm->qcFlags);
}
combineAndQC[(norm->qcFlags&0x33)|((norm->combiningFlags&3)<<2)]=1;
if(norm->combiningFlags&1) {
if(norm->udataCC!=0) {
fprintf(stderr, "gennorm warning: U+%04lx combines forward but udataCC==%u\n", (long)code, norm->udataCC);
}
}
if(norm->combiningFlags&2) {
if((norm->qcFlags&0x11)==0) {
fprintf(stderr, "gennorm warning: U+%04lx combines backward but qcNF?C==0\n", (long)code);
}
#if 0
if(norm->udataCC==0) {
printf("U+%04lx combines backward but udataCC==0\n", (long)code);
}
#endif
}
if((norm->combiningFlags&3)==3 && beVerbose) {
printf("U+%04lx combines both ways\n", (long)code);
}
length=norm->lenNFD;
if(length>0) {
Norm *otherNorm;
UChar32 c;
int32_t i;
c=norm->nfd[0];
otherNorm=createNorm(c);
if(otherNorm->canonStart==NULL) {
otherNorm->canonStart=uset_open(code, code);
if(otherNorm->canonStart==NULL) {
fprintf(stderr, "gennorm error: out of memory in uset_open()\n");
exit(U_MEMORY_ALLOCATION_ERROR);
}
} else {
uset_add(otherNorm->canonStart, code);
if(!uset_contains(otherNorm->canonStart, code)) {
fprintf(stderr, "gennorm error: uset_add(setOf(U+%4x), U+%4x)\n", (int)c, (int)code);
exit(U_INTERNAL_PROGRAM_ERROR);
}
}
for(i=1; i<length; ++i) {
createNorm(norm->nfd[i])->unsafeStart=TRUE;
}
}
}
static uint32_t
make32BitNorm(Norm *norm) {
UChar extra[100];
const Norm *other;
uint32_t word;
int32_t i, length, beforeZero=0, count, start;
if(norm->udataCC==0) {
if((norm->qcFlags&_NORM_QC_NFC)==0 && norm->lenNFD>0) {
if( norm->canonBothCCs>=0x100 ||
((other=getNorm(norm->nfd[0]))!=NULL && (other->qcFlags&_NORM_QC_NFC)!=0)
) {
fprintf(stderr,
"error: true NFC starter canonical decomposition[%u] does not begin\n"
" with a true NFC starter: U+%04lx U+%04lx%s\n",
norm->lenNFD, (long)norm->nfd[0], (long)norm->nfd[1],
norm->lenNFD<=2 ? "" : " ...");
exit(U_INVALID_TABLE_FILE);
}
}
if((norm->qcFlags&_NORM_QC_NFKC)==0) {
if(norm->lenNFKD>0) {
if( norm->compatBothCCs>=0x100 ||
((other=getNorm(norm->nfkd[0]))!=NULL && (other->qcFlags&_NORM_QC_NFKC)!=0)
) {
fprintf(stderr,
"error: true NFKC starter compatibility decomposition[%u] does not begin\n"
" with a true NFKC starter: U+%04lx U+%04lx%s\n",
norm->lenNFKD, (long)norm->nfkd[0], (long)norm->nfkd[1], norm->lenNFKD<=2 ? "" : " ...");
exit(U_INVALID_TABLE_FILE);
}
} else if(norm->lenNFD>0) {
if( norm->canonBothCCs>=0x100 ||
((other=getNorm(norm->nfd[0]))!=NULL && (other->qcFlags&_NORM_QC_NFKC)!=0)
) {
fprintf(stderr,
"error: true NFKC starter canonical decomposition[%u] does not begin\n"
" with a true NFKC starter: U+%04lx U+%04lx%s\n",
norm->lenNFD, (long)norm->nfd[0], (long)norm->nfd[1],
norm->lenNFD<=2 ? "" : " ...");
exit(U_INVALID_TABLE_FILE);
}
}
}
}
word=norm->qcFlags;
word|=(uint32_t)norm->udataCC<<_NORM_CC_SHIFT;
if(norm->combiningFlags&3) {
word|=(uint32_t)(norm->combiningFlags&3)<<6;
}
if(norm->combiningIndex!=0) {
extra[0]=norm->combiningIndex;
beforeZero=1;
}
count=beforeZero;
if((norm->lenNFD|norm->lenNFKD)!=0) {
extra[count++]=0;
length=norm->lenNFD;
if(length>0) {
if(norm->canonBothCCs!=0) {
extra[beforeZero]|=0x80;
extra[count++]=norm->canonBothCCs;
}
start=count;
for(i=0; i<length; ++i) {
UTF_APPEND_CHAR_UNSAFE(extra, count, norm->nfd[i]);
}
extra[beforeZero]|=(UChar)(count-start);
}
length=norm->lenNFKD;
if(length>0) {
if(norm->compatBothCCs!=0) {
extra[beforeZero]|=0x8000;
extra[count++]=norm->compatBothCCs;
}
start=count;
for(i=0; i<length; ++i) {
UTF_APPEND_CHAR_UNSAFE(extra, count, norm->nfkd[i]);
}
extra[beforeZero]|=(UChar)((count-start)<<8);
}
}
if(count!=0) {
UChar *p;
if(norm->specialTag!=0) {
fprintf(stderr, "error: gennorm - illegal to have both extra data and a special tag (0x%x)\n", norm->specialTag);
exit(U_ILLEGAL_ARGUMENT_ERROR);
}
p=(UChar *)utm_allocN(extraMem, count);
uprv_memcpy(p, extra, count*2);
word|=(uint32_t)(beforeZero+(p-(UChar *)utm_getStart(extraMem)))<<_NORM_EXTRA_SHIFT;
} else if(norm->specialTag!=0) {
word|=(uint32_t)norm->specialTag<<_NORM_EXTRA_SHIFT;
}
return word;
}
static void
makeAll32() {
uint32_t *pNormData;
uint32_t n;
int32_t i, normLength, count;
count=(int32_t)utm_countItems(normMem);
for(i=0; i<count; ++i) {
norms[i].value32=make32BitNorm(norms+i);
}
pNormData=utrie_getData(norm32Trie, &normLength);
count=0;
for(i=0; i<normLength; ++i) {
n=pNormData[i];
if(0!=(pNormData[i]=norms[n].value32)) {
++count;
}
}
}
static void
makeFCD() {
uint32_t *pFCDData;
uint32_t n;
int32_t i, count, fcdLength;
uint16_t bothCCs;
count=utm_countItems(normMem);
for(i=0; i<count; ++i) {
bothCCs=norms[i].canonBothCCs;
if(bothCCs==0) {
bothCCs=norms[i].udataCC;
bothCCs|=bothCCs<<8;
}
norms[i].value32=bothCCs;
}
pFCDData=utrie_getData(fcdTrie, &fcdLength);
for(i=0; i<fcdLength; ++i) {
n=pFCDData[i];
pFCDData[i]=norms[n].value32;
}
}
static int32_t
usetContainsOne(const USet* set) {
if (uset_size(set) == 1) {
UChar32 start, end;
UErrorCode ec = U_ZERO_ERROR;
int32_t len = uset_getItem(set, 0, &start, &end, NULL, 0, &ec);
if (len == 0) return start;
}
return -1;
}
static void
makeCanonSetFn(void *context, uint32_t code, Norm *norm) {
if(norm->canonStart!=NULL && !uset_isEmpty(norm->canonStart)) {
uint16_t *table;
int32_t c, tableLength;
UErrorCode errorCode=U_ZERO_ERROR;
c=usetContainsOne(norm->canonStart);
if(code<=0xffff) {
table=canonStartSets+_NORM_MAX_CANON_SETS;
tableLength=canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH];
table[tableLength++]=(uint16_t)code;
if(c>=0 && c<=0xffff && (c&_NORM_CANON_SET_BMP_MASK)!=_NORM_CANON_SET_BMP_IS_INDEX) {
table[tableLength++]=(uint16_t)c;
} else {
table[tableLength++]=(uint16_t)(_NORM_CANON_SET_BMP_IS_INDEX|canonStartSetsTop);
c=-1;
}
canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]=(uint16_t)tableLength;
} else {
table=canonStartSets+_NORM_MAX_CANON_SETS+_NORM_MAX_SET_SEARCH_TABLE_LENGTH;
tableLength=canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH];
table[tableLength++]=(uint16_t)(code>>16);
table[tableLength++]=(uint16_t)code;
if(c>=0) {
table[tableLength-2]|=(uint16_t)(0x8000|((c>>8)&0x1f00));
table[tableLength++]=(uint16_t)c;
} else {
table[tableLength++]=(uint16_t)canonStartSetsTop;
}
canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]=(uint16_t)tableLength;
}
if(c<0) {
++canonSetsCount;
canonStartSetsTop+=
uset_serialize(norm->canonStart,
canonStartSets+canonStartSetsTop,
_NORM_MAX_CANON_SETS-canonStartSetsTop,
&errorCode);
}
canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]=(uint16_t)canonStartSetsTop;
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gennorm error: uset_serialize()->%s (canonStartSetsTop=%d)\n", u_errorName(errorCode), (int)canonStartSetsTop);
exit(errorCode);
}
if(tableLength>_NORM_MAX_SET_SEARCH_TABLE_LENGTH) {
fprintf(stderr, "gennorm error: search table for canonical starter sets too long\n");
exit(U_INDEX_OUTOFBOUNDS_ERROR);
}
}
}
static int32_t
combine(uint32_t lead, uint32_t trail) {
CombiningTriple *triples;
uint32_t i, count;
triples=utm_getStart(combiningTriplesMem);
count=utm_countItems(combiningTriplesMem);
for(i=0; i<count && lead!=triples[i].lead; ++i) {}
for(; i<count && lead==triples[i].lead; ++i) {
if(trail==triples[i].trail) {
return (int32_t)triples[i].combined;
}
}
return -1;
}
static UBool
doesComposeConsume(const uint32_t *s, int32_t length, uint32_t c, uint8_t cc) {
int32_t starter, i;
while(length>1 && cc<getCCFromCP(s[length-1])) {
--length;
}
starter=(int32_t)s[0];
for(i=1; i<length; ++i) {
starter=combine((uint32_t)starter, s[i]);
if(starter<0) {
fprintf(stderr, "error: unable to consume normal decomposition in doesComposeConsume(<%04x, %04x, ...>[%d], U+%04x, %u)\n",
(int)s[0], (int)s[1], (int)length, (int)c, cc);
exit(U_INTERNAL_PROGRAM_ERROR);
}
}
return combine((uint32_t)starter, c)>=0;
}
static UBool
canChangeWithFollowing(const uint32_t *s, int32_t length, uint8_t trailCC) {
if(trailCC<=1) {
return FALSE;
}
if(length==2) {
CombiningTriple *triples;
uint32_t c, i, count;
uint8_t cc;
triples=utm_getStart(combiningTriplesMem);
count=utm_countItems(combiningTriplesMem);
c=s[0];
for(i=0; i<count && c!=triples[i].lead; ++i) {}
for(; i<count && c==triples[i].lead; ++i) {
cc=getCCFromCP(triples[i].trail);
if(cc>0 && cc<trailCC) {
return TRUE;
}
}
} else {
uint32_t c2;
uint16_t i;
uint8_t cc;
for(i=combineBothTop; i<combineBackTop; ++i) {
c2=combiningCPs[i]&0xffffff;
cc=getCCFromCP(c2);
if(cc>0 && cc<trailCC && doesComposeConsume(s, length-1, c2, cc)) {
return TRUE;
}
}
}
return FALSE;
}
static uint32_t
getSkippableFlags(const Norm *norm) {
if(norm->specialTag==_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_HANGUL) {
return 0;
}
if(norm->udataCC!=0) {
return 0;
}
if( (norm->qcFlags&(_NORM_QC_NFC&_NORM_QC_ANY_NO))!=0 ||
(norm->combiningFlags&3)!=0) {
return 0;
}
if(norm->lenNFD!=0 && canChangeWithFollowing(norm->nfd, norm->lenNFD, (uint8_t)norm->canonBothCCs)) {
return _NORM_AUX_NFC_SKIP_F_MASK;
}
return 0;
}
static void
makeAux() {
Norm *norm;
uint32_t *pData;
int32_t i, length;
pData=utrie_getData(auxTrie, &length);
for(i=0; i<length; ++i) {
norm=norms+pData[i];
pData[i]=
((uint32_t)(norm->combiningFlags&0x80)<<(_NORM_AUX_COMP_EX_SHIFT-7))|
(uint32_t)norm->fncIndex;
if(norm->unsafeStart || norm->udataCC!=0) {
pData[i]|=_NORM_AUX_UNSAFE_MASK;
}
pData[i]|=getSkippableFlags(norm);
}
}
static uint32_t U_CALLCONV
getFoldedNormValue(UNewTrie *trie, UChar32 start, int32_t offset) {
uint32_t value, leadNorm32=0;
UChar32 limit;
UBool inBlockZero;
limit=start+0x400;
while(start<limit) {
value=utrie_get32(trie, start, &inBlockZero);
if(inBlockZero) {
start+=UTRIE_DATA_BLOCK_LENGTH;
} else {
if(value!=0) {
leadNorm32|=value;
}
++start;
}
}
if(leadNorm32&_NORM_CC_MASK) {
leadNorm32|=_NORM_CC_MASK;
}
leadNorm32&=~((uint32_t)0xffffffff<<_NORM_EXTRA_SHIFT);
if(leadNorm32==0) {
return 0;
}
leadNorm32|=(
(uint32_t)_NORM_EXTRA_INDEX_TOP+
(uint32_t)((offset-UTRIE_BMP_INDEX_LENGTH)>>UTRIE_SURROGATE_BLOCK_BITS)
)<<_NORM_EXTRA_SHIFT;
return leadNorm32;
}
static uint32_t U_CALLCONV
getFoldedFCDValue(UNewTrie *trie, UChar32 start, int32_t offset) {
uint32_t value;
UChar32 limit;
UBool inBlockZero;
limit=start+0x400;
while(start<limit) {
value=utrie_get32(trie, start, &inBlockZero);
if(inBlockZero) {
start+=UTRIE_DATA_BLOCK_LENGTH;
} else if(value!=0) {
return (uint32_t)offset;
} else {
++start;
}
}
return 0;
}
static uint32_t U_CALLCONV
getFoldedAuxValue(UNewTrie *trie, UChar32 start, int32_t offset) {
uint32_t value, oredValues;
UChar32 limit;
UBool inBlockZero;
oredValues=0;
limit=start+0x400;
while(start<limit) {
value=utrie_get32(trie, start, &inBlockZero);
if(inBlockZero) {
start+=UTRIE_DATA_BLOCK_LENGTH;
} else {
oredValues|=value;
++start;
}
}
if(oredValues!=0) {
offset>>=UTRIE_SURROGATE_BLOCK_BITS;
if(offset>_NORM_AUX_FNC_MASK) {
fprintf(stderr, "gennorm error: folding offset too large (auxTrie)\n");
exit(U_INDEX_OUTOFBOUNDS_ERROR);
}
return (uint32_t)offset|(oredValues&~_NORM_AUX_FNC_MASK);
} else {
return 0;
}
}
extern void
processData() {
#if 0
uint16_t i;
#endif
processCombining();
enumTrie(postParseFn, NULL);
#if 0
for(i=1; i<64; ++i) {
if(combineAndQC[i]) {
printf("combiningFlags==0x%02x qcFlags(NF?C)==0x%02x\n", (i&0xc)>>2, i&0x33);
}
}
#endif
setHangulJamoSpecials();
enumTrie(makeCanonSetFn, NULL);
if( NULL==utrie_clone(norm32Trie, normTrie, NULL, 0) ||
NULL==utrie_clone(fcdTrie, normTrie, NULL, 0) ||
NULL==utrie_clone(auxTrie, normTrie, NULL, 0)
) {
fprintf(stderr, "error: unable to clone the normalization trie\n");
exit(U_MEMORY_ALLOCATION_ERROR);
}
makeAll32();
makeFCD();
makeAux();
if(beVerbose) {
#if 0
printf("number of stage 2 entries: %ld\n", stage2Mem->index);
printf("size of stage 1 (BMP) & 2 (uncompacted) + extra data: %ld bytes\n", _NORM_STAGE_1_BMP_COUNT*2+stage2Mem->index*4+extraMem->index*2);
#endif
printf("combining CPs tops: fwd %u both %u back %u\n", combineFwdTop, combineBothTop, combineBackTop);
printf("combining table count: %u\n", combiningTableTop);
}
}
#endif
extern void
generateData(const char *dataDir) {
static uint8_t normTrieBlock[100000], fcdTrieBlock[100000], auxTrieBlock[100000];
UNewDataMemory *pData;
UErrorCode errorCode=U_ZERO_ERROR;
int32_t size, dataLength;
#if UCONFIG_NO_NORMALIZATION
size=0;
#else
U_STRING_DECL(nxCJKCompatPattern, "[:Ideographic:]", 15);
U_STRING_DECL(nxUnicode32Pattern, "[:^Age=3.2:]", 12);
USet *set;
int32_t normTrieSize, fcdTrieSize, auxTrieSize;
normTrieSize=utrie_serialize(norm32Trie, normTrieBlock, sizeof(normTrieBlock), getFoldedNormValue, FALSE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: utrie_serialize(normalization properties) failed, %s\n", u_errorName(errorCode));
exit(errorCode);
}
fcdTrieSize=utrie_serialize(fcdTrie, fcdTrieBlock, sizeof(fcdTrieBlock), getFoldedFCDValue, TRUE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: utrie_serialize(FCD data) failed, %s\n", u_errorName(errorCode));
exit(errorCode);
}
auxTrieSize=utrie_serialize(auxTrie, auxTrieBlock, sizeof(auxTrieBlock), getFoldedAuxValue, TRUE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: utrie_serialize(auxiliary data) failed, %s\n", u_errorName(errorCode));
exit(errorCode);
}
if(canonStartSetsTop<_NORM_MAX_CANON_SETS) {
uprv_memmove(canonStartSets+canonStartSetsTop,
canonStartSets+_NORM_MAX_CANON_SETS,
canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]*2);
}
canonStartSetsTop+=canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH];
if(canonStartSetsTop<(_NORM_MAX_CANON_SETS+_NORM_MAX_SET_SEARCH_TABLE_LENGTH)) {
uprv_memmove(canonStartSets+canonStartSetsTop,
canonStartSets+_NORM_MAX_CANON_SETS+_NORM_MAX_SET_SEARCH_TABLE_LENGTH,
canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]*2);
}
canonStartSetsTop+=canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH];
U_STRING_INIT(nxCJKCompatPattern, "[:Ideographic:]", 15);
U_STRING_INIT(nxUnicode32Pattern, "[:^Age=3.2:]", 12);
canonStartSets[_NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET]=canonStartSetsTop;
set=uset_openPattern(nxCJKCompatPattern, -1, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: uset_openPattern([:Ideographic:]&[:NFD_QC=No:]) failed, %s\n", u_errorName(errorCode));
exit(errorCode);
}
uset_retainAll(set, nfdQCNoSet);
canonStartSetsTop+=uset_serialize(set, canonStartSets+canonStartSetsTop, LENGTHOF(canonStartSets)-canonStartSetsTop, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: uset_serialize([:Ideographic:]&[:NFD_QC=No:]) failed, %s\n", u_errorName(errorCode));
exit(errorCode);
}
uset_close(set);
canonStartSets[_NORM_SET_INDEX_NX_UNICODE32_OFFSET]=canonStartSetsTop;
set=uset_openPattern(nxUnicode32Pattern, -1, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: uset_openPattern([:^Age=3.2:]) failed, %s\n", u_errorName(errorCode));
exit(errorCode);
}
canonStartSetsTop+=uset_serialize(set, canonStartSets+canonStartSetsTop, LENGTHOF(canonStartSets)-canonStartSetsTop, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "error: uset_serialize([:^Age=3.2:]) failed, %s\n", u_errorName(errorCode));
exit(errorCode);
}
uset_close(set);
canonStartSets[_NORM_SET_INDEX_NX_RESERVED_OFFSET]=canonStartSetsTop;
if((utm_countItems(extraMem)+combiningTableTop)&1) {
combiningTable[combiningTableTop++]=0x1234;
}
if(canonStartSetsTop&1) {
canonStartSets[canonStartSetsTop++]=0x1235;
}
size=
_NORM_INDEX_TOP*4+
normTrieSize+
utm_countItems(extraMem)*2+
combiningTableTop*2+
fcdTrieSize+
auxTrieSize+
canonStartSetsTop*2;
if(beVerbose) {
printf("size of normalization trie %5u bytes\n", (int)normTrieSize);
printf("size of 16-bit extra memory %5u UChars/uint16_t\n", (int)utm_countItems(extraMem));
printf(" of that: FC_NFKC_Closure size %5u UChars/uint16_t\n", ((uint16_t *)utm_getStart(extraMem))[0]);
printf("size of combining table %5u uint16_t\n", combiningTableTop);
printf("size of FCD trie %5u bytes\n", (int)fcdTrieSize);
printf("size of auxiliary trie %5u bytes\n", (int)auxTrieSize);
printf("size of canonStartSets[] %5u uint16_t\n", (int)canonStartSetsTop);
printf(" number of indexes %5u uint16_t\n", _NORM_SET_INDEX_TOP);
printf(" size of sets %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]-_NORM_SET_INDEX_TOP);
printf(" number of sets %5d\n", (int)canonSetsCount);
printf(" size of BMP search table %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]);
printf(" size of supplementary search table %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]);
printf(" length of exclusion sets %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_NX_RESERVED_OFFSET]-canonStartSets[_NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET]);
printf("size of " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE " contents: %ld bytes\n", (long)size);
}
indexes[_NORM_INDEX_TRIE_SIZE]=normTrieSize;
indexes[_NORM_INDEX_UCHAR_COUNT]=(uint16_t)utm_countItems(extraMem);
indexes[_NORM_INDEX_COMBINE_DATA_COUNT]=combiningTableTop;
indexes[_NORM_INDEX_COMBINE_FWD_COUNT]=combineFwdTop;
indexes[_NORM_INDEX_COMBINE_BOTH_COUNT]=(uint16_t)(combineBothTop-combineFwdTop);
indexes[_NORM_INDEX_COMBINE_BACK_COUNT]=(uint16_t)(combineBackTop-combineBothTop);
indexes[_NORM_INDEX_FCD_TRIE_SIZE]=fcdTrieSize;
indexes[_NORM_INDEX_AUX_TRIE_SIZE]=auxTrieSize;
indexes[_NORM_INDEX_CANON_SET_COUNT]=canonStartSetsTop;
#endif
pData=udata_create(dataDir, DATA_TYPE, DATA_NAME, &dataInfo,
haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gennorm: unable to create the output file, error %d\n", errorCode);
exit(errorCode);
}
#if !UCONFIG_NO_NORMALIZATION
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, normTrieBlock, normTrieSize);
udata_writeBlock(pData, utm_getStart(extraMem), utm_countItems(extraMem)*2);
udata_writeBlock(pData, combiningTable, combiningTableTop*2);
udata_writeBlock(pData, fcdTrieBlock, fcdTrieSize);
udata_writeBlock(pData, auxTrieBlock, auxTrieSize);
udata_writeBlock(pData, canonStartSets, canonStartSetsTop*2);
#endif
dataLength=udata_finish(pData, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gennorm: error %d writing the output file\n", errorCode);
exit(errorCode);
}
if(dataLength!=size) {
fprintf(stderr, "gennorm error: data length %ld != calculated size %ld\n",
(long)dataLength, (long)size);
exit(U_INTERNAL_PROGRAM_ERROR);
}
}
#if !UCONFIG_NO_NORMALIZATION
extern void
cleanUpData(void) {
int32_t i, count;
count=utm_countItems(normMem);
for(i=0; i<count; ++i) {
uset_close(norms[i].canonStart);
}
utm_close(normMem);
utm_close(utf32Mem);
utm_close(extraMem);
utm_close(combiningTriplesMem);
utrie_close(normTrie);
utrie_close(norm32Trie);
utrie_close(fcdTrie);
utrie_close(auxTrie);
uset_close(nfdQCNoSet);
uprv_free(normTrie);
uprv_free(norm32Trie);
uprv_free(fcdTrie);
uprv_free(auxTrie);
}
#endif